def setUp(self):
self.sess = helpers.make_session_from_config()
# Create test collection
self.test_coll = self.sess.collections.create(self.test_coll_path)
# Create test object
helpers.make_object(self.sess, self.test_obj_path, self.content_str)
def test_query_with_in_condition(self):
collection = self.coll_path
filename = 'test_query_id_in_list.txt'
file_path = '{collection}/{filename}'.format(**locals())
obj1 = helpers.make_object(self.sess, file_path+'-1')
obj2 = helpers.make_object(self.sess, file_path+'-2')
ids = [x.id for x in (obj1,obj2)]
for number in range(3): # slice for empty(:0), first(:1) or both(:2)
search_tuple = (ids[:number] if number >= 1 else [0] + ids[:number])
q = self.sess.query(DataObject.name).filter(In( DataObject.id, search_tuple ))
self.assertEqual (number, rows_returned(q))
def test_copy_existing_obj_to_relative_dest_fails_irods4796(self):
if self.sess.server_version <= (4, 2, 7):
self.skipTest('iRODS servers <= 4.2.7 will give nondescriptive error')
obj_name = 'this_object_will_exist_once_made'
exists_path = '{}/{}'.format(self.coll_path, obj_name)
helpers.make_object(self.sess, exists_path)
self.assertTrue(self.sess.data_objects.exists(exists_path))
non_existing_zone = 'this_zone_absent'
relative_dst_path = '{non_existing_zone}/{obj_name}'.format(**locals())
options = {}
with self.assertRaises(ex.USER_INPUT_PATH_ERR):
self.sess.data_objects.copy(exists_path, relative_dst_path, **options)
def test_multiple_reads(self):
collection = self.coll_path
# make files
filenames = []
for filename in ['foo', 'bar', 'baz']:
path = '{collection}/{filename}'.format(**locals())
helpers.make_object(self.sess, path=path, content=path)
filenames.append(path)
# read files
for filename in filenames:
obj = self.sess.data_objects.get(filename)
with obj.open('r') as f:
self.assertEqual(f.read().decode(), obj.path)
def test_multiple_reads(self):
collection = self.coll_path
# make files
filenames = []
for filename in ['foo', 'bar', 'baz']:
path = '{collection}/{filename}'.format(**locals())
helpers.make_object(self.sess, path=path, content=path)
filenames.append(path)
# read files
for filename in filenames:
obj = self.sess.data_objects.get(filename)
with obj.open('r') as f:
self.assertEqual(f.read().decode(), obj.path)
def test_coll_read_ticket_between_rodsusers(self):
t = None
data_objs = []
tmpfiles = []
try:
# Create ticket for read access to alice's home collection.
alice = self.login(self.alice)
tc = Ticket(alice)
home = self.irods_homedir(alice)
tc.issue('read', home.path)
# Create 'x' and 'y' in alice's home collection
data_objs = [
helpers.make_object(alice,
home.path + "/" + name,
content='abcxyz') for name in ('x', 'y')
]
with self.login(self.bob) as bob:
ts = Ticket(bob, tc.string)
ts.supply()
# Check collection access ticket allows bob to list both subobjects
self.assertEqual(len(self.list_objects(bob)), 2)
# and that we can get (and read) them properly.
for name in ('x', 'y'):
with tempfile.NamedTemporaryFile(delete=False) as tmpf:
tmpfiles += [tmpf]
bob.data_objects.get(home.path + "/" + name, tmpf.name,
**{kw.FORCE_FLAG_KW: ''})
with open(tmpf.name, 'r') as tmpread:
self.assertEqual(tmpread.read(), 'abcxyz')
td = Ticket(alice)
td.issue('read', home.path + "/x")
with self.login(self.bob) as bob:
ts = Ticket(bob, td.string)
ts.supply()
# Check data access ticket allows bob to list only one data object
self.assertEqual(len(self.list_objects(bob)), 1)
# ... and fetch that object (verifying content)
with tempfile.NamedTemporaryFile(delete=False) as tmpf:
tmpfiles += [tmpf]
bob.data_objects.get(home.path + "/x", tmpf.name,
**{kw.FORCE_FLAG_KW: ''})
with open(tmpf.name, 'r') as tmpread:
self.assertEqual(tmpread.read(), 'abcxyz')
# ... but not fetch the other data object owned by alice.
with self.assertRaises(ex.DataObjectDoesNotExist):
bob.data_objects.get(home.path + "/y")
finally:
if t: t.delete()
for d in data_objs:
d.unlink(force=True)
for file_ in tmpfiles:
os.unlink(file_.name)
alice.cleanup()
def test_list_acl(self):
# test args
collection = self.coll_path
filename = 'foo'
# get current user info
user = self.sess.users.get(
config.IRODS_USER_USERNAME, config.IRODS_SERVER_ZONE)
# make object in test collection
path = "{collection}/{filename}".format(**locals())
obj = helpers.make_object(self.sess, path)
# get object
obj = self.sess.data_objects.get(path)
# test exception
with self.assertRaises(TypeError):
self.sess.permissions.get(filename)
# get object's ACLs
# only one for now, the owner's own access
acl = self.sess.permissions.get(obj)[0]
# check values
self.assertEqual(acl.access_name, 'own')
self.assertEqual(acl.user_zone, user.zone)
self.assertEqual(acl.user_name, user.name)
# check repr()
self.assertEqual(
repr(acl), "<iRODSAccess {0} {1} {2} {3}>".format('own', path, user.name, user.zone))
# remove object
self.sess.data_objects.unlink(path)
def test_move_obj_to_coll(self):
# test args
collection = self.coll_path
new_coll_name = 'my_coll'
file_name = 'foo'
# make object in test collection
path = "{collection}/{file_name}".format(**locals())
obj = helpers.make_object(self.sess, path)
# get object id
saved_id = obj.id
# make new collection and move object to it
new_coll_path = "{collection}/{new_coll_name}".format(**locals())
new_coll = helpers.make_collection(self.sess, new_coll_path)
self.sess.data_objects.move(path, new_coll_path)
# get new object id
new_path = "{collection}/{new_coll_name}/{file_name}".format(
**locals())
obj = self.sess.data_objects.get(new_path)
# compare ids
self.assertEqual(obj.id, saved_id)
# remove new collection
new_coll.remove(recurse=True, force=True)
def setUp(self):
self.sess = helpers.make_session()
# Create test collection
self.coll_path = '/{}/home/{}/test_dir'.format(self.sess.zone, self.sess.username)
self.collection = self.sess.collections.create(self.coll_path)
self.obj_name = 'test1'
self.content_str = u'blah'
self.write_str = u'0123456789'
self.write_str1 = u'INTERRUPT'
self.test_obj_path = '{coll_path}/{obj_name}'.format(**vars(self))
# Create test object
helpers.make_object(self.sess, self.test_obj_path, self.content_str)
def test_obj_create_to_default_resource(self):
if self.sess.server_version < (4, 0, 0):
self.skipTest('For iRODS 4+')
# make another UFS resource
session = self.sess
resource_name = 'ufs'
resource_type = 'unixfilesystem'
resource_host = session.host
resource_path = '/tmp/' + resource_name
session.resources.create(resource_name, resource_type, resource_host,
resource_path)
# set default resource to new UFS resource
session.default_resource = resource_name
# test object
collection = self.coll_path
filename = 'create_def_resc_test_file'
obj_path = "{collection}/{filename}".format(**locals())
content = ''.join(random.choice(string.printable) for _ in range(1024))
# make object in test collection
obj = helpers.make_object(session, obj_path, content=content)
# get object and confirm resource
self.assertEqual(obj.replicas[0].resource_name, resource_name)
# delete obj and second resource
obj.unlink(force=True)
session.resources.remove(resource_name)
def test_rename_obj(self):
# test args
collection = self.coll_path
old_name = 'foo'
new_name = 'bar'
# make object in test collection
path = "{collection}/{old_name}".format(**locals())
obj = helpers.make_object(self.sess, path)
# for coverage
repr(obj)
for replica in obj.replicas:
repr(replica)
# get object id
saved_id = obj.id
# rename object
new_path = "{collection}/{new_name}".format(**locals())
self.sess.data_objects.move(path, new_path)
# get updated object
obj = self.sess.data_objects.get(new_path)
# compare ids
self.assertEqual(obj.id, saved_id)
# remove object
self.sess.data_objects.unlink(new_path)
def test_set_data_acl(self):
# test args
collection = self.coll_path
filename = 'foo'
# get current user info
user = self.sess.users.get(self.sess.username, self.sess.zone)
# make object in test collection
path = "{collection}/{filename}".format(**locals())
obj = helpers.make_object(self.sess, path)
# get object
obj = self.sess.data_objects.get(path)
# set permission to write
acl1 = iRODSAccess('write', path, user.name, user.zone)
self.sess.permissions.set(acl1)
# get object's ACLs
acl = self.sess.permissions.get(obj)[0] # owner's write access
# check values
self.assertEqual(acl.access_name, 'modify object')
self.assertEqual(acl.user_zone, user.zone)
self.assertEqual(acl.user_name, user.name)
# reset permission to own
acl1 = iRODSAccess('own', path, user.name, user.zone)
self.sess.permissions.set(acl1)
# remove object
self.sess.data_objects.unlink(path)
def test_obj_create_to_default_resource(self):
if self.sess.server_version < (4, 0, 0):
self.skipTest('For iRODS 4+')
# make another UFS resource
session = self.sess
resource_name = 'ufs'
resource_type = 'unixfilesystem'
resource_host = session.host
resource_path = '/tmp/' + resource_name
session.resources.create(resource_name, resource_type, resource_host, resource_path)
# set default resource to new UFS resource
session.default_resource = resource_name
# test object
collection = self.coll_path
filename = 'create_def_resc_test_file'
obj_path = "{collection}/{filename}".format(**locals())
content = ''.join(random.choice(string.printable) for _ in range(1024))
# make object in test collection
obj = helpers.make_object(session, obj_path, content=content)
# get object and confirm resource
self.assertEqual(obj.replicas[0].resource_name, resource_name)
# delete obj and second resource
obj.unlink(force=True)
session.resources.remove(resource_name)
def test_set_data_acl(self):
# test args
collection = self.coll_path
filename = 'foo'
# get current user info
user = self.sess.users.get(
config.IRODS_USER_USERNAME, config.IRODS_SERVER_ZONE)
# make object in test collection
path = "{collection}/{filename}".format(**locals())
obj = helpers.make_object(self.sess, path)
# get object
obj = self.sess.data_objects.get(path)
# set permission to write
acl1 = iRODSAccess('write', path, user.name, user.zone)
self.sess.permissions.set(acl1)
# get object's ACLs
acl = self.sess.permissions.get(obj)[0] # owner's write access
# check values
self.assertEqual(acl.access_name, 'modify object')
self.assertEqual(acl.user_zone, user.zone)
self.assertEqual(acl.user_name, user.name)
# reset permission to own
acl1 = iRODSAccess('own', path, user.name, user.zone)
self.sess.permissions.set(acl1)
# remove object
self.sess.data_objects.unlink(path)
def test_list_acl(self):
# test args
collection = self.coll_path
filename = 'foo'
# get current user info
user = self.sess.users.get(self.sess.username, self.sess.zone)
# make object in test collection
path = "{collection}/{filename}".format(**locals())
obj = helpers.make_object(self.sess, path)
# get object
obj = self.sess.data_objects.get(path)
# test exception
with self.assertRaises(TypeError):
self.sess.permissions.get(filename)
# get object's ACLs
# only one for now, the owner's own access
acl = self.sess.permissions.get(obj)[0]
# check values
self.assertEqual(acl.access_name, 'own')
self.assertEqual(acl.user_zone, user.zone)
self.assertEqual(acl.user_name, user.name)
# check repr()
self.assertEqual(
repr(acl), "<iRODSAccess own {path} {name} {zone}>".format(path=path, **vars(user)))
# remove object
self.sess.data_objects.unlink(path)
def test_rename_obj(self):
# test args
collection = self.coll_path
old_name = 'foo'
new_name = 'bar'
# make object in test collection
path = "{collection}/{old_name}".format(**locals())
obj = helpers.make_object(self.sess, path)
# for coverage
repr(obj)
for replica in obj.replicas:
repr(replica)
# get object id
saved_id = obj.id
# rename object
new_path = "{collection}/{new_name}".format(**locals())
self.sess.data_objects.move(path, new_path)
# get updated object
obj = self.sess.data_objects.get(new_path)
# compare ids
self.assertEqual(obj.id, saved_id)
# remove object
self.sess.data_objects.unlink(new_path)
def test_raw_acls__207(self):
data = helpers.make_object(self.sess,"/".join((self.coll_path,"test_obj")))
eg = eu = fg = fu = None
try:
eg = self.sess.user_groups.create ('egrp')
eu = self.sess.users.create ('edith','rodsuser')
eg.addmember(eu.name,eu.zone)
fg = self.sess.user_groups.create ('fgrp')
fu = self.sess.users.create ('frank','rodsuser')
fg.addmember(fu.name,fu.zone)
my_ownership = set([('own', self.sess.username, self.sess.zone)])
#--collection--
perms1data = [ iRODSAccess ('write',self.coll_path, eg.name, self.sess.zone),
iRODSAccess ('read', self.coll_path, fu.name, self.sess.zone)
]
for perm in perms1data: self.sess.permissions.set ( perm )
p1 = self.sess.permissions.get ( self.coll, report_raw_acls = True)
self.assertEqual(self.perms_lists_symm_diff( perms1data, p1 ), my_ownership)
#--data object--
perms2data = [ iRODSAccess ('write',data.path, fg.name, self.sess.zone),
iRODSAccess ('read', data.path, eu.name, self.sess.zone)
]
for perm in perms2data: self.sess.permissions.set ( perm )
p2 = self.sess.permissions.get ( data, report_raw_acls = True)
self.assertEqual(self.perms_lists_symm_diff( perms2data, p2 ), my_ownership)
finally:
ids_for_delete = [ u.id for u in (fu,fg,eu,eg) if u is not None ]
for u in [ iRODSUser(self.sess.users,row)
for row in self.sess.query(User).filter(In(User.id, ids_for_delete)) ]:
u.remove()
def test_move_obj_to_coll(self):
# test args
collection = self.coll_path
new_coll_name = 'my_coll'
file_name = 'foo'
# make object in test collection
path = "{collection}/{file_name}".format(**locals())
obj = helpers.make_object(self.sess, path)
# get object id
saved_id = obj.id
# make new collection and move object to it
new_coll_path = "{collection}/{new_coll_name}".format(**locals())
new_coll = helpers.make_collection(self.sess, new_coll_path)
self.sess.data_objects.move(path, new_coll_path)
# get new object id
new_path = "{collection}/{new_coll_name}/{file_name}".format(
**locals())
obj = self.sess.data_objects.get(new_path)
# compare ids
self.assertEqual(obj.id, saved_id)
# remove new collection
new_coll.remove(recurse=True, force=True)
def test_meta_repr(self):
# test obj
collection = self.coll_path
filename = 'test_meta_repr.txt'
test_obj_path = '{collection}/{filename}'.format(**locals())
# make object
obj = helpers.make_object(self.sess, test_obj_path)
# test AVU
attribute, value, units = ('test_attr', 'test_value', 'test_units')
# add metadata to test object
meta = self.sess.metadata.add(DataObject, test_obj_path,
iRODSMeta(attribute, value, units))
# get metadata
meta = self.sess.metadata.get(DataObject, test_obj_path)
id = meta[0].id
# assert
self.assertEqual(
repr(meta[0]), "<iRODSMeta {id} {attribute} {value} {units}>".format(**locals()))
# remove test object
obj.unlink(force=True)
def setUp(self):
self.sess = helpers.make_session()
# Create test collection
self.coll_path = '/{}/home/{}/test_dir'.format(self.sess.zone,
self.sess.username)
self.collection = self.sess.collections.create(self.coll_path)
self.obj_name = 'test1'
self.content_str = u'blah'
self.write_str = u'0123456789'
self.write_str1 = u'INTERRUPT'
self.test_obj_path = '{coll_path}/{obj_name}'.format(**vars(self))
# Create test object
helpers.make_object(self.sess, self.test_obj_path, self.content_str)
def test_create_with_checksum(self):
# skip if server is older than 4.2
if self.server_version < (4, 2, 0):
self.skipTest('Expects iRODS 4.2 server-side configuration')
# server config
server_config_dir = '/etc/irods'
test_re_file = os.path.join(server_config_dir, 'test.re')
server_config_file = os.path.join(
server_config_dir, 'server_config.json')
try:
with helpers.file_backed_up(server_config_file):
# make pep rule
test_rule = "acPostProcForPut { msiDataObjChksum ($objPath, 'forceChksum=', *out )}"
# write pep rule into test_re
with open(test_re_file, 'w') as f:
f.write(test_rule)
# make new server configuration with additional re file
new_server_config = self.make_new_server_config_json(
server_config_file)
# repave the existing server_config.json to add test_re
with open(server_config_file, 'w') as f:
f.write(new_server_config)
# must make a new connection for the agent to pick up the
# updated configuration
self.sess.cleanup()
# test object
collection = self.coll_path
filename = 'checksum_test_file'
obj_path = "{collection}/{filename}".format(**locals())
contents = 'blah' * 100
checksum = base64.b64encode(
hashlib.sha256(contents).digest()).decode()
# make object in test collection
options = {kw.OPR_TYPE_KW: 1} # PUT_OPR
obj = helpers.make_object(self.sess, obj_path, content=contents, options=options)
# verify object's checksum
self.assertEqual(
obj.checksum, "sha2:{checksum}".format(**locals()))
# cleanup
os.unlink(test_re_file)
except IOError as e:
# a likely fail scenario
if e.errno == 13:
self.skipTest("No permission to modify server configuration")
raise
except:
raise
def test_create_with_checksum(self):
# skip if server is older than 4.2
if self.server_version < (4, 2, 0):
self.skipTest('Expects iRODS 4.2 server-side configuration')
# server config
server_config_dir = '/etc/irods'
test_re_file = os.path.join(server_config_dir, 'test.re')
server_config_file = os.path.join(server_config_dir,
'server_config.json')
try:
with helpers.file_backed_up(server_config_file):
# make pep rule
test_rule = "acPostProcForPut { msiDataObjChksum ($objPath, 'forceChksum=', *out )}"
# write pep rule into test_re
with open(test_re_file, 'w') as f:
f.write(test_rule)
# make new server configuration with additional re file
new_server_config = self.make_new_server_config_json(
server_config_file)
# repave the existing server_config.json to add test_re
with open(server_config_file, 'w') as f:
f.write(new_server_config)
# must make a new connection for the agent to pick up the
# updated configuration
self.sess.cleanup()
# test object
collection = self.coll_path
filename = 'checksum_test_file'
obj_path = "{collection}/{filename}".format(**locals())
contents = 'blah' * 100
checksum = base64.b64encode(
hashlib.sha256(contents).digest()).decode()
# make object in test collection
obj = helpers.make_object(self.sess, obj_path, contents)
# verify object's checksum
self.assertEqual(obj.checksum,
"sha2:{checksum}".format(**locals()))
# cleanup
os.unlink(test_re_file)
except IOError as e:
# a likely fail scenario
if e.errno == 13:
self.fail("No permission to modify server configuration")
raise
except:
raise
def setUp(self):
"""Create objects for test"""
self.sess = helpers.make_session()
user = self.sess.users.get(self.sess.username)
if user.type != 'rodsadmin':
self.skipTest('''Test runnable only by rodsadmin.''')
admin = self.sess
delete_my_tickets(admin)
# Create test collection
self.coll_path = '/{}/home/{}/ticket_test_dir'.format(
admin.zone, admin.username)
self.coll = helpers.make_collection(admin, self.coll_path)
# Create anonymous test user
self.user = admin.users.create('anonymous', 'rodsuser')
self.rodsuser_params = {
'host': admin.host,
'port': admin.port,
'user': '******',
'password': '',
'zone': admin.zone
}
# make new data object in the test collection with some initialized content
self.INITIALIZED_DATA = b'1' * 16
self.data_path = '{self.coll_path}/ticketed_data'.format(**locals())
helpers.make_object(admin,
self.data_path,
content=self.INITIALIZED_DATA)
self.MODIFIED_DATA = b'2' * 16
# make new tickets for the various combinations
self.tickets = {'coll': {}, 'data': {}}
for obj_type in ('coll', 'data'):
for access in ('read', 'write'):
ticket = Ticket(admin)
self.tickets[obj_type][access] = ticket.string
ticket.issue(access, getattr(self, obj_type + '_path'))
def test_simultaneous_multiple_AVU_joins(self):
objects = []
decoys = []
try:
collection = self.coll_path
filename = 'test_multiple_AVU_joins'
file_path = '{collection}/{filename}'.format(**locals())
for x in range(3, 9):
obj = helpers.make_object(self.sess, file_path +
'-{}'.format(x)) # with metadata
objects.append(obj)
obj.metadata.add('A_meta', '1{}'.format(x))
obj.metadata.add('B_meta', '2{}'.format(x))
decoys.append(
helpers.make_object(
self.sess,
file_path + '-dummy{}'.format(x))) # without metadata
self.assertTrue(len(objects) > 0)
# -- test simple repeat of same column --
q = self.sess.query(DataObject,DataObjectMeta).\
filter(DataObjectMeta.name == 'A_meta', DataObjectMeta.value < '20').\
filter(DataObjectMeta.name == 'B_meta', DataObjectMeta.value >= '20')
self.assertTrue(rows_returned(q) == len(objects))
# -- test no-stomp of previous filter --
self.assertTrue(('B_meta',
'28') in [(x.name, x.value)
for x in objects[-1].metadata.items()])
q = self.sess.query(DataObject,DataObjectMeta).\
filter(DataObjectMeta.name == 'B_meta').filter(DataObjectMeta.value < '28').\
filter(DataObjectMeta.name == 'B_meta').filter(Like(DataObjectMeta.value, '2_'))
self.assertTrue(rows_returned(q) == len(objects) - 1)
# -- test multiple AVU's by same attribute name --
objects[-1].metadata.add('B_meta', '29')
q = self.sess.query(DataObject,DataObjectMeta).\
filter(DataObjectMeta.name == 'B_meta').filter(DataObjectMeta.value == '28').\
filter(DataObjectMeta.name == 'B_meta').filter(DataObjectMeta.value == '29')
self.assertTrue(rows_returned(q) == 1)
finally:
for x in (objects + decoys):
x.unlink(force=True)
helpers.remove_unused_metadata(self.sess)
def test_multiple_criteria_on_one_column_name(self):
collection = self.coll_path
filename = 'test_multiple_AVU_joins'
file_path = '{collection}/{filename}'.format(**locals())
objects = []
nobj = 0
for x in range(3,9):
nobj += 2
obj1 = helpers.make_object(self.sess, file_path+'-{}'.format(x))
obj2 = helpers.make_object(self.sess, file_path+'-dummy{}'.format(x))
objects.extend([obj1,obj2])
self.assertTrue( nobj > 0 and len(objects) == nobj )
q = self.sess.query(Collection,DataObject)
dummy_test = [d for d in q if d[DataObject.name][-1:] != '8'
and d[DataObject.name][-7:-1] == '-dummy' ]
self.assertTrue( len(dummy_test) > 0 )
q = q. filter(Like(DataObject.name, '%-dummy_')).\
filter(Collection.name == collection) .\
filter(DataObject.name != (filename + '-dummy8'))
results = [r[DataObject.name] for r in q]
self.assertTrue(len(results) == len(dummy_test))
def test_add_metadata_from_rule(self):
'''
Runs a rule whose body and input parameters are created in our script.
The rule adds metadata attributes to an iRODS object
and we check for the presence of said attributes.
'''
session = self.sess
# test metadata
attr_name = "test_attr"
attr_value = "test_value"
# make test object
ts = time.time()
zone = session.zone
username = session.username
object_name = 'foo_{ts}.txt'.format(**locals())
object_path = "/{zone}/home/{username}/{object_name}".format(
**locals())
obj = helpers.make_object(session, object_path)
# rule body
rule_body = textwrap.dedent('''\
test {{
# add metadata
*attribute.*name = *value;
msiAssociateKeyValuePairsToObj(*attribute, *object, "-d")
}}''')
# rule parameters
input_params = { # extra quotes for string literals
'*object': '"{object_path}"'.format(**locals()),
'*name': '"{attr_name}"'.format(**locals()),
'*value': '"{attr_value}"'.format(**locals())
}
output = 'ruleExecOut'
# run test rule
myrule = Rule(session,
body=rule_body,
irods_3_literal_style=True,
params=input_params,
output=output)
myrule.execute()
# check that metadata is there
meta = session.metadata.get(DataObject, object_path)
assert meta[0].name == attr_name
assert meta[0].value == attr_value
# remove test object
obj.unlink(force=True)
def test_add_metadata_from_rule(self):
'''
Runs a rule whose body and input parameters are created in our script.
The rule adds metadata attributes to an iRODS object
and we check for the presence of said attributes.
'''
session = self.sess
# test metadata
attr_name = "test_attr"
attr_value = "test_value"
# make test object
ts = time.time()
zone = session.zone
username = session.username
object_name = 'foo_{ts}.txt'.format(**locals())
object_path = "/{zone}/home/{username}/{object_name}".format(
**locals())
obj = helpers.make_object(session, object_path)
# rule body
rule_body = textwrap.dedent('''\
test {{
# add metadata
*attribute.*name = *value;
msiAssociateKeyValuePairsToObj(*attribute, *object, "-d")
}}''')
# rule parameters
input_params = { # extra quotes for string literals
'*object': '"{object_path}"'.format(**locals()),
'*name': '"{attr_name}"'.format(**locals()),
'*value': '"{attr_value}"'.format(**locals())
}
output = 'ruleExecOut'
# run test rule
myrule = Rule(session, body=rule_body,
params=input_params, output=output)
myrule.execute()
# check that metadata is there
meta = session.metadata.get(DataObject, object_path)
assert meta[0].name == attr_name
assert meta[0].value == attr_value
# remove test object
obj.unlink(force=True)
def test_add_metadata_from_rule_file(self):
'''
Tests running a rule from a client-side .r file.
The rule adds metadata attributes to an iRODS object
and we check for the presence of said attributes.
'''
session = self.sess
# test metadata
attr_name = "test_attr"
attr_value = "test_value"
# make test object
ts = time.time()
zone = session.zone
username = session.username
object_name = 'foo_{ts}.txt'.format(**locals())
object_path = "/{zone}/home/{username}/{object_name}".format(
**locals())
obj = helpers.make_object(session, object_path)
# make rule file
rule_file_path = "/tmp/test_{ts}.r".format(**locals())
rule = textwrap.dedent('''\
test {{
# add metadata
*attribute.*name = *value;
msiAssociateKeyValuePairsToObj(*attribute, *object, "-d")
}}
INPUT *object="{object_path}",*name="{attr_name}",*value="{attr_value}"
OUTPUT ruleExecOut'''.format(**locals()))
with open(rule_file_path, "w") as rule_file:
rule_file.write(rule)
# run test rule
myrule = Rule(session, rule_file_path)
myrule.execute()
# check that metadata is there
meta = session.metadata.get(DataObject, object_path)
assert meta[0].name == attr_name
assert meta[0].value == attr_value
# remove test object
obj.unlink(force=True)
# remove rule file
os.remove(rule_file_path)
def test_add_metadata_from_rule_file(self):
'''
Tests running a rule from a client-side .r file.
The rule adds metadata attributes to an iRODS object
and we check for the presence of said attributes.
'''
session = self.sess
# test metadata
attr_name = "test_attr"
attr_value = "test_value"
# make test object
ts = time.time()
zone = session.zone
username = session.username
object_name = 'foo_{ts}.txt'.format(**locals())
object_path = "/{zone}/home/{username}/{object_name}".format(
**locals())
obj = helpers.make_object(session, object_path)
# make rule file
rule_file_path = "/tmp/test_{ts}.r".format(**locals())
rule = textwrap.dedent('''\
test {{
# add metadata
*attribute.*name = *value;
msiAssociateKeyValuePairsToObj(*attribute, *object, "-d")
}}
INPUT *object="{object_path}",*name="{attr_name}",*value="{attr_value}"
OUTPUT ruleExecOut'''.format(**locals()))
with open(rule_file_path, "w") as rule_file:
rule_file.write(rule)
# run test rule
myrule = Rule(session, rule_file_path)
myrule.execute()
# check that metadata is there
meta = session.metadata.get(DataObject, object_path)
assert meta[0].name == attr_name
assert meta[0].value == attr_value
# remove test object
obj.unlink(force=True)
# remove rule file
os.remove(rule_file_path)
def test_copy_obj_to_obj(self):
# test args
collection = self.coll_path
src_file_name = 'foo'
dest_file_name = 'bar'
# make object in test collection
src_path = "{collection}/{src_file_name}".format(**locals())
src_obj = helpers.make_object(self.sess, src_path, options={kw.REG_CHKSUM_KW: ''})
# copy object
options = {kw.VERIFY_CHKSUM_KW: ''}
dest_path = "{collection}/{dest_file_name}".format(**locals())
self.sess.data_objects.copy(src_path, dest_path, options)
# compare checksums
dest_obj = self.sess.data_objects.get(dest_path)
self.assertEqual(src_obj.checksum, dest_obj.checksum)
def test_obj_truncate(self):
collection = self.coll_path
filename = 'test_obj_truncate.txt'
file_path = '{collection}/{filename}'.format(**locals())
# random long content
content = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890'
truncated_content = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
# make object
obj = helpers.make_object(self.sess, file_path, content=content)
# truncate object
obj.truncate(len(truncated_content))
# read file
obj = self.sess.data_objects.get(file_path)
with obj.open('r') as f:
self.assertEqual(f.read().decode(), truncated_content)
def test_obj_truncate(self):
collection = self.coll_path
filename = 'test_obj_truncate.txt'
file_path = '{collection}/{filename}'.format(**locals())
# random long content
content = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890'
truncated_content = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
# make object
obj = helpers.make_object(self.sess, file_path, content=content)
# truncate object
obj.truncate(len(truncated_content))
# read file
obj = self.sess.data_objects.get(file_path)
with obj.open('r') as f:
self.assertEqual(f.read().decode(), truncated_content)
def test_copy_obj_to_obj(self):
# test args
collection = self.coll_path
src_file_name = 'foo'
dest_file_name = 'bar'
# make object in test collection
options = {kw.REG_CHKSUM_KW: ''}
src_path = "{collection}/{src_file_name}".format(**locals())
src_obj = helpers.make_object(self.sess, src_path, **options)
# copy object
options = {kw.VERIFY_CHKSUM_KW: ''}
dest_path = "{collection}/{dest_file_name}".format(**locals())
self.sess.data_objects.copy(src_path, dest_path, **options)
# compare checksums
dest_obj = self.sess.data_objects.get(dest_path)
self.assertEqual(src_obj.checksum, dest_obj.checksum)
def test_obj_replicate(self):
# test data
resc_name = 'temporary_test_resource'
if self.sess.server_version < (4, 0, 0):
resc_type = 'unix file system'
resc_class = 'cache'
else:
resc_type = 'unixfilesystem'
resc_class = ''
resc_host = self.sess.host # use remote host when available in CI
resc_path = '/tmp/' + resc_name
# make second resource
self.sess.resources.create(resc_name,
resc_type,
resc_host,
resc_path,
resource_class=resc_class)
# make test object on default resource
collection = self.coll_path
filename = 'test_replicate.txt'
file_path = '{collection}/{filename}'.format(**locals())
obj = helpers.make_object(self.sess, file_path)
# replicate object to 2nd resource
obj.replicate(resc_name)
# refresh object
obj = self.sess.data_objects.get(obj.path)
# check that object is on both resources
resources = [replica.resource_name for replica in obj.replicas]
self.assertEqual(len(resources), 2)
self.assertIn(resc_name, resources)
# force remove object
obj.unlink(force=True)
# delete second resource
self.sess.resources.remove(resc_name)
def test_force_unlink(self):
collection = self.coll_path
filename = 'test_force_unlink.txt'
file_path = '{collection}/{filename}'.format(**locals())
# make object
obj = helpers.make_object(self.sess, file_path)
# force remove object
obj.unlink(force=True)
# should be gone
with self.assertRaises(DataObjectDoesNotExist):
obj = self.sess.data_objects.get(file_path)
# make sure it's not in the trash either
conditions = [DataObject.name == filename,
Criterion('like', Collection.name, "/dev/trash/%%")]
query = self.sess.query(DataObject.id, DataObject.name, Collection.name).filter(*conditions)
results = query.all()
self.assertEqual(len(results), 0)
def test_copy_obj_to_coll(self):
# test args
collection = self.coll_path
file_name = 'foo'
dest_coll_name = 'copy_dest_coll'
dest_coll_path = "{collection}/{dest_coll_name}".format(**locals())
dest_obj_path = "{collection}/{dest_coll_name}/{file_name}".format(
**locals())
# make object in test collection
path = "{collection}/{file_name}".format(**locals())
src_obj = helpers.make_object(self.sess, path, options={kw.REG_CHKSUM_KW: ''})
# make new collection and copy object into it
options = {kw.VERIFY_CHKSUM_KW: ''}
helpers.make_collection(self.sess, dest_coll_path)
self.sess.data_objects.copy(path, dest_coll_path, options)
# compare checksums
dest_obj = self.sess.data_objects.get(dest_obj_path)
self.assertEqual(src_obj.checksum, dest_obj.checksum)
def test_copy_obj_to_coll(self):
# test args
collection = self.coll_path
file_name = 'foo'
dest_coll_name = 'copy_dest_coll'
dest_coll_path = "{collection}/{dest_coll_name}".format(**locals())
dest_obj_path = "{collection}/{dest_coll_name}/{file_name}".format(
**locals())
# make object in test collection
path = "{collection}/{file_name}".format(**locals())
options = {kw.REG_CHKSUM_KW: ''}
src_obj = helpers.make_object(self.sess, path, **options)
# make new collection and copy object into it
options = {kw.VERIFY_CHKSUM_KW: ''}
helpers.make_collection(self.sess, dest_coll_path)
self.sess.data_objects.copy(path, dest_coll_path, **options)
# compare checksums
dest_obj = self.sess.data_objects.get(dest_obj_path)
self.assertEqual(src_obj.checksum, dest_obj.checksum)
def test_obj_replicate(self):
# test data
resc_name = 'temporary_test_resource'
if self.sess.server_version < (4, 0, 0):
resc_type = 'unix file system'
resc_class = 'cache'
else:
resc_type = 'unixfilesystem'
resc_class = ''
resc_host = self.sess.host # use remote host when available in CI
resc_path = '/tmp/' + resc_name
# make second resource
self.sess.resources.create(
resc_name, resc_type, resc_host, resc_path, resource_class=resc_class)
# make test object on default resource
collection = self.coll_path
filename = 'test_replicate.txt'
file_path = '{collection}/{filename}'.format(**locals())
obj = helpers.make_object(self.sess, file_path)
# replicate object to 2nd resource
obj.replicate(resc_name)
# refresh object
obj = self.sess.data_objects.get(obj.path)
# check that object is on both resources
resources = [replica.resource_name for replica in obj.replicas]
self.assertEqual(len(resources), 2)
self.assertIn(resc_name, resources)
# force remove object
obj.unlink(force=True)
# delete second resource
self.sess.resources.remove(resc_name)
def test_ticket_expiry(self):
with helpers.make_session() as ses:
t1 = t2 = dobj = None
try:
gm_now = time.gmtime()
gm_later = time.gmtime(calendar.timegm(gm_now) + 10)
home = self.irods_homedir(ses)
dobj = helpers.make_object(ses,
home.path + '/dummy',
content='abcxyz')
later_ts = gmtime_to_timestamp(gm_later)
later_epoch = calendar.timegm(gm_later)
t1 = Ticket(ses)
t2 = Ticket(ses)
tickets = [
_.issue('read', dobj.path).string for _ in (
t1,
t2,
)
]
t1.modify(
'expire', later_ts
) # - Specify expiry with the human readable timestamp.
t2.modify('expire', later_epoch
) # - Specify expiry formatted as epoch seconds.
# Check normal access succeeds prior to expiration
for ticket_string in tickets:
with self.login(self.alice) as alice:
Ticket(alice, ticket_string).supply()
alice.data_objects.get(dobj.path)
# Check that both time formats have effected the same expiry time (The catalog returns epoch secs.)
timestamps = []
for ticket_string in tickets:
t = ses.query(TicketQuery.Ticket).filter(
TicketQuery.Ticket.string == ticket_string).one()
timestamps.append(t[TicketQuery.Ticket.expiry_ts])
self.assertEqual(len(timestamps), 2)
self.assertEqual(timestamps[0], timestamps[1])
# Wait for tickets to expire.
epoch = int(time.time())
while epoch <= later_epoch:
time.sleep(later_epoch - epoch + 1)
epoch = int(time.time())
Expected_Exception = ex.CAT_TICKET_EXPIRED if ses.server_version >= (4,2,9) \
else ex.SYS_FILE_DESC_OUT_OF_RANGE
# Check tickets no longer allow access.
for ticket_string in tickets:
with self.login(
self.alice) as alice, tempfile.NamedTemporaryFile(
) as f:
Ticket(alice, ticket_string).supply()
with self.assertRaises(Expected_Exception):
alice.data_objects.get(dobj.path, f.name,
**{kw.FORCE_FLAG_KW: ''})
finally:
if t1: t1.delete()
if t2: t2.delete()
if dobj: dobj.unlink(force=True)
def test_object_read_and_write_tickets(self):
if self.alice is None or self.bob is None:
self.skipTest(
"A rodsuser (alice and/or bob) could not be created.")
t = None
data_objs = []
tmpfiles = []
try:
# Create ticket for read access to alice's home collection.
alice = self.login(self.alice)
home = self.irods_homedir(alice)
# Create 'R' and 'W' in alice's home collection.
data_objs = [
helpers.make_object(alice,
home.path + "/" + name,
content='abcxyz') for name in ('R', 'W')
]
tickets = {
'R': Ticket(alice).issue('read', home.path + "/R").string,
'W': Ticket(alice).issue('write', home.path + "/W").string
}
# Test only write ticket allows upload.
with self.login(self.bob) as bob:
rw_names = {}
for name in ('R', 'W'):
Ticket(bob, tickets[name]).supply()
with tempfile.NamedTemporaryFile(delete=False) as tmpf:
tmpfiles += [tmpf]
rw_names[name] = tmpf.name
tmpf.write(b'hello')
if name == 'W':
bob.data_objects.put(tmpf.name, home.path + "/" + name)
else:
try:
bob.data_objects.put(tmpf.name,
home.path + "/" + name)
except ex.CAT_NO_ACCESS_PERMISSION:
pass
else:
raise AssertionError(
"A read ticket allowed a data object write operation to happen without error."
)
# Test upload was successful, by getting and confirming contents.
with self.login(
self.bob
) as bob: # This check must be in a new session or we get CollectionDoesNotExist. - Possibly a new issue [ ]
for name in ('R', 'W'):
Ticket(bob, tickets[name]).supply()
bob.data_objects.get(home.path + "/" + name,
rw_names[name],
**{kw.FORCE_FLAG_KW: ''})
with open(rw_names[name], 'r') as tmpread:
self.assertEqual(tmpread.read(),
'abcxyz' if name == 'R' else 'hello')
finally:
if t: t.delete()
for d in data_objs:
d.unlink(force=True)
for file_ in tmpfiles:
os.unlink(file_.name)
alice.cleanup()
def test_obj_exists(self):
obj_name = 'this_object_will_exist_once_made'
exists_path = '{}/{}'.format(self.coll_path, obj_name)
helpers.make_object(self.sess, exists_path)
self.assertTrue(self.sess.data_objects.exists(exists_path))
def test_irodsmetacollection_data_obj(self):
'''
Tested as data_object metadata
'''
# test settings
avu_count = 5
# make test object
test_obj_path = self.coll_path + '/test_irodsmetacollection'
test_obj = helpers.make_object(self.sess, test_obj_path)
# test AVUs
triplets = [('test_attr'+str(i), 'test_value', 'test_units') for i in range(avu_count)]
# get coll meta
imc = test_obj.metadata
# try invalid key
with self.assertRaises(KeyError):
imc.get_one('bad_key')
# invalid key type
with self.assertRaises(TypeError):
imc.get_one(list())
# try empty update values
with self.assertRaises(ValueError):
imc.add()
# add AVUs
for triplet in triplets:
imc.add(*triplet)
# add another AVU with existing attribute name
attr_name = triplets[0][0]
duplicate_triplet = (attr_name, 'other_value', 'test_units')
imc.add(*duplicate_triplet)
# get_one should fail
with self.assertRaises(KeyError):
imc.get_one(attr_name)
# remove triplet
imc.remove(*duplicate_triplet)
imc.get_one(attr_name)
# get keys
for key in imc.keys():
self.assertIn(key, [triplet[0] for triplet in triplets])
# get items
for avu in imc.items():
self.assertIsInstance(avu, iRODSMeta)
self.assertIn(avu.name, [triplet[0] for triplet in triplets])
self.assertIn(avu.value, [triplet[1] for triplet in triplets])
self.assertIn(avu.units, [triplet[2] for triplet in triplets])
# try contains
self.assertIn(triplets[0][0], imc)
# try contains with bad key type
with self.assertRaises(TypeError):
int() in imc
# set item
imc[attr_name] = iRODSMeta(attr_name, 'boo')
# get item
imc[attr_name]
# del item with bad key type
with self.assertRaises(TypeError):
del imc[int()]
# del item
del imc[attr_name]
with self.assertRaises(KeyError):
imc[attr_name]
# remove all metadta
imc.remove_all()
self.assertEqual(len(imc), 0)
# remove test collection
test_obj.unlink(force=True)
def test_obj_exists(self):
obj_name = 'this_object_will_exist_once_made'
exists_path = '{}/{}'.format(self.coll_path, obj_name)
helpers.make_object(self.sess, exists_path)
self.assertTrue(self.sess.data_objects.exists(exists_path))
