def test_atomic_metadata_operations_244(self):
user = self.sess.users.get("rods")
group = self.sess.user_groups.get("public")
m = ("attr_244", "value", "units")
with self.assertRaises(BadAVUOperationValue):
AVUOperation(operation="add", avu=m)
with self.assertRaises(BadAVUOperationValue):
AVUOperation(operation="not_add_or_remove", avu=iRODSMeta(*m))
with self.assertRaises(BadAVUOperationKeyword):
AVUOperation(operation="add",
avu=iRODSMeta(*m),
extra_keyword=None)
with self.assertRaises(InvalidAtomicAVURequest):
user.metadata.apply_atomic_operations(tuple())
user.metadata.apply_atomic_operations(
) # no AVUs applied - no-op without error
for n, obj in enumerate((group, user, self.coll, self.obj)):
avus = [
iRODSMeta('some_attribute', str(i), 'some_units')
for i in range(n * 100, (n + 1) * 100)
]
obj.metadata.apply_atomic_operations(
*[AVUOperation(operation="add", avu=avu_) for avu_ in avus])
obj.metadata.apply_atomic_operations(
*[AVUOperation(operation="remove", avu=avu_) for avu_ in avus])
def test_add_obj_meta(self):
"""
"""
# add metadata to test object
self.sess.metadata.add(DataObject, self.obj_path,
iRODSMeta(self.attr0, self.value0))
self.sess.metadata.add(DataObject, self.obj_path,
iRODSMeta(self.attr1, self.value1, self.unit1))
# get object metadata
meta = self.sess.metadata.get(DataObject, self.obj_path)
# sort results by metadata id
def getKey(iRODSMeta):
return iRODSMeta.id
meta = sorted(meta, key=getKey)
# assertions
assert meta[0].name == self.attr0
assert meta[0].value == self.value0
assert meta[1].name == self.attr1
assert meta[1].value == self.value1
assert meta[1].units == self.unit1
def upload(self):
# get current local timestamp
stat = os.stat(self.local_path)
# create folder structure on remote
folders = os.path.dirname(self.remote_path).split(os.sep)[1:]
collpath = os.sep
for folder in folders:
collpath = os.path.join(collpath, folder)
try:
self._irods_session.collections.get(collpath)
except:
self._irods_session.collections.create(collpath)
# upload file and store local timestamp in metadata since irods sets the files modification time to
# the upload time rather than retaining the local modification time
self._irods_session.data_objects.put(self.local_path, self.remote_path)
# erase meta data (if exists) before adding it. there is no update routine available in the API
for m in self._irods_session.metadata.get(DataObject, self.remote_path):
if m.name in ('mtime', 'atime', 'ctime'):
self._irods_session.metadata.remove(DataObject, self.remote_path, iRODSMeta(m.name, m.value, m.units))
self._irods_session.metadata.add(DataObject, self.remote_path, iRODSMeta('mtime', str(stat.st_mtime), 's'))
self._irods_session.metadata.add(DataObject, self.remote_path, iRODSMeta('atime', str(stat.st_atime), 's'))
self._irods_session.metadata.add(DataObject, self.remote_path, iRODSMeta('ctime', str(stat.st_ctime), 's'))
def test_add_obj_meta(self):
# add metadata to test object
self.sess.metadata.add(DataObject, self.obj_path,
iRODSMeta(self.attr0, self.value0))
self.sess.metadata.add(DataObject, self.obj_path,
iRODSMeta(self.attr1, self.value1, self.unit1))
# Throw in some unicode for good measure
attribute, value = 'attr2', u'☭⛷★⚽'
self.sess.metadata.add(DataObject, self.obj_path,
iRODSMeta(attribute, value))
# get object metadata
meta = self.sess.metadata.get(DataObject, self.obj_path)
# sort results by metadata id
def getKey(AVU):
return AVU.avu_id
meta = sorted(meta, key=getKey)
# assertions
assert meta[0].name == self.attr0
assert meta[0].value == self.value0
assert meta[1].name == self.attr1
assert meta[1].value == self.value1
assert meta[1].units == self.unit1
assert meta[2].name == attribute
assert meta[2].value == value
def test_add_obj_meta(self):
# add metadata to test object
self.sess.metadata.add(DataObject, self.obj_path,
iRODSMeta(self.attr0, self.value0))
self.sess.metadata.add(DataObject, self.obj_path,
iRODSMeta(self.attr1, self.value1, self.unit1))
# Throw in some unicode for good measure
attribute, value = 'attr2', u'☭⛷★⚽'
self.sess.metadata.add(DataObject, self.obj_path,
iRODSMeta(attribute, value))
# get object metadata
meta = self.sess.metadata.get(DataObject, self.obj_path)
# sort results by metadata id
def getKey(AVU):
return AVU.avu_id
meta = sorted(meta, key=getKey)
# assertions
assert meta[0].name == self.attr0
assert meta[0].value == self.value0
assert meta[1].name == self.attr1
assert meta[1].value == self.value1
assert meta[1].units == self.unit1
assert meta[2].name == attribute
testValue = (value if PY3 else value.encode('utf8'))
assert meta[2].value == testValue
def test_remove_obj_meta(self):
# add metadata to test object
self.sess.metadata.add(DataObject, self.obj_path,
iRODSMeta(self.attr0, self.value0))
# check that metadata is there
meta = self.sess.metadata.get(DataObject, self.obj_path)
assert meta[0].name == self.attr0
# remove metadata from object
self.sess.metadata.remove(DataObject, self.obj_path,
iRODSMeta(self.attr0, self.value0))
# check that metadata is gone
meta = self.sess.metadata.get(DataObject, self.obj_path)
assert len(meta) == 0
def get(self, model_cls, path):
resource_type = self._model_class_to_resource_type(model_cls)
model = {
'd': DataObjectMeta,
'c': CollectionMeta,
'r': ResourceMeta,
'u': UserMeta
}[resource_type]
conditions = {
'd': [
Collection.name == dirname(path),
DataObject.name == basename(path)
],
'c': [Collection.name == path],
'r': [Resource.name == path],
'u': [User.name == path]
}[resource_type]
results = self.sess.query(model.id, model.name, model.value, model.units)\
.filter(*conditions).all()
return [
iRODSMeta(row[model.name],
row[model.value],
row[model.units],
id=row[model.id]) for row in results
]
def apply_netcopy_complete_metadata(session, netcopy_info: NetcopyInfo,
target: str) -> None:
"""Apply netcopy complete meta data to collection AVUs."""
with cleanuping(session):
coll = session.data_objects.get(target)
for avu in netcopy_info.to_avus():
coll.metadata[avu[0]] = iRODSMeta(*avu)
def apply_runinfo_metadata(session, run_info: RunInfo, target: str) -> None:
"""Apply ``RunInfo`` meta data to collection AVUs."""
target_coll = str(pathlib.Path(target).parent)
with cleanuping(session):
coll = session.collections.get(target_coll)
for avu in run_info.to_avus():
coll.metadata[avu[0]] = iRODSMeta(*avu)
def get(self, model_cls, path):
resource_type = self._model_class_to_resource_type(model_cls)
model = {
'd': DataObjectMeta,
'c': CollectionMeta,
'r': ResourceMeta,
'u': UserMeta
}[resource_type]
conditions = {
'd': [
Collection.name == dirname(path),
DataObject.name == basename(path)
],
'c': [Collection.name == path],
'r': [Resource.name == path],
'u': [User.name == path]
}[resource_type]
results = self.sess.query(model.id, model.name, model.value, model.units)\
.filter(*conditions).all()
return [iRODSMeta(
row[model.name],
row[model.value],
row[model.units],
id=row[model.id]
) for row in results]
def test_remove_obj_meta(self):
# add metadata to test object
self.sess.metadata.add(DataObject, self.obj_path,
iRODSMeta(self.attr0, self.value0))
# check that metadata is there
meta = self.sess.metadata.get(DataObject, self.obj_path)
assert meta[0].name == self.attr0
# remove metadata from object
self.sess.metadata.remove(DataObject, self.obj_path,
iRODSMeta(self.attr0, self.value0))
# check that metadata is gone
meta = self.sess.metadata.get(DataObject, self.obj_path)
assert len(meta) == 0
def test_meta_repr(self):
# test obj
collection = self.coll_path
filename = 'test_meta_repr.txt'
test_obj_path = '{collection}/{filename}'.format(**locals())
# make object
obj = helpers.make_object(self.sess, test_obj_path)
# test AVU
attribute, value, units = ('test_attr', 'test_value', 'test_units')
# add metadata to test object
meta = self.sess.metadata.add(DataObject, test_obj_path,
iRODSMeta(attribute, value, units))
# get metadata
meta = self.sess.metadata.get(DataObject, test_obj_path)
id = meta[0].id
# assert
self.assertEqual(
repr(meta[0]), "<iRODSMeta {id} {attribute} {value} {units}>".format(**locals()))
# remove test object
obj.unlink(force=True)
def test_resc_meta(self):
rescname = 'demoResc'
self.sess.resources.get(rescname).metadata.remove_all()
self.sess.metadata.set(Resource, rescname,
iRODSMeta('zero', 'marginal', 'cost'))
self.sess.metadata.add(Resource, rescname,
iRODSMeta('zero', 'marginal'))
self.sess.metadata.set(Resource, rescname,
iRODSMeta('for', 'ever', 'after'))
meta = self.sess.resources.get(rescname).metadata
self.assertTrue(len(meta) == 3)
resource = self.sess.resources.get(rescname)
all_AVUs = resource.metadata.items()
for avu in all_AVUs:
resource.metadata.remove(avu)
self.assertTrue(0 == len(self.sess.resources.get(rescname).metadata))
def post_data_obj_modify(hdlr_mod, logger, session, meta, **options):
target = meta["target"]
path = meta["path"]
s = os.stat(path)
mode = s.st_mode
obj = session.data_objects.get(target)
obj.metadata[filesystem_mode] = iRODSMeta(filesystem_mode, str(mode))
def test_group_metadata(self):
group_name = "test_group"
# group should not be already present
with self.assertRaises(UserGroupDoesNotExist):
self.sess.user_groups.get(group_name)
group = None
try:
# create group
group = self.sess.user_groups.create(group_name)
# add metadata to group
triple = ['key', 'value', 'unit']
group.metadata[triple[0]] = iRODSMeta(*triple)
result = self.sess.query(UserMeta, UserGroup).filter(
UserGroup.name == group_name, UserMeta.name == 'key').one()
self.assertTrue([
result[k]
for k in (UserMeta.name, UserMeta.value, UserMeta.units)
] == triple)
finally:
if group:
group.remove()
helpers.remove_unused_metadata(self.sess)
def test_add_obj_meta_empty(self):
'''Should raise exception
'''
# try to add metadata with empty value
with self.assertRaises(ValueError):
self.sess.metadata.add(DataObject, self.obj_path,
iRODSMeta('attr_with_empty_value', ''))
def test_add_obj_meta_empty(self):
'''Should raise exception
'''
# try to add metadata with empty value
with self.assertRaises(ValueError):
self.sess.metadata.add(DataObject, self.obj_path,
iRODSMeta('attr_with_empty_value', ''))
def apply_runparameters_metadata(session, values: typing.Dict[str, str],
target: str) -> None:
"""Apply ``runParameters.xml`` meta data to collection AVUs."""
target_coll = str(pathlib.Path(target).parent)
with cleanuping(session):
coll = session.collections.get(target_coll)
for key, value in values.items():
coll.metadata[key] = iRODSMeta(key, value, "")
def put_metaf (self, file_path, metadata, absolute=False):
""" Attach the given metadata on the file specified relative to the iRods
current working directory (default) OR relative to the users root directory,
if the absolute argument is True.
"""
obj = self.getf(file_path, absolute=absolute)
for key, val in metadata.items():
skey = str(key)
obj.metadata[skey] = iRODSMeta(skey, str(val))
def test_add_coll_meta(self):
# add metadata to test collection
self.sess.metadata.add(Collection, self.coll_path,
iRODSMeta(self.attr0, self.value0))
# get collection metadata
meta = self.sess.metadata.get(Collection, self.coll_path)
# assertions
assert meta[0].name == self.attr0
assert meta[0].value == self.value0
# remove collection metadata
self.sess.metadata.remove(Collection, self.coll_path,
iRODSMeta(self.attr0, self.value0))
# check that metadata is gone
meta = self.sess.metadata.get(Collection, self.coll_path)
assert len(meta) == 0
def execute_plan(self, plan, overwrite=False, refresh=False):
filepath = plan.path
planned_AVUs = plan.metadata #List of AVUs
is_collection = plan.is_collection
existing_metadata = irods_wrapper.get_metadata(self.session, filepath,
is_collection)
with self.process_pool as p: # On close, context manager returns process to pool
print(f"Filepath: {filepath} AVUs: {planned_AVUs}")
if refresh:
for avu in existing_metadata.items():
existing_metadata.remove(avu)
for avu in planned_AVUs:
if avu.attribute:
new_AVU = iRODSMeta(avu.attribute, avu.value, avu.unit)
sexisting_metadata[avu.attribute] = new_AVU
return
if overwrite:
for avu in planned_AVUs:
try:
existing_AVU = existing_metadata.get_one(avu.attribute)
if existing_AVU.value != avu.value or existing_AVU.units != avu.unit:
new_AVU = iRODSMeta(avu.attribute, avu.value,
avu.unit)
existing_metadata[avu.attribute] = new_AVU
except:
if avu.attribute:
new_AVU = iRODSMeta(avu.attribute, avu.value,
avu.unit)
existing_metadata[avu.attribute] = new_AVU
return
if not refresh and not overwrite:
for avu in planned_AVUs:
try:
existing_metadata.get_one(avu.attribute)
except:
new_AVU = iRODSMeta(avu.attribute, avu.value, avu.unit)
if avu.attribute:
existing_metadata[avu.attribute] = new_AVU
def test_add_coll_meta(self):
# add metadata to test collection
self.sess.metadata.add(Collection, self.coll_path,
iRODSMeta(self.attr0, self.value0))
# get collection metadata
meta = self.sess.metadata.get(Collection, self.coll_path)
# assertions
assert meta[0].name == self.attr0
assert meta[0].value == self.value0
# remove collection metadata
self.sess.metadata.remove(Collection, self.coll_path,
iRODSMeta(self.attr0, self.value0))
# check that metadata is gone
meta = self.sess.metadata.get(Collection, self.coll_path)
assert len(meta) == 0
def test_atomic_metadata_operations_255(self):
my_resc = self.sess.resources.create('dummyResc', 'passthru')
avus = [iRODSMeta('a', 'b', 'c'), iRODSMeta('d', 'e', 'f')]
objects = [
self.sess.users.get("rods"),
self.sess.user_groups.get("public"), my_resc,
self.sess.collections.get(self.coll_path),
self.sess.data_objects.get(self.obj_path)
]
try:
for obj in objects:
self.assertEqual(len(obj.metadata.items()), 0)
for n, item in enumerate(avus):
obj.metadata.apply_atomic_operations(
AVUOperation(operation='add', avu=item))
self.assertEqual(len(obj.metadata.items()), n + 1)
finally:
for obj in objects:
obj.metadata.remove_all()
my_resc.remove()
def test_add_obj_meta(self):
"""
"""
# add metadata to test object
self.sess.metadata.add(DataObject, self.obj_path,
iRODSMeta(self.attr0, self.value0))
self.sess.metadata.add(DataObject, self.obj_path,
iRODSMeta(self.attr1, self.value1, self.unit1))
# get object metadata
meta = self.sess.metadata.get(DataObject, self.obj_path)
# assertions
assert(meta[0].name == self.attr0)
assert(meta[0].value == self.value0)
assert(meta[1].name == self.attr1)
assert(meta[1].value == self.value1)
assert(meta[1].units == self.unit1)
def test_add_obj_meta(self):
"""
"""
# add metadata to test object
self.sess.metadata.add(DataObject, self.obj_path,
iRODSMeta(self.attr0, self.value0))
self.sess.metadata.add(DataObject, self.obj_path,
iRODSMeta(self.attr1, self.value1, self.unit1))
# get object metadata
meta = self.sess.metadata.get(DataObject, self.obj_path)
# assertions
assert (meta[0].name == self.attr0)
assert (meta[0].value == self.value0)
assert (meta[1].name == self.attr1)
assert (meta[1].value == self.value1)
assert (meta[1].units == self.unit1)
def test_metadata_manipulations_with_admin_kw__364__365(self):
try:
d = user = None
adm = self.sess
# Create a rodsuser, and a session for that roduser.
user = adm.users.create('bobby', 'rodsuser')
user.modify('password', 'bpass')
with iRODSSession(port=adm.port,
zone=adm.zone,
host=adm.host,
user=user.name,
password='******') as ses:
# Create a data object owned by the rodsuser. Set AVUs in various ways and guarantee each attempt
# has the desired effect.
d = ses.data_objects.create(
'/{adm.zone}/home/{user.name}/testfile'.format(**locals()))
d.metadata.set('a', 'aa', '1')
self.assertIn(('a', 'aa', '1'), d.metadata.items())
d.metadata.set('a', 'aa')
self.assertEqual([('a', 'aa')],
[tuple(_) for _ in d.metadata.items()])
d.metadata['a'] = iRODSMeta('a', 'bb')
self.assertEqual([('a', 'bb')],
[tuple(_) for _ in d.metadata.items()])
# Now the admin does two AVU-set operations. A successful test of these operations' success
# includes that both ('x','y') has been added and ('a','b','c') has overwritten ('a','bb').
da = adm.data_objects.get(d.path)
da.metadata.set('a', 'b', 'c', **{kw.ADMIN_KW: ''})
da.metadata(admin=True)['x'] = iRODSMeta('x', 'y')
d = ses.data_objects.get(
d.path) # assure metadata are not cached
self.assertEqual(set([('x', 'y'), ('a', 'b', 'c')]),
set(tuple(_) for _ in d.metadata.items()))
finally:
if d: d.unlink(force=True)
if user: user.remove()
def add_usage(data_obj_path, rescname):
sess = session_object()
dobj = None
try:
dobj = sess.data_objects.get(data_obj_path)
except:
pass
if dobj:
resc = sess.resources.get(rescname)
datobj_token = dobj.path + '--' + token32()
resc.metadata.add(iRODSMeta(IRODS_RESC_ACQUIRE_KEY, datobj_token))
return datobj_token
def _upload(self):
# get current local timestamp
stat = os.stat(self.local_path)
# create folder structure on remote
folders = os.path.dirname(self.remote_path).split(os.sep)[1:]
# add zone name to path
collpath = os.sep + folders.pop(0) + os.sep + folders.pop(0)
for folder in folders:
collpath = os.path.join(collpath, folder)
try:
self._irods_session.collections.get(collpath)
# ignore subdirectories where user does not have access
except (CAT_NO_ACCESS_PERMISSION):
pass
except:
self._irods_session.collections.create(collpath)
# upload file and store local timestamp in metadata since irods sets the files modification time to
# the upload time rather than retaining the local modification time
self._irods_session.data_objects.put(self.local_path, self.remote_path)
# erase meta data (if exists) before adding it. there is no update routine available in the API
for m in self._irods_session.metadata.get(DataObject,
self.remote_path):
if m.name in ("mtime", "atime", "ctime"):
self._irods_session.metadata.remove(
DataObject, self.remote_path,
iRODSMeta(m.name, m.value, m.units))
self._irods_session.metadata.add(
DataObject, self.remote_path,
iRODSMeta("mtime", str(stat.st_mtime), "s"))
self._irods_session.metadata.add(
DataObject, self.remote_path,
iRODSMeta("atime", str(stat.st_atime), "s"))
self._irods_session.metadata.add(
DataObject, self.remote_path,
iRODSMeta("ctime", str(stat.st_ctime), "s"))
def test_user_metadata(self):
user_name = "testuser"
user = None
try:
user = self.sess.users.create(user_name, 'rodsuser')
# metadata collection is the right type?
self.assertIsInstance(user.metadata, iRODSMetaCollection)
# add three AVUs, two having the same key
user.metadata['key0'] = iRODSMeta('key0', 'value', 'units')
sorted_triples = sorted([['key1', 'value0', 'units0'],
['key1', 'value1', 'units1']])
for m in sorted_triples:
user.metadata.add(iRODSMeta(*m))
# general query gives the right results?
result_0 = self.sess.query(UserMeta, User)\
.filter( User.name == user_name, UserMeta.name == 'key0').one()
self.assertTrue([
result_0[k]
for k in (UserMeta.name, UserMeta.value, UserMeta.units)
] == ['key0', 'value', 'units'])
results_1 = self.sess.query(UserMeta, User)\
.filter(User.name == user_name, UserMeta.name == 'key1')
retrieved_triples = [[
res[k] for k in (UserMeta.name, UserMeta.value, UserMeta.units)
] for res in results_1]
self.assertTrue(sorted_triples == sorted(retrieved_triples))
finally:
if user:
user.remove()
helpers.remove_unused_metadata(self.sess)
def set_xattr(self, path, key, value):
logger.info("set_xattr : %s - %s" % (key, value))
try:
logger.info(
"set_xattr: set extended attribute to a file %s %s=%s" %
(path, key, value))
self.session.metadata.set(DataObject, path, iRODSMeta(key, value))
logger.info("set_xattr: done")
except Exception, e:
logger.error("set_xattr: " + traceback.format_exc())
traceback.print_exc()
raise e
def test_atomic_metadata_operation_for_data_object_244(self):
AVUs_Equal = lambda avu1, avu2, fn=(lambda x: x): fn(avu1) == fn(avu2)
AVU_As_Tuple = lambda avu, length=3: (avu.name, avu.value, avu.units
)[:length]
AVU_Units_String = lambda avu: "" if not avu.units else avu.units
m = iRODSMeta("attr_244", "value", "units")
self.obj.metadata.add(m)
meta = self.sess.metadata.get(DataObject, self.obj_path)
self.assertEqual(len(meta), 1)
self.assertTrue(AVUs_Equal(m, meta[0], AVU_As_Tuple))
self.obj.metadata.apply_atomic_operations( # remove original AVU and replace
AVUOperation(operation="remove",
avu=m), # with two altered versions
AVUOperation(operation="add",
avu=iRODSMeta(
m.name, m.value,
"units_244")), # (one of them without units) ...
AVUOperation(operation="add", avu=iRODSMeta(m.name, m.value)))
meta = self.sess.metadata.get(
DataObject, self.obj_path) # ... check integrity of change
self.assertEqual(sorted([AVU_Units_String(i) for i in meta]),
["", "units_244"])
def refresh_last_update_metadata(logger, session, meta):
"""Update the ``last_update`` and ``status`` meta data value."""
_ = logger
# Get path in irods that corresponds to root and update the meta data there.
path = pathlib.Path(meta["path"])
root = pathlib.Path(meta["root"])
target = pathlib.Path(meta["target"])
logger.info("meta = %s" % meta)
print("meta = %s" % meta)
rel_root_path = path.relative_to(root) # relative to root
print("... %s" % rel_root_path)
rel_folder_path = "/".join(
str(rel_root_path).split("/")[1:]) # relative to run folder
print("... %s" % rel_folder_path)
root_target = str(target)[:-(len(str(rel_folder_path)) + 1)]
print("... %s" % root_target)
with cleanuping(session) as wrapped_session:
coll = wrapped_session.collections.get(root_target)
# Replace ``last_update`` and ``status`` meta data.
coll.metadata[KEY_LAST_UPDATE] = iRODSMeta(
KEY_LAST_UPDATE,
datetime.datetime.now().isoformat(), "")
coll.metadata[KEY_STATUS] = iRODSMeta(KEY_STATUS, "running", "")
def test_atomic_metadata_operation_for_resource_244(self):
(root, leaf) = ('ptX', 'rescX')
with self.create_simple_resc_hierarchy(root, leaf):
root_resc = self.sess.resources.get(root) # resource objects
leaf_resc = self.sess.resources.get(leaf)
root_tuple = ('role', 'root', 'new units #1'
) # AVU tuples to apply
leaf_tuple = ('role', 'leaf', 'new units #2')
root_resc.metadata.add(
*root_tuple[:2]) # first apply without units ...
leaf_resc.metadata.add(*leaf_tuple[:2])
for resc, resc_tuple in ((root_resc, root_tuple), (leaf_resc,
leaf_tuple)):
resc.metadata.apply_atomic_operations( # metadata set operation (remove + add) to add units
AVUOperation(operation="remove",
avu=iRODSMeta(*resc_tuple[:2])),
AVUOperation(operation="add",
avu=iRODSMeta(*resc_tuple[:3])))
resc_meta = self.sess.metadata.get(Resource, resc.name)
avus_to_tuples = lambda avu_list: sorted(
[(i.name, i.value, i.units) for i in avu_list])
self.assertEqual(avus_to_tuples(resc_meta),
avus_to_tuples([iRODSMeta(*resc_tuple)]))
def attach_metadata(data_object, metadata_dict):
metadata_dict = _normalize_metadata_dict(metadata_dict) # coerce to strs
# remove any existing avus whose names would collide
# (note this also turns multiply-valued keys from our metadata_dict into
# singly valued, which is what we want and expect elsewhere)
for avu in data_object.metadata.items():
if avu.name in metadata_dict and avu.value != metadata_dict[avu.name]:
log.info(f"Removing metadata key {avu.key} from {data_object}")
data_object.metadata.remove(avu)
for key in metadata_dict.keys():
new_avu = iRODSMeta(key, metadata_dict[key])
log.info(
f"Setting metadata key {key}={metadata_dict[key]} on {data_object}"
)
data_object.metadata.add(new_avu)
def prepare(self, path, repository):
logger.info("iRODS")
self.connect()
self.coll = self.session.collections.get(path)
self.repository = repository
self.read_collection_metadata()
self.rulemanager = RuleManager(self.session, self.coll)
self.rulemanager.rule_open()
# clear all exporterState AVU values and re-add in-queue-for-export
# in case of remaining failed report AVUs like: upload-failed , failed-dataset-creation etc ..
new_status = f"{repository}:{ExporterState.IN_QUEUE_FOR_EXPORT.value}"
self.coll.metadata['exporterState'] = iRODSMeta('exporterState', new_status)
self.update_metadata_status(ExporterState.IN_QUEUE_FOR_EXPORT.value, ExporterState.CREATE_EXPORTER.value)
def update_repository_info(ibcontext, **kwargs):
logger = logging.getLogger('ipublish')
cfg = ibcontext['irods'].get_config(kwargs)
logger.debug(cfg)
key = {
'irods_zone': get_irods_zone(cfg),
'irods_collection': cfg['irods_collection']
}
data = ibcontext['cache'].read(key)
with ibcontext['irods'].session() as sess:
for k, value in data['repository_info'].items():
logger.debug('set meta {0} {1}={2}'.format(cfg['irods_collection'],
k, value))
sess.metadata.add(Collection, cfg['irods_collection'],
iRODSMeta(k, value))
def test_copy_obj_meta(self):
# test destination object for copy
dest_obj_path = self.coll_path + '/test2'
self.sess.data_objects.create(dest_obj_path)
# add metadata to test object
self.sess.metadata.add(DataObject, self.obj_path,
iRODSMeta(self.attr0, self.value0))
# copy metadata
self.sess.metadata.copy(
DataObject, DataObject, self.obj_path, dest_obj_path)
# get destination object metadata
dest_meta = self.sess.metadata.get(DataObject, dest_obj_path)
# check metadata
assert dest_meta[0].name == self.attr0
def test_irodsmetacollection_data_obj(self):
'''
Tested as data_object metadata
'''
# test settings
avu_count = 5
# make test object
test_obj_path = self.coll_path + '/test_irodsmetacollection'
test_obj = helpers.make_object(self.sess, test_obj_path)
# test AVUs
triplets = [('test_attr'+str(i), 'test_value', 'test_units') for i in range(avu_count)]
# get coll meta
imc = test_obj.metadata
# try invalid key
with self.assertRaises(KeyError):
imc.get_one('bad_key')
# invalid key type
with self.assertRaises(TypeError):
imc.get_one(list())
# try empty update values
with self.assertRaises(ValueError):
imc.add()
# add AVUs
for triplet in triplets:
imc.add(*triplet)
# add another AVU with existing attribute name
attr_name = triplets[0][0]
duplicate_triplet = (attr_name, 'other_value', 'test_units')
imc.add(*duplicate_triplet)
# get_one should fail
with self.assertRaises(KeyError):
imc.get_one(attr_name)
# remove triplet
imc.remove(*duplicate_triplet)
imc.get_one(attr_name)
# get keys
for key in imc.keys():
self.assertIn(key, [triplet[0] for triplet in triplets])
# get items
for avu in imc.items():
self.assertIsInstance(avu, iRODSMeta)
self.assertIn(avu.name, [triplet[0] for triplet in triplets])
self.assertIn(avu.value, [triplet[1] for triplet in triplets])
self.assertIn(avu.units, [triplet[2] for triplet in triplets])
# try contains
self.assertIn(triplets[0][0], imc)
# try contains with bad key type
with self.assertRaises(TypeError):
int() in imc
# set item
imc[attr_name] = iRODSMeta(attr_name, 'boo')
# get item
imc[attr_name]
# del item with bad key type
with self.assertRaises(TypeError):
del imc[int()]
# del item
del imc[attr_name]
with self.assertRaises(KeyError):
imc[attr_name]
# remove all metadta
imc.remove_all()
self.assertEqual(len(imc), 0)
# remove test collection
test_obj.unlink(force=True)