def create_data_object(
test_with_baton: TestWithBaton,
name: str,
metadata: IrodsMetadata = IrodsMetadata(),
access_controls: Iterable[AccessControl] = None,
) -> DataObject:
"""
Factory method to create an iRODS data object that has metadata, an ACL and replicas. Creates in current directory.
:param test_with_baton: framework to allow testing with baton
:param name: the name given to the created data object
:param metadata: the metadata to give the file
:param access_controls: access control list that the data object should have
:return: the created iRODS file
"""
user = test_with_baton.irods_server.users[0]
setup_helper = SetupHelper(test_with_baton.icommands_location)
path = setup_helper.create_data_object(name)
setup_helper.add_metadata_to(path, metadata)
checksum = setup_helper.get_checksum(path)
replicas = []
for i in range(2):
replica_storage = setup_helper.create_replica_storage()
setup_helper.replicate_data_object(path, replica_storage)
replica = DataObjectReplica(i + 1, checksum, replica_storage.host, replica_storage.name, True)
replicas.append(replica)
setup_helper.update_checksums(path)
# Difficult to get all the details of replica 0 using icommands so remove
setup_helper.run_icommand(["irm", "-n", "0", path])
if access_controls is None:
access_controls = [AccessControl(User(user.username, user.zone), AccessControl.Level.OWN)]
else:
_set_access_controls(test_with_baton, path, access_controls)
data_object = DataObject(path, access_controls, metadata, replicas)
synchronise_timestamps(test_with_baton, data_object)
return data_object
def create_entity_tree(
test_with_baton: TestWithBaton, root: str, node: EntityNode, access_controls: Iterable[AccessControl] = None
) -> Iterable[IrodsEntity]:
"""
TODO
:param test_with_baton:
:param root:
:param node:
:param access_controls:
:return:
"""
entities = []
setup_helper = SetupHelper(test_with_baton.icommands_location)
if isinstance(node, DataObjectNode):
entity = DataObject(setup_helper.create_data_object(node.name))
else:
entity = Collection(setup_helper.create_collection(node.name))
_set_access_controls(test_with_baton, entity.path, access_controls)
entity.access_controls = access_controls
new_path = "%s/%s" % (root, entity.get_name())
setup_helper.run_icommand(["imv", entity.path, new_path])
entity.path = new_path
entities.append(entity)
if isinstance(node, CollectionNode):
for child in node.children:
descendants = create_entity_tree(test_with_baton, "%s/%s" % (root, node.name), child, access_controls)
entities.extend(descendants)
if isinstance(node, CollectionNode):
assert len(entities) == len(list(node.get_all_descendants()) + [node])
else:
assert len(entities) == 1
return entities
class TestBatonUpdateMapper(unittest.TestCase):
"""
Tests for `BatonUpdateMapper`.
"""
def setUp(self):
self.test_with_baton = TestWithBaton(baton_setup=BATON_SETUP)
self.test_with_baton.setup()
self.setup_helper = SetupHelper(self.test_with_baton.icommands_location)
install_queries(REQUIRED_SPECIFIC_QUERIES, self.setup_helper)
zone = self.test_with_baton.irods_server.users[0].zone
self.mapper = BatonUpdateMapper(self.test_with_baton.baton_location, zone)
def test_get_all_since_with_date_in_future(self):
updates = self.mapper.get_all_since(datetime.fromtimestamp(_MAX_IRODS_TIMESTAMP))
self.assertEqual(len(updates), 0)
def test_get_all_since_with_date_in_past(self):
start_timestamp = self._get_latest_update_timestamp()
updates = self.mapper.get_all_since(start_timestamp)
self.assertEqual(len(updates), 0)
def test_get_all_since_with_data_object_updates(self):
start_timestamp = self._get_latest_update_timestamp()
location_1 = self.setup_helper.create_data_object(_DATA_OBJECT_NAMES[0])
location_2 = self.setup_helper.create_data_object(_DATA_OBJECT_NAMES[1])
updates = self.mapper.get_all_since(start_timestamp)
self.assertEqual(len(updates), 2)
self.assertEqual(len(updates.get_entity_updates(location_1)), 1)
self.assertEqual(len(updates.get_entity_updates(location_2)), 1)
# TODO: More detailed check on updates
def test_get_all_since_with_updates_to_data_object_replica(self):
start_timestamp = self._get_latest_update_timestamp()
location = self.setup_helper.create_data_object(_DATA_OBJECT_NAMES[0])
resource = self.setup_helper.create_replica_storage()
self.setup_helper.replicate_data_object(location, resource)
self.setup_helper.update_checksums(location)
checksum = self.setup_helper.get_checksum(location)
replicas = DataObjectReplicaCollection([DataObjectReplica(i, checksum) for i in range(2)])
expected_modification = DataObjectModification(modified_replicas=replicas)
expected_metadata = Metadata(DataObjectModificationJSONEncoder().default(expected_modification))
updates = self.mapper.get_all_since(start_timestamp)
self.assertEquals(len(updates), 1)
self.assertIn(updates[0].target, location)
self.assertCountEqual(updates[0].metadata, expected_metadata)
def test_get_all_since_with_metadata_update(self):
path = self.setup_helper.create_data_object(_DATA_OBJECT_NAMES[0])
start_timestamp = self._get_latest_update_timestamp()
metadata_1 = Metadata({
_METADATA_KEYS[0]: _METADATA_VALUES[0],
_METADATA_KEYS[1]: _METADATA_VALUES[1]
})
self.setup_helper.add_metadata_to(path, metadata_1)
# Update pre-existing metadata item
metadata_2 = Metadata({_METADATA_KEYS[0]: _METADATA_VALUES[2]})
self.setup_helper.add_metadata_to(path, metadata_2)
expected_irods_metadata = IrodsMetadata({
_METADATA_KEYS[0]: {_METADATA_VALUES[0], _METADATA_VALUES[2]},
_METADATA_KEYS[1]: {_METADATA_VALUES[1]}
})
modification = DataObjectModification(modified_metadata=expected_irods_metadata)
expected_update_metadata = Metadata(DataObjectModificationJSONEncoder().default(modification))
updates = self.mapper.get_all_since(start_timestamp)
self.assertEqual(len(updates), 1)
relevant_updates = updates.get_entity_updates(path)
# Expect the mapper to have combined all updates into one (https://github.com/wtsi-hgi/cookie-monster/issues/3)
self.assertEqual(len(relevant_updates), 1)
self.assertEqual(relevant_updates[0].target, path)
logging.debug(relevant_updates[0].metadata)
logging.debug(expected_update_metadata)
self.assertCountEqual(relevant_updates[0].metadata, expected_update_metadata)
def _get_latest_update_timestamp(self) -> datetime:
"""
Gets the timestamp of the latest update. If there has been no updates, returns minimum timestamp.
This timestamp is useful to get before running a test for use in filtering out any updates that iRODS
already has. The Dockerized iRODS 3.3.1, for example, will have updates on start.
:return: timestamp of latest update
"""
inital_updates = self.mapper.get_all_since(datetime.min)
if len(inital_updates) == 0:
return datetime.min
return inital_updates.get_most_recent()[0].timestamp
def tearDown(self):
self.test_with_baton.tear_down()
