示例#1
0
def create_data_object(
    test_with_baton: TestWithBaton,
    name: str,
    metadata: IrodsMetadata = IrodsMetadata(),
    access_controls: Iterable[AccessControl] = None,
) -> DataObject:
    """
    Factory method to create an iRODS data object that has metadata, an ACL and replicas. Creates in current directory.
    :param test_with_baton: framework to allow testing with baton
    :param name: the name given to the created data object
    :param metadata: the metadata to give the file
    :param access_controls: access control list that the data object should have
    :return: the created iRODS file
    """
    user = test_with_baton.irods_server.users[0]
    setup_helper = SetupHelper(test_with_baton.icommands_location)

    path = setup_helper.create_data_object(name)
    setup_helper.add_metadata_to(path, metadata)
    checksum = setup_helper.get_checksum(path)
    replicas = []
    for i in range(2):
        replica_storage = setup_helper.create_replica_storage()
        setup_helper.replicate_data_object(path, replica_storage)
        replica = DataObjectReplica(i + 1, checksum, replica_storage.host, replica_storage.name, True)
        replicas.append(replica)
    setup_helper.update_checksums(path)

    # Difficult to get all the details of replica 0 using icommands so remove
    setup_helper.run_icommand(["irm", "-n", "0", path])

    if access_controls is None:
        access_controls = [AccessControl(User(user.username, user.zone), AccessControl.Level.OWN)]
    else:
        _set_access_controls(test_with_baton, path, access_controls)

    data_object = DataObject(path, access_controls, metadata, replicas)
    synchronise_timestamps(test_with_baton, data_object)

    return data_object
class TestBatonUpdateMapper(unittest.TestCase):
    """
    Tests for `BatonUpdateMapper`.
    """
    def setUp(self):
        self.test_with_baton = TestWithBaton(baton_setup=BATON_SETUP)
        self.test_with_baton.setup()
        self.setup_helper = SetupHelper(self.test_with_baton.icommands_location)
        install_queries(REQUIRED_SPECIFIC_QUERIES, self.setup_helper)

        zone = self.test_with_baton.irods_server.users[0].zone
        self.mapper = BatonUpdateMapper(self.test_with_baton.baton_location, zone)

    def test_get_all_since_with_date_in_future(self):
        updates = self.mapper.get_all_since(datetime.fromtimestamp(_MAX_IRODS_TIMESTAMP))
        self.assertEqual(len(updates), 0)

    def test_get_all_since_with_date_in_past(self):
        start_timestamp = self._get_latest_update_timestamp()

        updates = self.mapper.get_all_since(start_timestamp)
        self.assertEqual(len(updates), 0)

    def test_get_all_since_with_data_object_updates(self):
        start_timestamp = self._get_latest_update_timestamp()
        location_1 = self.setup_helper.create_data_object(_DATA_OBJECT_NAMES[0])
        location_2 = self.setup_helper.create_data_object(_DATA_OBJECT_NAMES[1])

        updates = self.mapper.get_all_since(start_timestamp)
        self.assertEqual(len(updates), 2)
        self.assertEqual(len(updates.get_entity_updates(location_1)), 1)
        self.assertEqual(len(updates.get_entity_updates(location_2)), 1)
        # TODO: More detailed check on updates

    def test_get_all_since_with_updates_to_data_object_replica(self):
        start_timestamp = self._get_latest_update_timestamp()
        location = self.setup_helper.create_data_object(_DATA_OBJECT_NAMES[0])
        resource = self.setup_helper.create_replica_storage()
        self.setup_helper.replicate_data_object(location, resource)
        self.setup_helper.update_checksums(location)

        checksum = self.setup_helper.get_checksum(location)
        replicas = DataObjectReplicaCollection([DataObjectReplica(i, checksum) for i in range(2)])
        expected_modification = DataObjectModification(modified_replicas=replicas)
        expected_metadata = Metadata(DataObjectModificationJSONEncoder().default(expected_modification))

        updates = self.mapper.get_all_since(start_timestamp)
        self.assertEquals(len(updates), 1)
        self.assertIn(updates[0].target, location)
        self.assertCountEqual(updates[0].metadata, expected_metadata)

    def test_get_all_since_with_metadata_update(self):
        path = self.setup_helper.create_data_object(_DATA_OBJECT_NAMES[0])
        start_timestamp = self._get_latest_update_timestamp()

        metadata_1 = Metadata({
            _METADATA_KEYS[0]: _METADATA_VALUES[0],
            _METADATA_KEYS[1]: _METADATA_VALUES[1]
        })
        self.setup_helper.add_metadata_to(path, metadata_1)
        # Update pre-existing metadata item
        metadata_2 = Metadata({_METADATA_KEYS[0]: _METADATA_VALUES[2]})
        self.setup_helper.add_metadata_to(path, metadata_2)
        expected_irods_metadata = IrodsMetadata({
            _METADATA_KEYS[0]: {_METADATA_VALUES[0], _METADATA_VALUES[2]},
            _METADATA_KEYS[1]: {_METADATA_VALUES[1]}
        })

        modification = DataObjectModification(modified_metadata=expected_irods_metadata)
        expected_update_metadata = Metadata(DataObjectModificationJSONEncoder().default(modification))

        updates = self.mapper.get_all_since(start_timestamp)
        self.assertEqual(len(updates), 1)
        relevant_updates = updates.get_entity_updates(path)
        # Expect the mapper to have combined all updates into one (https://github.com/wtsi-hgi/cookie-monster/issues/3)
        self.assertEqual(len(relevant_updates), 1)
        self.assertEqual(relevant_updates[0].target, path)
        logging.debug(relevant_updates[0].metadata)
        logging.debug(expected_update_metadata)
        self.assertCountEqual(relevant_updates[0].metadata, expected_update_metadata)

    def _get_latest_update_timestamp(self) -> datetime:
        """
        Gets the timestamp of the latest update. If there has been no updates, returns minimum timestamp.

        This timestamp is useful to get before running a test for use in filtering out any updates that iRODS
        already has. The Dockerized iRODS 3.3.1, for example, will have updates on start.
        :return: timestamp of latest update
        """
        inital_updates = self.mapper.get_all_since(datetime.min)
        if len(inital_updates) == 0:
            return datetime.min
        return inital_updates.get_most_recent()[0].timestamp

    def tearDown(self):
        self.test_with_baton.tear_down()