def create_specific_query_with_baton_json_representation() -> Tuple[SpecificQuery, Dict]:
"""
Creates a specific query and returns it along with the JSON representation of it given by baton.
Uses baton to get the JSON representation on the first use: the JSON is retrieved from a cache in subsequent uses.
:return: a tuple where the first element is the created specific query and the second is its JSON representation
according to baton
"""
global _specific_query, _specific_query_as_json
# Starting baton is expensive - get view of baton JSON and cache
if _specific_query is None:
test_with_baton = TestWithBaton()
test_with_baton.setup()
baton_runner = BatonRunner(test_with_baton.baton_location, test_with_baton.irods_server.users[0].zone)
baton_query = baton_runner.run_baton_query(BatonBinary.BATON, ["-s", IRODS_SPECIFIC_QUERY_FIND_QUERY_BY_ALIAS,
"-b", IRODS_SPECIFIC_QUERY_FIND_QUERY_BY_ALIAS])
_specific_query_as_json = baton_runner.run_baton_query(
BatonBinary.BATON_SPECIFIC_QUERY, input_data=baton_query)[0]
_specific_query = SpecificQuery(
IRODS_SPECIFIC_QUERY_FIND_QUERY_BY_ALIAS, _specific_query_as_json[0][BATON_SPECIFIC_QUERY_SQL_PROPERTY])
return deepcopy(_specific_query), deepcopy(_specific_query_as_json)
class TestBatonInstalledSpecificQueryMapper(unittest.TestCase):
"""
Tests for `BatonSpecificQueryMapper`.
"""
def setUp(self):
self.test_with_baton = TestWithBaton(baton_setup=BATON_SETUP)
self.test_with_baton.setup()
self.setup_helper = SetupHelper(self.test_with_baton.icommands_location)
self.mapper = BatonSpecificQueryMapper(self.test_with_baton.baton_location)
def test_get_all(self):
iquest_ls_response = self.setup_helper.run_icommand(["iquest", "--sql", "ls"])
expected = TestBatonInstalledSpecificQueryMapper._parse_iquest_ls(iquest_ls_response)
specific_queries = self.mapper.get_all()
self.assertCountEqual(specific_queries, expected)
@staticmethod
def _parse_iquest_ls(iquest_ls_response: str) -> Sequence[SpecificQuery]:
"""
Gets the installed specific queries by parsing the output returned by "iquest --sql ls".
:param iquest_ls_response: the response returned by the iquest command
:return: the specific queries installed on the iRODS server
"""
iquest_ls_response_lines = iquest_ls_response.split('\n')
assert (len(iquest_ls_response_lines) + 1) % 3 == 0
specific_queries = []
for i in range(int((len(iquest_ls_response_lines) + 1) / 3)):
i3 = int(3 * i)
alias = iquest_ls_response_lines[i3]
sql = iquest_ls_response_lines[i3 + 1]
specific_queries.append(SpecificQuery(alias, sql))
return specific_queries
def create_collection_with_baton_json_representation() -> Tuple[Collection, Dict]:
"""
Creates a collection and returns it along with the JSON representation of it given by baton.
Uses baton to get the JSON representation on the first use: the JSON is retrieved from a cache in subsequent uses.
:return: a tuple where the first element is the created collection and the second is its JSON representation
according to baton
"""
global _collection, _collection_as_json
# Starting baton is expensive - get view of baton JSON and cache
if _collection is None:
test_with_baton = TestWithBaton()
test_with_baton.setup()
metadata = IrodsMetadata({"attribute_a": {"value_1", "value_2"}, "attribute_b": {"value_3"}})
_collection = create_collection(test_with_baton, "collection_1", metadata)
baton_query = {
BATON_COLLECTION_PROPERTY: _collection.path
}
baton_runner = BatonRunner(test_with_baton.baton_location, test_with_baton.irods_server.users[0].zone)
_collection_as_json = baton_runner.run_baton_query(
BatonBinary.BATON_LIST, ["--acl", "--avu"], input_data=baton_query)[0]
return deepcopy(_collection), deepcopy(_collection_as_json)
def setUp(self):
self.test_with_baton = TestWithBaton(baton_setup=BATON_SETUP)
self.test_with_baton.setup()
self.setup_helper = SetupHelper(self.test_with_baton.icommands_location)
install_queries(REQUIRED_SPECIFIC_QUERIES, self.setup_helper)
zone = self.test_with_baton.irods_server.users[0].zone
self.mapper = BatonUpdateMapper(self.test_with_baton.baton_location, zone)
def setUp(self):
self.test_with_baton = TestWithBaton(baton_setup=BATON_SETUP)
self.test_with_baton.setup()
self.setup_helper = SetupHelper(self.test_with_baton.icommands_location)
self.mapper = self.create_mapper()
self.metadata = IrodsMetadata({"key_1": {"value_1", "value_2"}, "key_2": {"value_3"}})
self._entity = None
self._entities = None
class TestBatonCustomObjectMapper(unittest.TestCase):
"""
Tests for `BatonCustomObjectMapper`.
"""
def setUp(self):
self.test_with_baton = TestWithBaton(baton_setup=BATON_SETUP)
self.test_with_baton.setup()
self.mapper = StubBatonCustomObjectMapper(self.test_with_baton.baton_location)
self.mapper._object_deserialiser = MagicMock(wraps=self.mapper._object_deserialiser)
def tearDown(self):
self.test_with_baton.tear_down()
def test_get_using_specific_query(self):
results = self.mapper._get_with_prepared_specific_query(PreparedSpecificQuery("ls"))
self.assertIsInstance(results, list)
self.assertEqual(len(results), self.mapper._object_deserialiser.call_count)
def setUp(self):
self.test_with_baton = TestWithBaton(baton_setup=BATON_SETUP)
self.test_with_baton.setup()
self.setup_helper = SetupHelper(self.test_with_baton.icommands_location)
self.metadata_1 = IrodsMetadata({ATTRIBUTES[0]: {"something_else", VALUES[0]}})
self.metadata_2 = IrodsMetadata({ATTRIBUTES[1]: {VALUES[1]}})
self.metadata_1_2 = combine_metadata([self.metadata_1, self.metadata_2])
self.search_criterion_1 = SearchCriterion(ATTRIBUTES[0], VALUES[0], ComparisonOperator.EQUALS)
self.search_criterion_2 = SearchCriterion(ATTRIBUTES[1], VALUES[1], ComparisonOperator.EQUALS)
class TestConnection(unittest.TestCase):
"""
Tests for `Connection` class.
"""
def setUp(self):
self.test_with_baton = TestWithBaton(baton_setup=BATON_SETUP)
def test_correct_mapper_properties(self):
self.test_with_baton.setup()
connection = Connection(self.test_with_baton.baton_location)
self.assertIsInstance(connection.data_object, BatonDataObjectMapper)
self.assertIsInstance(connection.collection, BatonCollectionMapper)
self.assertIsInstance(connection.specific_query, BatonSpecificQueryMapper)
def test_skip_baton_binaries_validation(self):
self.assertRaises(ValueError, Connection, "invalid", False)
def tearDown(self):
self.test_with_baton.tear_down()
def setUp(self):
self.test_with_baton = TestWithBaton(baton_setup=BATON_SETUP)
self.test_with_baton.setup()
self.setup_helper = SetupHelper(self.test_with_baton.icommands_location)
self.mapper = self.create_mapper()
self.users = []
for username in _USERNAMES:
user = User(username, self.test_with_baton.irods_server.users[0].zone)
self.setup_helper.create_user(user.name, user.zone)
self.users.append(user)
self.access_controls = [AccessControl(self.users[0], AccessControl.Level.WRITE),
AccessControl(self.users[1], AccessControl.Level.READ)]
self.access_control = AccessControl(self.users[2], AccessControl.Level.OWN)
def main():
"""
Main method.
"""
baton_version, irods_version = _parse_arguments()
baton_setup = _get_baton_setup(baton_version, irods_version)
test_with_baton = TestWithBaton(baton_setup=baton_setup)
test_with_baton.setup()
setup_information = {
"baton": test_with_baton.baton_location,
"icommands": test_with_baton.icommands_location
}
logging.info("Dumping setup information")
print(json.dumps(setup_information))
sys.stdout.flush()
def call_tear_down(*args, **kargs):
test_with_baton.tear_down()
signal.signal(signal.SIGHUP, call_tear_down)
signal.pause()
class TestTestWithBaton(unittest.TestCase, BatonSetupContainer, metaclass=ABCMeta):
"""
Unit tests for `TestWithBaton`.
"""
def setUp(self):
self.test_with_baton = TestWithBaton(self.baton_setup.value[0], self.baton_setup.value[1])
self.test_with_baton.setup()
self.setup_helper = SetupHelper(self.test_with_baton.icommands_location)
def tearDown(self):
self.test_with_baton.tear_down()
def test_can_use_icommand_binary(self):
user = self.test_with_baton.irods_server.users[0]
zone = user.zone
username = user.username
self.assertEquals(self.setup_helper.run_icommand(["ils"]), "/%s/home/%s:" % (zone, username))
def test_can_use_baton_binary(self):
process = subprocess.Popen(["%s/baton" % self.test_with_baton.baton_location],
stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=None)
out, error = process.communicate()
self.assertEqual(str.strip(out.decode("utf-8")), "{\"avus\":[]}")
self.assertEqual(error, None)
def test_tear_down(self):
baton_location = self.test_with_baton.baton_location
icommands_location = self.test_with_baton.icommands_location
self.test_with_baton.tear_down()
self.assertFalse(os.path.exists(baton_location))
self.assertFalse(os.path.exists(icommands_location))
self.assertIsNone(self.test_with_baton.baton_location)
self.assertIsNone(self.test_with_baton.icommands_location)
def test_cannot_setup_if_already_setup(self):
self.assertRaises(RuntimeError, self.test_with_baton.setup)
def test_cannot_setup_after_tear_down(self):
self.test_with_baton.tear_down()
self.assertRaises(RuntimeError, self.test_with_baton.setup)
class TestBatonRunner(unittest.TestCase):
"""
Tests for `_BatonRunner`.
"""
def setUp(self):
self.test_with_baton = TestWithBaton(baton_setup=BATON_SETUP)
def tearDown(self):
self.test_with_baton.tear_down()
def test_validate_baton_binaries_location_with_invalid_location(self):
self.assertIsInstance(BatonRunner.validate_baton_binaries_location("invalid"), ValueError)
def test_validate_baton_binaries_location_with_non_binaries_location(self):
self.assertIsInstance(BatonRunner.validate_baton_binaries_location("."), ValueError)
def test_validate_baton_binaries_location_with_binaries_location(self):
self.test_with_baton.setup()
self.assertIsNone(BatonRunner.validate_baton_binaries_location(self.test_with_baton.baton_location))
def test_init_with_invalid_baton_directory(self):
self.assertRaises(ValueError, StubBatonRunner, ".", "")
def test_init_with_valid_baton_directory(self):
self.test_with_baton.setup()
StubBatonRunner(self.test_with_baton.baton_location)
def test_run_baton_query(self):
self.test_with_baton.setup()
baton_runner = StubBatonRunner(self.test_with_baton.baton_location)
baton_out_as_json = baton_runner.run_baton_query(BatonBinary.BATON)[0]
self.assertIn("avus", baton_out_as_json)
self.assertEquals(baton_out_as_json["avus"], [])
def test_run_command_timeout(self):
timeout = timedelta(microseconds=1)
baton_runner = StubBatonRunner("", timeout_queries_after=timeout, skip_baton_binaries_validation=True)
self.assertRaises(TimeoutExpired, baton_runner._run_command, ["sleep", "999"])
class TestBatonUpdateMapper(unittest.TestCase):
"""
Tests for `BatonUpdateMapper`.
"""
def setUp(self):
self.test_with_baton = TestWithBaton(baton_setup=BATON_SETUP)
self.test_with_baton.setup()
self.setup_helper = SetupHelper(self.test_with_baton.icommands_location)
install_queries(REQUIRED_SPECIFIC_QUERIES, self.setup_helper)
zone = self.test_with_baton.irods_server.users[0].zone
self.mapper = BatonUpdateMapper(self.test_with_baton.baton_location, zone)
def test_get_all_since_with_date_in_future(self):
updates = self.mapper.get_all_since(datetime.fromtimestamp(_MAX_IRODS_TIMESTAMP))
self.assertEqual(len(updates), 0)
def test_get_all_since_with_date_in_past(self):
start_timestamp = self._get_latest_update_timestamp()
updates = self.mapper.get_all_since(start_timestamp)
self.assertEqual(len(updates), 0)
def test_get_all_since_with_data_object_updates(self):
start_timestamp = self._get_latest_update_timestamp()
location_1 = self.setup_helper.create_data_object(_DATA_OBJECT_NAMES[0])
location_2 = self.setup_helper.create_data_object(_DATA_OBJECT_NAMES[1])
updates = self.mapper.get_all_since(start_timestamp)
self.assertEqual(len(updates), 2)
self.assertEqual(len(updates.get_entity_updates(location_1)), 1)
self.assertEqual(len(updates.get_entity_updates(location_2)), 1)
# TODO: More detailed check on updates
def test_get_all_since_with_updates_to_data_object_replica(self):
start_timestamp = self._get_latest_update_timestamp()
location = self.setup_helper.create_data_object(_DATA_OBJECT_NAMES[0])
resource = self.setup_helper.create_replica_storage()
self.setup_helper.replicate_data_object(location, resource)
self.setup_helper.update_checksums(location)
checksum = self.setup_helper.get_checksum(location)
replicas = DataObjectReplicaCollection([DataObjectReplica(i, checksum) for i in range(2)])
expected_modification = DataObjectModification(modified_replicas=replicas)
expected_metadata = Metadata(DataObjectModificationJSONEncoder().default(expected_modification))
updates = self.mapper.get_all_since(start_timestamp)
self.assertEquals(len(updates), 1)
self.assertIn(updates[0].target, location)
self.assertCountEqual(updates[0].metadata, expected_metadata)
def test_get_all_since_with_metadata_update(self):
path = self.setup_helper.create_data_object(_DATA_OBJECT_NAMES[0])
start_timestamp = self._get_latest_update_timestamp()
metadata_1 = Metadata({
_METADATA_KEYS[0]: _METADATA_VALUES[0],
_METADATA_KEYS[1]: _METADATA_VALUES[1]
})
self.setup_helper.add_metadata_to(path, metadata_1)
# Update pre-existing metadata item
metadata_2 = Metadata({_METADATA_KEYS[0]: _METADATA_VALUES[2]})
self.setup_helper.add_metadata_to(path, metadata_2)
expected_irods_metadata = IrodsMetadata({
_METADATA_KEYS[0]: {_METADATA_VALUES[0], _METADATA_VALUES[2]},
_METADATA_KEYS[1]: {_METADATA_VALUES[1]}
})
modification = DataObjectModification(modified_metadata=expected_irods_metadata)
expected_update_metadata = Metadata(DataObjectModificationJSONEncoder().default(modification))
updates = self.mapper.get_all_since(start_timestamp)
self.assertEqual(len(updates), 1)
relevant_updates = updates.get_entity_updates(path)
# Expect the mapper to have combined all updates into one (https://github.com/wtsi-hgi/cookie-monster/issues/3)
self.assertEqual(len(relevant_updates), 1)
self.assertEqual(relevant_updates[0].target, path)
logging.debug(relevant_updates[0].metadata)
logging.debug(expected_update_metadata)
self.assertCountEqual(relevant_updates[0].metadata, expected_update_metadata)
def _get_latest_update_timestamp(self) -> datetime:
"""
Gets the timestamp of the latest update. If there has been no updates, returns minimum timestamp.
This timestamp is useful to get before running a test for use in filtering out any updates that iRODS
already has. The Dockerized iRODS 3.3.1, for example, will have updates on start.
:return: timestamp of latest update
"""
inital_updates = self.mapper.get_all_since(datetime.min)
if len(inital_updates) == 0:
return datetime.min
return inital_updates.get_most_recent()[0].timestamp
def tearDown(self):
self.test_with_baton.tear_down()
class _TestBatonIrodsEntityMetadataMapper(unittest.TestCase):
"""
Tests for `_BatonIrodsMetadataMapper`.
"""
@abstractmethod
def create_mapper(self) -> _BatonIrodsMetadataMapper:
"""
Creates a mapper to test with.
:return: the created mapper
"""
@abstractmethod
def create_irods_entity(self, name: str, metadata: IrodsMetadata=IrodsMetadata()) -> IrodsEntity:
"""
Creates an iRODS entity to test with
:param name: the name of the entity to create
:param metadata: the metadata to give to the entity
:return: the created entity
"""
def setUp(self):
self.test_with_baton = TestWithBaton(baton_setup=BATON_SETUP)
self.test_with_baton.setup()
self.setup_helper = SetupHelper(self.test_with_baton.icommands_location)
self.mapper = self.create_mapper()
self.metadata = IrodsMetadata({"key_1": {"value_1", "value_2"}, "key_2": {"value_3"}})
self._entity = None
self._entities = None
def tearDown(self):
self.test_with_baton.tear_down()
@property
def entity(self) -> IrodsEntity:
"""
Lazily creates the test entity (prevents spending time creating unused entities in iRODS).
:return: an example entity
"""
if self._entity is None:
self._entity = self.create_irods_entity("%s_entity" % NAMES[0])
return self._entity
@property
def entities(self) -> List[IrodsEntity]:
"""
Lazily creates a set of test entities (prevents spending time creating unused entities in iRODS).
:return: an example collection of entities
"""
if self._entities is None:
self._entities = [self.create_irods_entity(name) for name in NAMES]
return self._entities
def test_get_all_with_no_paths(self):
self.assertEqual(self.mapper.get_all([]), [])
def test_get_all_with_single_path_that_is_invalid(self):
self.assertRaises(FileNotFoundError, self.mapper.get_all, "/invalid")
def test_get_all_with_single_path(self):
entity = self.create_irods_entity(NAMES[0], self.metadata)
self.assertEqual(self.mapper.get_all(entity.path), self.metadata)
def test_get_all_with_multiple_paths_including_an_invalid_path(self):
self.assertRaises(FileNotFoundError, self.mapper.get_all, [self.entity.path, "/invalid"])
def test_get_all_with_multiple_paths(self):
entities = [self.create_irods_entity(name, self.metadata) for name in NAMES]
paths = [entity.path for entity in entities]
self.assertEqual(self.mapper.get_all(paths), [entity.metadata for entity in entities])
def test_add_with_single_path_that_is_invalid(self):
self.assertRaises(FileNotFoundError, self.mapper.add, "/invalid", self.metadata)
def test_add_with_single_path(self):
self.mapper.add(self.entity.path, self.metadata)
self.assertEqual(self.mapper.get_all(self.entity.path), self.metadata)
def test_add_with_multiple_paths_including_an_invalid_path(self):
self.assertRaises(FileNotFoundError, self.mapper.add, [self.entity.path, "/invalid"], self.metadata)
def test_add_with_multiple_paths_but_invalid_number_of_associated_metadata(self):
paths = [entity.path for entity in self.entities]
assert len(paths) > 2
self.assertRaises(ValueError, self.mapper.add, paths, [self.metadata, self.metadata])
def test_add_with_multiple_paths_and_single_metadata(self):
paths = [entity.path for entity in self.entities]
self.mapper.add(paths, self.metadata)
self.assertEqual(self.mapper.get_all(paths), [self.metadata for _ in self.entities])
def test_add_with_multiple_paths_and_multiple_metadata(self):
paths = [entity.path for entity in self.entities]
metadata = [self.metadata for _ in range(len(self.entities))]
self.mapper.add(paths, metadata)
self.assertEqual(self.mapper.get_all(paths), metadata)
def test_add_no_metadata(self):
self.mapper.add(self.entity.path, IrodsMetadata())
self.assertEqual(self.mapper.get_all(self.entity.path), IrodsMetadata())
def test_add_metadata_with_same_key(self):
entity = self.create_irods_entity(NAMES[0], self.metadata)
del self.metadata["key_1"]
assert len(self.metadata) == 1
self.assertRaises(KeyError, self.mapper.add, entity.path, self.metadata)
def test_add_adds_metadata(self):
entity = self.create_irods_entity(NAMES[0], self.metadata)
additional_metadata = IrodsMetadata({"other": {"values"}})
self.mapper.add(entity.path, additional_metadata)
self.assertEqual(self.mapper.get_all(entity.path),
IrodsMetadata({**dict(entity.metadata), **dict(additional_metadata)}))
def test_add_appends_if_key_exists_and_not_same_value(self):
values = ["value_1", "value_2"]
key = "key"
entity = self.create_irods_entity(NAMES[0], IrodsMetadata({key: {values[0]}}))
self.mapper.add(entity.path, IrodsMetadata({key: {values[1]}}))
self.assertEqual(IrodsMetadata({key: {values[0], values[1]}}), self.mapper.get_all(entity.path))
def test_set_with_no_paths(self):
self.mapper.set([], self.metadata)
def test_set_with_single_path_that_is_invalid(self):
self.assertRaises(FileNotFoundError, self.mapper.set, "/invalid", self.metadata)
def test_set_with_single_path(self):
self.mapper.set(self.entity.path, self.metadata)
self.assertEqual(self.mapper.get_all(self.entity.path), self.metadata)
def test_set_with_multiple_paths_including_an_invalid_path(self):
self.assertRaises(FileNotFoundError, self.mapper.set, [self.entity.path, "/invalid"], self.metadata)
def test_set_with_multiple_paths_and_single_metadata(self):
paths = [entity.path for entity in self.entities]
self.mapper.set(paths, self.metadata)
self.assertEqual(self.mapper.get_all(paths), [self.metadata for _ in self.entities])
def test_set_with_multiple_paths_and_multiple_metadata(self):
paths = [entity.path for entity in self.entities]
metadata = [self.metadata for _ in range(len(self.entities))]
self.mapper.set(paths, metadata)
self.assertEqual(self.mapper.get_all(paths), metadata)
def test_set_overrides_existing_metadata(self):
overriden_key = list(self.metadata.keys())[0]
value = {"new_value"}
entity_1 = self.create_irods_entity(NAMES[0], self.metadata)
del self.metadata[overriden_key]
entity_2 = self.create_irods_entity(NAMES[1], self.metadata)
self.mapper.set([entity_1.path, entity_2.path], IrodsMetadata({overriden_key: value}))
self.metadata[overriden_key] = value
self.assertEqual(self.mapper.get_all(entity_1.path), self.metadata)
self.assertEqual(self.mapper.get_all(entity_2.path), self.metadata)
def test_remove_with_no_paths(self):
self.mapper.remove([], self.metadata)
def test_remove_with_single_path_that_is_invalid(self):
self.assertRaises(FileNotFoundError, self.mapper.remove, "/invalid", self.metadata)
def test_remove_with_single_path(self):
entity = self.create_irods_entity(NAMES[0], self.metadata)
self.mapper.remove(entity.path, self.metadata)
self.assertEqual(self.mapper.get_all(entity.path), IrodsMetadata())
def test_remove_with_multiple_paths_including_an_invalid_path(self):
entity = self.create_irods_entity(NAMES[0], self.metadata)
self.assertRaises(FileNotFoundError, self.mapper.remove, [entity.path, "/invalid"], self.metadata)
def test_remove_with_multiple_paths_and_single_metadata(self):
entities = [self.create_irods_entity(name, self.metadata) for name in NAMES]
paths = [entity.path for entity in entities]
self.mapper.remove(paths, self.metadata)
self.assertEqual(self.mapper.get_all(paths), [IrodsMetadata() for _ in paths])
def test_remove_with_multiple_paths_and_multiple_metadata(self):
entities = [self.create_irods_entity(name, self.metadata) for name in NAMES]
paths = [entity.path for entity in entities]
metadata = [entity.metadata for entity in entities]
self.mapper.remove(paths, metadata)
self.assertEqual(self.mapper.get_all(paths), [IrodsMetadata() for _ in paths])
def test_remove_unset_metadata(self):
self.assertRaises(KeyError, self.mapper.remove, self.entity.path, self.metadata)
def test_remove_partially_unset_metadata(self):
partial_metadata = deepcopy(self.metadata)
del partial_metadata["key_1"]
entity = self.create_irods_entity(NAMES[0], partial_metadata)
self.assertRaises(KeyError, self.mapper.remove, entity.path, self.metadata)
def test_remove_subset_of_metadata(self):
entity = self.create_irods_entity(NAMES[0], self.metadata)
assert len(self.metadata) == 2
partial_metadata_1 = deepcopy(self.metadata)
del partial_metadata_1["key_1"]
partial_metadata_2 = deepcopy(self.metadata)
del partial_metadata_2["key_2"]
self.mapper.remove(entity.path, partial_metadata_1)
self.assertEqual(self.mapper.get_all(entity.path), partial_metadata_2)
def test_remove_all_with_no_paths(self):
self.mapper.remove_all([])
def test_remove_all_with_single_path_that_is_invalid(self):
self.assertRaises(FileNotFoundError, self.mapper.remove_all, "/invalid")
def test_remove_all_with_single_path(self):
entity = self.create_irods_entity(NAMES[0], self.metadata)
self.mapper.remove_all(entity.path)
self.assertEqual(self.mapper.get_all(entity.path), IrodsMetadata())
def test_remove_all_with_multiple_paths_including_an_invalid_path(self):
entity = self.create_irods_entity(NAMES[0], self.metadata)
self.assertRaises(FileNotFoundError, self.mapper.remove_all, [entity.path, "/invalid"])
def test_remove_all_with_multiple_paths(self):
entities = [self.create_irods_entity(name, self.metadata) for name in NAMES]
paths = [entity.path for entity in entities]
self.mapper.remove_all(paths)
self.assertEqual(self.mapper.get_all(paths), [IrodsMetadata() for _ in range(len(entities))])
def setUp(self):
self.test_with_baton = TestWithBaton(baton_setup=BATON_SETUP)
self.test_with_baton.setup()
self.setup_helper = SetupHelper(self.test_with_baton.icommands_location)
self.mapper = BatonSpecificQueryMapper(self.test_with_baton.baton_location)
def setUp(self):
self.test_with_baton = TestWithBaton(baton_setup=BATON_SETUP)
self.test_with_baton.setup()
self.mapper = StubBatonCustomObjectMapper(self.test_with_baton.baton_location)
self.mapper._object_deserialiser = MagicMock(wraps=self.mapper._object_deserialiser)
def setUp(self):
self.test_with_baton = TestWithBaton(self.baton_setup.value[0], self.baton_setup.value[1])
self.test_with_baton.setup()
self.setup_helper = SetupHelper(self.test_with_baton.icommands_location)
class _TestBatonAccessControlMapper(unittest.TestCase):
"""
Tests for `_BatonAccessControlMapper`.
"""
@abstractmethod
def create_mapper(self) -> _BatonAccessControlMapper:
"""
Creates a mapper to test with.
:return: the created mapper
"""
@abstractmethod
def create_irods_entity(self, name: str, access_controls: Iterable[AccessControl]) -> IrodsEntity:
"""
Creates an iRODS entity to test with
:param name: the name of the entity to create
:param access_controls: the access controls the entity should have
:return: the created entity
"""
def setUp(self):
self.test_with_baton = TestWithBaton(baton_setup=BATON_SETUP)
self.test_with_baton.setup()
self.setup_helper = SetupHelper(self.test_with_baton.icommands_location)
self.mapper = self.create_mapper()
self.users = []
for username in _USERNAMES:
user = User(username, self.test_with_baton.irods_server.users[0].zone)
self.setup_helper.create_user(user.name, user.zone)
self.users.append(user)
self.access_controls = [AccessControl(self.users[0], AccessControl.Level.WRITE),
AccessControl(self.users[1], AccessControl.Level.READ)]
self.access_control = AccessControl(self.users[2], AccessControl.Level.OWN)
def tearDown(self):
self.test_with_baton.tear_down()
def test_get_all_with_invalid_path(self):
self.assertRaises(FileNotFoundError, self.mapper.get_all, "/invalid")
def test_get_all_with_no_paths(self):
self.assertEqual(self.mapper.get_all([]), [])
def test_get_all_with_single_path(self):
entity = self.create_irods_entity(NAMES[0], self.access_controls)
self.assertEqual(self.mapper.get_all(entity.path), set(self.access_controls))
def test_get_all_with_multiple_path(self):
entity_1 = self.create_irods_entity(NAMES[0], self.access_controls)
entity_2 = self.create_irods_entity(NAMES[1], [self.access_control])
entity_3 = self.create_irods_entity(NAMES[2], [])
self.assertEqual(self.mapper.get_all([entity_1.path, entity_2.path, entity_3.path]),
[set(self.access_controls), {self.access_control}, set()])
def test_add_or_replace_with_invalid_path(self):
self.assertRaises(FileNotFoundError, self.mapper.add_or_replace, "/invalid", self.access_controls)
def test_add_or_replace_with_no_paths(self):
self.mapper.add_or_replace([], self.access_controls[0])
def test_add_or_replace_single_access_control(self):
entity = self.create_irods_entity(NAMES[0], [self.access_control])
self.mapper.add_or_replace(entity.path, self.access_controls[0])
self.assertCountEqual(self.mapper.get_all(entity.path), [self.access_control, self.access_controls[0]])
def test_add_or_replace_single_access_control_when_already_exists(self):
entity = self.create_irods_entity(NAMES[0], [self.access_control])
self.mapper.add_or_replace(entity.path, self.access_control)
self.assertEqual(self.mapper.get_all(entity.path), {self.access_control})
def test_add_or_replace_multiple_access_controls(self):
entity = self.create_irods_entity(NAMES[0], [self.access_control])
self.mapper.add_or_replace(entity.path, self.access_controls)
self.assertCountEqual(self.mapper.get_all(entity.path), [self.access_control] + self.access_controls)
def test_add_or_replace_multiple_access_controls_when_one_access_control_already_exists(self):
entity = self.create_irods_entity(NAMES[0], [self.access_control])
self.mapper.add_or_replace(entity.path, self.access_controls + [self.access_control])
self.assertEqual(self.mapper.get_all(entity.path), set(self.access_controls + [self.access_control]))
def test_set_with_invalid_path(self):
self.assertRaises(FileNotFoundError, self.mapper.set, "/invalid", self.access_controls)
def test_set_with_no_paths(self):
self.mapper.set([], self.access_controls)
def test_set_when_no_existing_access_controls(self):
entity = self.create_irods_entity(NAMES[0], ())
self.mapper.set(entity.path, self.access_controls)
self.assertEqual(self.mapper.get_all(entity.path), set(self.access_controls))
def test_set_when_existing_non_duplicate_access_controls(self):
entity = self.create_irods_entity(NAMES[0], [self.access_control])
self.mapper.set(entity.path, self.access_controls)
self.assertEqual(self.mapper.get_all(entity.path), set(self.access_controls))
def test_set_when_existing_duplicate_access_controls(self):
entity = self.create_irods_entity(NAMES[0], self.access_controls)
self.mapper.set(entity.path, self.access_controls)
self.assertEqual(self.mapper.get_all(entity.path), set(self.access_controls))
def test_revoke_with_invalid_path(self):
self.assertRaises(FileNotFoundError, self.mapper.revoke, "/invalid", self.access_control.user)
def test_revoke_with_no_paths(self):
self.mapper.revoke([], self.access_control.user)
def test_revoke_unset_access_control(self):
entity = self.create_irods_entity(NAMES[0], ())
self.mapper.revoke(entity.path, self.access_control.user)
self.assertEqual(self.mapper.get_all(entity.path), set())
def test_revoke_subset_of_access_controls(self):
entity = self.create_irods_entity(NAMES[0], self.access_controls + [self.access_control])
self.mapper.revoke(entity.path, [access_control.user for access_control in self.access_controls])
self.assertEqual(self.mapper.get_all(entity.path), {self.access_control})
def test_revoke_access_controls_using_string_representation_of_user(self):
entity = self.create_irods_entity(NAMES[0], [self.access_control])
self.mapper.revoke(entity.path, [str(self.access_control.user)])
self.assertEqual(self.mapper.get_all(entity.path), set())
def test_revoke_all_with_invalid_path(self):
self.assertRaises(FileNotFoundError, self.mapper.revoke_all, "/invalid")
def test_revoke_all_with_no_paths(self):
self.mapper.revoke_all([])
def test_revoke_all_with_single_path_and_no_access_controls(self):
entity = self.create_irods_entity(NAMES[0], ())
self.mapper.revoke_all(entity.path)
self.assertEqual(self.mapper.get_all(entity.path), set())
def test_revoke_all_with_single_path(self):
entity = self.create_irods_entity(NAMES[0], self.access_controls)
self.mapper.revoke_all(entity.path)
self.assertEqual(self.mapper.get_all(entity.path), set())
def test_revoke_all_with_multiple_paths(self):
entities = [self.create_irods_entity(name, self.access_controls) for name in NAMES]
paths = [entity.path for entity in entities]
self.mapper.revoke_all(paths)
self.assertEqual(self.mapper.get_all(paths), [set() for _ in range(len(paths))])
def setUp(self):
self.test_with_baton = TestWithBaton(baton_setup=BATON_SETUP)
class _TestBatonIrodsEntityMapper(unittest.TestCase, metaclass=ABCMeta):
"""
Tests for subclasses of `_BatonIrodsEntityMapper`.
"""
def setUp(self):
self.test_with_baton = TestWithBaton(baton_setup=BATON_SETUP)
self.test_with_baton.setup()
self.setup_helper = SetupHelper(self.test_with_baton.icommands_location)
self.metadata_1 = IrodsMetadata({ATTRIBUTES[0]: {"something_else", VALUES[0]}})
self.metadata_2 = IrodsMetadata({ATTRIBUTES[1]: {VALUES[1]}})
self.metadata_1_2 = combine_metadata([self.metadata_1, self.metadata_2])
self.search_criterion_1 = SearchCriterion(ATTRIBUTES[0], VALUES[0], ComparisonOperator.EQUALS)
self.search_criterion_2 = SearchCriterion(ATTRIBUTES[1], VALUES[1], ComparisonOperator.EQUALS)
def tearDown(self):
self.test_with_baton.tear_down()
@abstractmethod
def create_mapper(self) -> _BatonIrodsEntityMapper:
"""
Creates a mapper to test with.
:return: the created mapper
"""
@abstractmethod
def create_irods_entity(self, name: str, metadata: IrodsMetadata=IrodsMetadata()) -> IrodsEntity:
"""
Creates an iRODS entity to test with
:param name: the name of the entity to create
:param metadata: the metadata to give to the entity
:return: the created entity
"""
def test_get_by_metadata_when_no_metadata(self):
retrieved_entities = self.create_mapper().get_by_metadata(
SearchCriterion(ATTRIBUTES[0], UNUSED_VALUE, ComparisonOperator.EQUALS))
self.assertEqual(len(retrieved_entities), 0)
def test_get_by_metadata_when_single_criterion_match_single_file(self):
irods_entity_1 = self.create_irods_entity(NAMES[0], self.metadata_1)
retrieved_entities = self.create_mapper().get_by_metadata(self.search_criterion_1)
self.assertEqual(retrieved_entities, [irods_entity_1])
def test_get_by_metadata_when_multiple_criterions_match_single_entity(self):
search_criteria = [self.search_criterion_1, self.search_criterion_2]
irods_entity_1 = self.create_irods_entity(NAMES[0], self.metadata_1_2)
self.create_irods_entity(NAMES[1], self.metadata_1)
retrieved_entities = self.create_mapper().get_by_metadata(search_criteria)
self.maxDiff = None
self.assertEqual(retrieved_entities, [irods_entity_1])
def test_get_by_metadata_when_single_criterion_match_multiple_entities(self):
irods_entity_1 = self.create_irods_entity(NAMES[0], self.metadata_1_2)
irods_entity_2 = self.create_irods_entity(NAMES[1], self.metadata_1)
self.create_irods_entity(NAMES[2], IrodsMetadata())
retrieved_entities = self.create_mapper().get_by_metadata(self.search_criterion_1)
self.maxDiff = None
self.assertEqual(retrieved_entities, [irods_entity_1, irods_entity_2])
def test_get_by_metadata_when_multiple_criterions_match_multiple_entities(self):
search_criteria = [self.search_criterion_1, self.search_criterion_2]
irods_entity_1 = self.create_irods_entity(NAMES[0], self.metadata_1_2)
irods_entity_2 = self.create_irods_entity(NAMES[1], self.metadata_1_2)
self.create_irods_entity(NAMES[2], IrodsMetadata())
retrieved_entities = self.create_mapper().get_by_metadata(search_criteria)
self.maxDiff = None
self.assertEqual(retrieved_entities, [irods_entity_1, irods_entity_2])
def test_get_by_metadata_when_metadata_not_required_for_entities(self):
irods_entity_1 = self.create_irods_entity(NAMES[0], self.metadata_1)
retrieved_entities = self.create_mapper().get_by_metadata(self.search_criterion_1, load_metadata=False)
self.assertIsNone(retrieved_entities[0].metadata)
irods_entity_1.metadata = None
self.assertEqual(retrieved_entities[0], irods_entity_1)
@unittest.skip("Unable to setup a new zone in iRODS")
def test_get_by_metadata_when_zone_restricted(self):
new_zone = "newZone"
self.setup_helper.run_icommand(["iadmin", "mkzone %s remote" % new_zone])
irods_entity_1 = self.create_irods_entity(NAMES[0], self.metadata_1)
irods_entity_2 = self.create_irods_entity(NAMES[1], self.metadata_1)
self.setup_helper.run_icommand(["icp", "-r", "%s /%s" % (irods_entity_2.path, new_zone)])
irods_entity_3 = deepcopy(irods_entity_2)
irods_entity_3.path = "/%s/%s" % (new_zone, irods_entity_2.path.split("/")[-1])
mapper = self.create_mapper()
# Check gets both without specifying zone
self.assertEqual(mapper.get_by_metadata(self.search_criterion_1), [irods_entity_1, irods_entity_2])
# Check can zone restrict
self.assertEqual(mapper.get_by_metadata(self.search_criterion_1, zone=new_zone), [irods_entity_3])
def test_get_by_path_when_no_paths_given(self):
retrieved_entities = self.create_mapper().get_by_path([])
self.assertEqual(len(retrieved_entities), 0)
def test_get_by_path_when_entity_does_not_exist(self):
self.assertRaises(FileNotFoundError, self.create_mapper().get_by_path, "/invalid/name")
def test_get_by_path_with_single_entity(self):
irods_entity_1 = self.create_irods_entity(NAMES[0], self.metadata_1)
retrieved_entity = self.create_mapper().get_by_path(irods_entity_1.path)
self.assertEqual(retrieved_entity, irods_entity_1)
def test_get_by_path_with_multiple_entities(self):
irods_entities = [
self.create_irods_entity(NAMES[i], self.metadata_1) for i in range(len(NAMES))]
paths = [irods_entity.path for irods_entity in irods_entities]
retrieved_entities = self.create_mapper().get_by_path(paths)
self.assertEqual(retrieved_entities, irods_entities)
def test_get_by_path_with_multiple_files_when_some_do_not_exist(self):
irods_entities = [
self.create_irods_entity(NAMES[i], self.metadata_1) for i in range(len(NAMES))]
paths = [irods_entity.path for irods_entity in irods_entities]
self.assertRaises(
FileNotFoundError, self.create_mapper().get_by_path, paths + ["/invalid/name"])
def test_get_by_path_when_metadata_not_required(self):
irods_entity_1 = self.create_irods_entity(NAMES[0], self.metadata_1)
retrieved_entity = self.create_mapper().get_by_path(irods_entity_1.path, load_metadata=False)
self.assertIsNone(retrieved_entity.metadata)
irods_entity_1.metadata = None
self.assertEqual(retrieved_entity, irods_entity_1)
def test_get_all_in_collection_when_collection_does_not_exist(self):
self.assertRaises(FileNotFoundError, self.create_mapper().get_all_in_collection, "/invalid")
def test_get_all_in_collection_when_one_of_multiple_collections_does_not_exist(self):
collection_paths = [self.setup_helper.create_collection("collection"), "/invalid"]
self.assertRaises(FileNotFoundError, self.create_mapper().get_all_in_collection, collection_paths)
def test_get_all_in_collection_when_no_paths_given(self):
retrieved = self.create_mapper().get_all_in_collection([])
self.assertEqual(len(retrieved), 0)
def test_get_all_in_collection_with_single_collection_containing_one_entity(self):
entity = self.create_irods_entity(NAMES[0], self.metadata_1)
retrieved_entities = self.create_mapper().get_all_in_collection(entity.get_collection_path())
self.assertEqual(retrieved_entities, [entity])
def test_get_all_in_collection_with_single_collection_containing_multiple_entities(self):
entity_1 = self.create_irods_entity(NAMES[0], self.metadata_1)
entity_2 = self.create_irods_entity(NAMES[1], self.metadata_2)
assert entity_1.get_collection_path() == entity_2.get_collection_path()
retrieved_entities = self.create_mapper().get_all_in_collection(entity_1.get_collection_path())
self.assertEqual(retrieved_entities, [entity_1, entity_2])
def test_get_all_in_collection_with_multiple_collections(self):
collections = []
entities = []
for i in range(3):
collection = self.setup_helper.create_collection("collection_%d" % i)
for j in range(len(NAMES)):
entity = self.create_irods_entity(NAMES[j], self.metadata_1)
moved_path = "%s/%s" % (collection, entity.get_name())
self.setup_helper.run_icommand(["imv", entity.path, moved_path])
entity.path = moved_path
synchronise_timestamps(self.test_with_baton, entity)
entities.append(entity)
collections.append(collection)
retrieved_entities = self.create_mapper().get_all_in_collection(collections)
self.assertEqual(retrieved_entities, entities)
def test_get_all_in_collection_when_metadata_not_required(self):
entity = self.create_irods_entity(NAMES[0], self.metadata_1)
self.create_irods_entity(NAMES[1], self.metadata_1)
retrieved_entities = self.create_mapper().get_all_in_collection(
entity.get_collection_path(), load_metadata=False)
self.assertIsNone(retrieved_entities[0].metadata)
entity.metadata = None
self.assertEqual(retrieved_entities[0], entity)
def test_get_all_in_collection_when_collection_contains_data_objects_and_collections(self):
data_object = create_data_object(self.test_with_baton, NAMES[0], self.metadata_1)
create_collection(self.test_with_baton, NAMES[1], self.metadata_2)
retrieved_entities = self.create_mapper().get_all_in_collection(data_object.get_collection_path())
self.assertEqual(len(retrieved_entities), 1)
self.assertIsInstance(retrieved_entities[0], type(self.create_irods_entity(NAMES[2])))
def test_access_control_property(self):
self.assertIsInstance(self.create_mapper().access_control, AccessControlMapper)
