def install_queries(required_specific_queries: Dict[str, str], setup_helper: SetupHelper):
"""
Installs the given specific queries required to in iRODS using the given iRODS setup helper.
:param required_specific_queries: dictionary of specific queries, where the key is the alias and the value is the
SQL query
:param setup_helper: iRODS setup helper
"""
for alias, query_location_relative_to_root in required_specific_queries.items():
query_location = normpath(join(dirname(realpath(__file__)), "..", "..", "..", "..", "..",
query_location_relative_to_root))
with open(query_location) as file:
query = file.read().replace('\n', ' ')
setup_helper.run_icommand(["iadmin", "asq", "%s" % query, alias])
class TestBatonInstalledSpecificQueryMapper(unittest.TestCase):
"""
Tests for `BatonSpecificQueryMapper`.
"""
def setUp(self):
self.test_with_baton = TestWithBaton(baton_setup=BATON_SETUP)
self.test_with_baton.setup()
self.setup_helper = SetupHelper(self.test_with_baton.icommands_location)
self.mapper = BatonSpecificQueryMapper(self.test_with_baton.baton_location)
def test_get_all(self):
iquest_ls_response = self.setup_helper.run_icommand(["iquest", "--sql", "ls"])
expected = TestBatonInstalledSpecificQueryMapper._parse_iquest_ls(iquest_ls_response)
specific_queries = self.mapper.get_all()
self.assertCountEqual(specific_queries, expected)
@staticmethod
def _parse_iquest_ls(iquest_ls_response: str) -> Sequence[SpecificQuery]:
"""
Gets the installed specific queries by parsing the output returned by "iquest --sql ls".
:param iquest_ls_response: the response returned by the iquest command
:return: the specific queries installed on the iRODS server
"""
iquest_ls_response_lines = iquest_ls_response.split('\n')
assert (len(iquest_ls_response_lines) + 1) % 3 == 0
specific_queries = []
for i in range(int((len(iquest_ls_response_lines) + 1) / 3)):
i3 = int(3 * i)
alias = iquest_ls_response_lines[i3]
sql = iquest_ls_response_lines[i3 + 1]
specific_queries.append(SpecificQuery(alias, sql))
return specific_queries
def create_data_object(
test_with_baton: TestWithBaton,
name: str,
metadata: IrodsMetadata = IrodsMetadata(),
access_controls: Iterable[AccessControl] = None,
) -> DataObject:
"""
Factory method to create an iRODS data object that has metadata, an ACL and replicas. Creates in current directory.
:param test_with_baton: framework to allow testing with baton
:param name: the name given to the created data object
:param metadata: the metadata to give the file
:param access_controls: access control list that the data object should have
:return: the created iRODS file
"""
user = test_with_baton.irods_server.users[0]
setup_helper = SetupHelper(test_with_baton.icommands_location)
path = setup_helper.create_data_object(name)
setup_helper.add_metadata_to(path, metadata)
checksum = setup_helper.get_checksum(path)
replicas = []
for i in range(2):
replica_storage = setup_helper.create_replica_storage()
setup_helper.replicate_data_object(path, replica_storage)
replica = DataObjectReplica(i + 1, checksum, replica_storage.host, replica_storage.name, True)
replicas.append(replica)
setup_helper.update_checksums(path)
# Difficult to get all the details of replica 0 using icommands so remove
setup_helper.run_icommand(["irm", "-n", "0", path])
if access_controls is None:
access_controls = [AccessControl(User(user.username, user.zone), AccessControl.Level.OWN)]
else:
_set_access_controls(test_with_baton, path, access_controls)
data_object = DataObject(path, access_controls, metadata, replicas)
synchronise_timestamps(test_with_baton, data_object)
return data_object
def create_entity_tree(
test_with_baton: TestWithBaton, root: str, node: EntityNode, access_controls: Iterable[AccessControl] = None
) -> Iterable[IrodsEntity]:
"""
TODO
:param test_with_baton:
:param root:
:param node:
:param access_controls:
:return:
"""
entities = []
setup_helper = SetupHelper(test_with_baton.icommands_location)
if isinstance(node, DataObjectNode):
entity = DataObject(setup_helper.create_data_object(node.name))
else:
entity = Collection(setup_helper.create_collection(node.name))
_set_access_controls(test_with_baton, entity.path, access_controls)
entity.access_controls = access_controls
new_path = "%s/%s" % (root, entity.get_name())
setup_helper.run_icommand(["imv", entity.path, new_path])
entity.path = new_path
entities.append(entity)
if isinstance(node, CollectionNode):
for child in node.children:
descendants = create_entity_tree(test_with_baton, "%s/%s" % (root, node.name), child, access_controls)
entities.extend(descendants)
if isinstance(node, CollectionNode):
assert len(entities) == len(list(node.get_all_descendants()) + [node])
else:
assert len(entities) == 1
return entities
class TestTestWithBaton(unittest.TestCase, BatonSetupContainer, metaclass=ABCMeta):
"""
Unit tests for `TestWithBaton`.
"""
def setUp(self):
self.test_with_baton = TestWithBaton(self.baton_setup.value[0], self.baton_setup.value[1])
self.test_with_baton.setup()
self.setup_helper = SetupHelper(self.test_with_baton.icommands_location)
def tearDown(self):
self.test_with_baton.tear_down()
def test_can_use_icommand_binary(self):
user = self.test_with_baton.irods_server.users[0]
zone = user.zone
username = user.username
self.assertEquals(self.setup_helper.run_icommand(["ils"]), "/%s/home/%s:" % (zone, username))
def test_can_use_baton_binary(self):
process = subprocess.Popen(["%s/baton" % self.test_with_baton.baton_location],
stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=None)
out, error = process.communicate()
self.assertEqual(str.strip(out.decode("utf-8")), "{\"avus\":[]}")
self.assertEqual(error, None)
def test_tear_down(self):
baton_location = self.test_with_baton.baton_location
icommands_location = self.test_with_baton.icommands_location
self.test_with_baton.tear_down()
self.assertFalse(os.path.exists(baton_location))
self.assertFalse(os.path.exists(icommands_location))
self.assertIsNone(self.test_with_baton.baton_location)
self.assertIsNone(self.test_with_baton.icommands_location)
def test_cannot_setup_if_already_setup(self):
self.assertRaises(RuntimeError, self.test_with_baton.setup)
def test_cannot_setup_after_tear_down(self):
self.test_with_baton.tear_down()
self.assertRaises(RuntimeError, self.test_with_baton.setup)
class _TestBatonIrodsEntityMapper(unittest.TestCase, metaclass=ABCMeta):
"""
Tests for subclasses of `_BatonIrodsEntityMapper`.
"""
def setUp(self):
self.test_with_baton = TestWithBaton(baton_setup=BATON_SETUP)
self.test_with_baton.setup()
self.setup_helper = SetupHelper(self.test_with_baton.icommands_location)
self.metadata_1 = IrodsMetadata({ATTRIBUTES[0]: {"something_else", VALUES[0]}})
self.metadata_2 = IrodsMetadata({ATTRIBUTES[1]: {VALUES[1]}})
self.metadata_1_2 = combine_metadata([self.metadata_1, self.metadata_2])
self.search_criterion_1 = SearchCriterion(ATTRIBUTES[0], VALUES[0], ComparisonOperator.EQUALS)
self.search_criterion_2 = SearchCriterion(ATTRIBUTES[1], VALUES[1], ComparisonOperator.EQUALS)
def tearDown(self):
self.test_with_baton.tear_down()
@abstractmethod
def create_mapper(self) -> _BatonIrodsEntityMapper:
"""
Creates a mapper to test with.
:return: the created mapper
"""
@abstractmethod
def create_irods_entity(self, name: str, metadata: IrodsMetadata=IrodsMetadata()) -> IrodsEntity:
"""
Creates an iRODS entity to test with
:param name: the name of the entity to create
:param metadata: the metadata to give to the entity
:return: the created entity
"""
def test_get_by_metadata_when_no_metadata(self):
retrieved_entities = self.create_mapper().get_by_metadata(
SearchCriterion(ATTRIBUTES[0], UNUSED_VALUE, ComparisonOperator.EQUALS))
self.assertEqual(len(retrieved_entities), 0)
def test_get_by_metadata_when_single_criterion_match_single_file(self):
irods_entity_1 = self.create_irods_entity(NAMES[0], self.metadata_1)
retrieved_entities = self.create_mapper().get_by_metadata(self.search_criterion_1)
self.assertEqual(retrieved_entities, [irods_entity_1])
def test_get_by_metadata_when_multiple_criterions_match_single_entity(self):
search_criteria = [self.search_criterion_1, self.search_criterion_2]
irods_entity_1 = self.create_irods_entity(NAMES[0], self.metadata_1_2)
self.create_irods_entity(NAMES[1], self.metadata_1)
retrieved_entities = self.create_mapper().get_by_metadata(search_criteria)
self.maxDiff = None
self.assertEqual(retrieved_entities, [irods_entity_1])
def test_get_by_metadata_when_single_criterion_match_multiple_entities(self):
irods_entity_1 = self.create_irods_entity(NAMES[0], self.metadata_1_2)
irods_entity_2 = self.create_irods_entity(NAMES[1], self.metadata_1)
self.create_irods_entity(NAMES[2], IrodsMetadata())
retrieved_entities = self.create_mapper().get_by_metadata(self.search_criterion_1)
self.maxDiff = None
self.assertEqual(retrieved_entities, [irods_entity_1, irods_entity_2])
def test_get_by_metadata_when_multiple_criterions_match_multiple_entities(self):
search_criteria = [self.search_criterion_1, self.search_criterion_2]
irods_entity_1 = self.create_irods_entity(NAMES[0], self.metadata_1_2)
irods_entity_2 = self.create_irods_entity(NAMES[1], self.metadata_1_2)
self.create_irods_entity(NAMES[2], IrodsMetadata())
retrieved_entities = self.create_mapper().get_by_metadata(search_criteria)
self.maxDiff = None
self.assertEqual(retrieved_entities, [irods_entity_1, irods_entity_2])
def test_get_by_metadata_when_metadata_not_required_for_entities(self):
irods_entity_1 = self.create_irods_entity(NAMES[0], self.metadata_1)
retrieved_entities = self.create_mapper().get_by_metadata(self.search_criterion_1, load_metadata=False)
self.assertIsNone(retrieved_entities[0].metadata)
irods_entity_1.metadata = None
self.assertEqual(retrieved_entities[0], irods_entity_1)
@unittest.skip("Unable to setup a new zone in iRODS")
def test_get_by_metadata_when_zone_restricted(self):
new_zone = "newZone"
self.setup_helper.run_icommand(["iadmin", "mkzone %s remote" % new_zone])
irods_entity_1 = self.create_irods_entity(NAMES[0], self.metadata_1)
irods_entity_2 = self.create_irods_entity(NAMES[1], self.metadata_1)
self.setup_helper.run_icommand(["icp", "-r", "%s /%s" % (irods_entity_2.path, new_zone)])
irods_entity_3 = deepcopy(irods_entity_2)
irods_entity_3.path = "/%s/%s" % (new_zone, irods_entity_2.path.split("/")[-1])
mapper = self.create_mapper()
# Check gets both without specifying zone
self.assertEqual(mapper.get_by_metadata(self.search_criterion_1), [irods_entity_1, irods_entity_2])
# Check can zone restrict
self.assertEqual(mapper.get_by_metadata(self.search_criterion_1, zone=new_zone), [irods_entity_3])
def test_get_by_path_when_no_paths_given(self):
retrieved_entities = self.create_mapper().get_by_path([])
self.assertEqual(len(retrieved_entities), 0)
def test_get_by_path_when_entity_does_not_exist(self):
self.assertRaises(FileNotFoundError, self.create_mapper().get_by_path, "/invalid/name")
def test_get_by_path_with_single_entity(self):
irods_entity_1 = self.create_irods_entity(NAMES[0], self.metadata_1)
retrieved_entity = self.create_mapper().get_by_path(irods_entity_1.path)
self.assertEqual(retrieved_entity, irods_entity_1)
def test_get_by_path_with_multiple_entities(self):
irods_entities = [
self.create_irods_entity(NAMES[i], self.metadata_1) for i in range(len(NAMES))]
paths = [irods_entity.path for irods_entity in irods_entities]
retrieved_entities = self.create_mapper().get_by_path(paths)
self.assertEqual(retrieved_entities, irods_entities)
def test_get_by_path_with_multiple_files_when_some_do_not_exist(self):
irods_entities = [
self.create_irods_entity(NAMES[i], self.metadata_1) for i in range(len(NAMES))]
paths = [irods_entity.path for irods_entity in irods_entities]
self.assertRaises(
FileNotFoundError, self.create_mapper().get_by_path, paths + ["/invalid/name"])
def test_get_by_path_when_metadata_not_required(self):
irods_entity_1 = self.create_irods_entity(NAMES[0], self.metadata_1)
retrieved_entity = self.create_mapper().get_by_path(irods_entity_1.path, load_metadata=False)
self.assertIsNone(retrieved_entity.metadata)
irods_entity_1.metadata = None
self.assertEqual(retrieved_entity, irods_entity_1)
def test_get_all_in_collection_when_collection_does_not_exist(self):
self.assertRaises(FileNotFoundError, self.create_mapper().get_all_in_collection, "/invalid")
def test_get_all_in_collection_when_one_of_multiple_collections_does_not_exist(self):
collection_paths = [self.setup_helper.create_collection("collection"), "/invalid"]
self.assertRaises(FileNotFoundError, self.create_mapper().get_all_in_collection, collection_paths)
def test_get_all_in_collection_when_no_paths_given(self):
retrieved = self.create_mapper().get_all_in_collection([])
self.assertEqual(len(retrieved), 0)
def test_get_all_in_collection_with_single_collection_containing_one_entity(self):
entity = self.create_irods_entity(NAMES[0], self.metadata_1)
retrieved_entities = self.create_mapper().get_all_in_collection(entity.get_collection_path())
self.assertEqual(retrieved_entities, [entity])
def test_get_all_in_collection_with_single_collection_containing_multiple_entities(self):
entity_1 = self.create_irods_entity(NAMES[0], self.metadata_1)
entity_2 = self.create_irods_entity(NAMES[1], self.metadata_2)
assert entity_1.get_collection_path() == entity_2.get_collection_path()
retrieved_entities = self.create_mapper().get_all_in_collection(entity_1.get_collection_path())
self.assertEqual(retrieved_entities, [entity_1, entity_2])
def test_get_all_in_collection_with_multiple_collections(self):
collections = []
entities = []
for i in range(3):
collection = self.setup_helper.create_collection("collection_%d" % i)
for j in range(len(NAMES)):
entity = self.create_irods_entity(NAMES[j], self.metadata_1)
moved_path = "%s/%s" % (collection, entity.get_name())
self.setup_helper.run_icommand(["imv", entity.path, moved_path])
entity.path = moved_path
synchronise_timestamps(self.test_with_baton, entity)
entities.append(entity)
collections.append(collection)
retrieved_entities = self.create_mapper().get_all_in_collection(collections)
self.assertEqual(retrieved_entities, entities)
def test_get_all_in_collection_when_metadata_not_required(self):
entity = self.create_irods_entity(NAMES[0], self.metadata_1)
self.create_irods_entity(NAMES[1], self.metadata_1)
retrieved_entities = self.create_mapper().get_all_in_collection(
entity.get_collection_path(), load_metadata=False)
self.assertIsNone(retrieved_entities[0].metadata)
entity.metadata = None
self.assertEqual(retrieved_entities[0], entity)
def test_get_all_in_collection_when_collection_contains_data_objects_and_collections(self):
data_object = create_data_object(self.test_with_baton, NAMES[0], self.metadata_1)
create_collection(self.test_with_baton, NAMES[1], self.metadata_2)
retrieved_entities = self.create_mapper().get_all_in_collection(data_object.get_collection_path())
self.assertEqual(len(retrieved_entities), 1)
self.assertIsInstance(retrieved_entities[0], type(self.create_irods_entity(NAMES[2])))
def test_access_control_property(self):
self.assertIsInstance(self.create_mapper().access_control, AccessControlMapper)
