def test_nested_nd_set_value():
"""Test that setting a :class:`~polymatheia.data.NavigableDict` does not needlessly coerce."""
tmp = NavigableDict({'a': 1})
tmp.a = NavigableDict({'one': 1})
assert tmp['a'].one == 1
assert tmp['a']['one'] == 1
assert tmp['a']['one'] == tmp.a.one
def test_basic_set_value():
"""Test that setting a single value works."""
tmp = NavigableDict({'a': 1})
assert tmp.a == 1
tmp.a = 2
assert tmp.a == 2
assert tmp['a'] == 2
def test_nested_set_value():
"""Test that setting a nested value works and coerces the nested value."""
tmp = NavigableDict({'a': 1})
tmp.a = {'one': 1}
assert tmp['a'].one == 1
assert tmp['a']['one'] == 1
assert tmp['a']['one'] == tmp.a.one
def test_merge_list():
"""Test that merging two lists works correctly."""
tmp = NavigableDict({'data': {'a': [1, 2]}})
tmp.merge({'data': {'a': [3, 4, 5]}})
assert len(tmp) == 1
assert len(tmp.data) == 1
assert len(tmp.data.a) == 5
def test_merge_overwrite():
"""Test that in the case of a type mismatch, the values are overwritten."""
tmp = NavigableDict({'data': {'a': 1}})
tmp.merge({'data': 'None', 'meta': 'Available'})
assert len(tmp) == 2
assert tmp.data == 'None'
assert tmp.meta == 'Available'
def test_setting_dotted_path_list_value():
"""Test that setting a value via a dotted path creates the required structure."""
tmp = NavigableDict({'data': {'numbers': [1, 2, 3]}})
tmp.set('data.numbers.0', 4)
tmp.set('data.numbers.2', 0)
assert tmp.data.numbers[0] == 4
assert tmp.data.numbers[2] == 0
def test_nested_get_value():
"""Test that accessing a nested value works."""
tmp = NavigableDict({'a': {'one': 1}})
assert tmp.a.one == 1
assert tmp['a'].one == 1
assert tmp['a']['one'] == 1
assert tmp['a']['one'] == tmp.a.one
assert tmp.get('a.one') == 1
def test_merge_dict():
"""Test that merging in a dict is coerced correctly."""
tmp = NavigableDict({'data': {'a': 1}})
tmp.merge({'data': {'b': 2}, 'meta': 'Available'})
assert len(tmp) == 2
assert len(tmp.data) == 2
assert tmp.data.a == 1
assert tmp.data.b == 2
assert tmp.meta == 'Available'
def test_merge_navigable_dict():
"""Test that merging in a NavigableDict works."""
tmp = NavigableDict({'data': {'a': 1}})
tmp.merge(NavigableDict({'data': {'b': 2}, 'meta': 'Available'}))
assert len(tmp) == 2
assert len(tmp.data) == 2
assert tmp.data.a == 1
assert tmp.data.b == 2
assert tmp.meta == 'Available'
def __init__(self, url):
"""Construct a new class:`~polymatheia.data.reader.SRUExplainRecordReader`.
:param url: The base URL of the SRU server
:type url: ``str``
"""
self._url = url
self._explain = SRUpy(self._url).explain()
self.schemas = [(schema["@name"], schema.title) for schema in
NavigableDict(self._explain).explain.schemaInfo.schema]
self.echo = NavigableDict(self._explain.echo)
def test_delete_value():
"""Test that deleting a value works."""
tmp = NavigableDict({'a': 1})
assert tmp.a == 1
del tmp.a
with pytest.raises(KeyError):
tmp.a
def test_join_list_transform():
"""Test the list joining transformation."""
record = NavigableDict({'a': {'tokens': ['This', 'is', 'a', 'test']}})
transform = Transform([('join', 'a', ' ', 'a.tokens')])
result = transform(record)
assert len(result) == 1
assert result.a == 'This is a test'
def test_combine_transform():
"""Test the combine transformation."""
record = NavigableDict({'a': {'one': {'_text': '1'}}, 'b': {'two': {'_text': '2'}}})
transform = Transform([('combine', 'a', 'a.one._text', 'b.two._text')])
result = transform(record)
assert len(result) == 1
assert result.a == ['1', '2']
def test_static_transform():
"""Test the static transformation."""
record = NavigableDict({'a': {'one': {'_text': '1'}}, 'b': {'two': {'_text': '2'}}})
transform = Transform(('static', 'a', 'Test'))
result = transform(record)
assert len(result) == 1
assert result.a == 'Test'
def test_fill_no_overwrite():
"""Test that filling ``None`` values works."""
record = NavigableDict({'a': {'two': {'_text': '1'}}})
transform = Transform([('sequence', ('copy', 'text', 'a.two._text'), ('fill', 'text', ''))])
result = transform(record)
assert len(result) == 1
assert result.text == '1'
def test_sequence_transform():
"""Test the sequence transformation."""
record = NavigableDict({'a': {'one': {'_text': '1'}}, 'b': {'two': {'_text': '2'}}})
transform = Transform([('sequence', ('combine', 'a', 'a.one._text', 'b.two._text'), ('join', 'a', ', ', 'a'))])
result = transform(record)
assert len(result) == 1
assert result.a == '1, 2'
def _run_search(self):
"""Run the actual search query."""
params = [('wskey', self._api_key), ('query', self._query),
('cursor', self._cursor)]
if self._max_records and self._max_records < 50:
params.append(('rows', self._max_records))
else:
params.append(('rows', 50))
if self._query_facets:
params.extend([('qf', qf) for qf in self._query_facets])
if self._media is not None:
params.append(('media', 'true' if self._media else 'false'))
if self._thumbnail is not None:
params.append(
('thumbnail', 'true' if self._thumbnail else 'false'))
if self._reusability is not None:
params.append(('reusability', self._reusability))
if self._profile is not None:
params.append(('profile', self._profile))
response = get('https://api.europeana.eu/record/v2/search.json',
params=params)
if response.status_code == 200:
data = response.json()
self._it = iter(data['items'])
self.result_count = data['totalResults']
self._offset = self._offset + data['itemsCount']
if 'nextCursor' in data:
self._cursor = data['nextCursor']
if 'facets' in data:
self.facets = [
NavigableDict(facet) for facet in data['facets']
]
else:
raise Exception(response.json()['error'])
def test_join_values_transform():
"""Test the value joining transformation."""
record = NavigableDict({'a': {'one': {'_text': '1'}}, 'b': {'two': {'_text': '2'}}})
transform = Transform([('join', 'a', ', ', 'a.one._text', 'b.two._text')])
result = transform(record)
assert len(result) == 1
assert result.a == '1, 2'
def __next__(self):
"""Return the next record as a :class:`~polymatheia.data.NavigableDict``.
:raises StopIteration: If no more Records are available
"""
if self._max_records is not None:
self._max_records = self._max_records - 1
if self._max_records < 0:
raise StopIteration()
try:
return NavigableDict(next(self._it))
except StopIteration:
if self._offset < self.result_count:
self._run_search()
return NavigableDict(next(self._it))
else:
raise StopIteration()
def test_copy_transform():
"""Test the copy transformation."""
record = NavigableDict({'a': {'one': {'_text': '1'}}, 'b': {'two': {'_text': '2'}}})
transform = Transform([('copy', 'a', 'a.one._text'), ('copy', 'b', 'b.two._text')])
result = transform(record)
assert len(result) == 2
assert result.a == '1'
assert result.b == '2'
def test_multiple_transform():
"""Test that having multiple transformations together works."""
record = NavigableDict({'a': {'one': {'_text': '1'}}, 'b': {'two': {'_text': '2'}}})
transform = Transform([('copy', 'a', 'a.one._text'), ('copy', 'b', 'b.two._text'), ('static', 'c', 'Test')])
result = transform(record)
assert len(result) == 3
assert result.a == '1'
assert result.b == '2'
assert result.c == 'Test'
def test_split_list_transform():
"""Test the list splitting transformation."""
record = NavigableDict({'a': {'tokens': ['This', 'is', 'a', 'test']}})
transform = Transform([('split', 'a_{}', '', 'a.tokens')])
result = transform(record)
assert len(result) == 4
assert result.a_1 == 'This'
assert result.a_2 == 'is'
assert result.a_3 == 'a'
assert result.a_4 == 'test'
def test_split_string_transform():
"""Test the string splitting transformation."""
record = NavigableDict({'a': {'_text': 'This is a test'}})
transform = Transform([('split', 'a_{}', ' ', 'a._text')])
result = transform(record)
assert len(result) == 4
assert result.a_1 == 'This'
assert result.a_2 == 'is'
assert result.a_3 == 'a'
assert result.a_4 == 'test'
def test_sequence_parallel():
"""Test that the combination of parallel transform and sequence transform works."""
record = NavigableDict({'a': {'one': {'_text': '1'}}, 'b': {'two': {'_text': '2'}}})
transform = Transform(('sequence', ('parallel', ('combine', 'a', 'a.one._text', 'b.two._text'),
('static', 'b', 'Test')),
('parallel', ('join', 'a', ', ', 'a'),
('copy', 'b', 'b'))))
result = transform(record)
assert len(result) == 2
assert result.a == '1, 2'
assert result.b == 'Test'
def test_parallel_transform():
"""Test that the parallel transform works as desired."""
record = NavigableDict({'a': {'one': {'_text': '1'}}, 'b': {'two': {'_text': '2'}}})
transform = Transform(('parallel', ('copy', 'data.a', 'a.one._text'),
('copy', 'data.b', 'b.two._text'),
('static', 'data.c', 'Test')))
result = transform(record)
assert len(result) == 1
assert result.data.a == '1'
assert result.data.b == '2'
assert result.data.c == 'Test'
def test_and_filter():
"""Test the and filter."""
fltr = Filter(('and',))
assert fltr(NavigableDict({'a': {'one': 1}})) is True
fltr = Filter(('and', ('eq', 'a.one', 1)))
assert fltr(NavigableDict({'a': {'one': 1}})) is True
assert fltr(NavigableDict({'a': {'one': 2}})) is False
assert fltr(NavigableDict({'a': {'one': '1'}})) is False
fltr = Filter(('and', ('eq', 'a.one', 1), ('eq', 'b.one', 1)))
assert fltr(NavigableDict({'a': {'one': 1}, 'b': {'one': 1}})) is True
assert fltr(NavigableDict({'a': {'one': 2}, 'b': {'one': 1}})) is False
assert fltr(NavigableDict({'a': {'one': 1}, 'b': {'one': 2}})) is False
assert fltr(NavigableDict({'a': {'one': 2}, 'b': {'one': 2}})) is False
def test_dotted_target_transform():
"""Tests that a dotted path as the target works."""
record = NavigableDict({'a': {'one': {'_text': '1'}}, 'b': {'two': {'_text': '2'}}})
transform = Transform([('copy', 'data.a', 'a.one._text'),
('copy', 'data.b', 'b.two._text'),
('static', 'data.c', 'Test')])
result = transform(record)
assert len(result) == 1
assert result.data.a == '1'
assert result.data.b == '2'
assert result.data.c == 'Test'
def test_nested_list():
"""Test that nested lists are correctly coerced."""
tmp = NavigableDict({'a': [{'one': 1}, NavigableDict({'two': 2}), 3]})
assert tmp.a[0].one == 1
assert tmp.a[1].two == 2
assert tmp.a[2] == 3
assert tmp.get('a.0.one') == 1
assert tmp.get('a[0].one') == 1
assert tmp.get('a.1.two') == 2
assert tmp.get('a.2') == 3
def test_setting_list_path():
"""Test that setting a value via a dotted path creates the required structure."""
tmp = NavigableDict({})
tmp.set(['data', 'type'], 'tests')
tmp.set(['data', 'id'], '1')
assert tmp.data.type == 'tests'
assert tmp.data.id == '1'
def __iter__(self):
"""Return a new class:`~polymatheia.data.NavigableDictIterator` as the iterator."""
sru_records = SRUpy(self._url).get_records(
query=self._query,
maximumRecords=self._max_records,
recordSchema=self._record_schema,
**self._kwargs)
self.record_count = sru_records.number_of_records
if sru_records.echo:
self.echo = NavigableDict(sru_records.echo)
return NavigableDictIterator(
sru_records,
mapper=lambda record: xml_to_navigable_dict(
etree.fromstring(
record.raw, parser=etree.XMLParser(remove_comments=True))))