def test_get_package_dict(self):
h = InventoryHarvester()
harvest_object = self._get_test_harvest_object()
package_dict_defaults = PackageDictDefaults()
source_config = {}
existing_pkg = MockObject(resources=[])
pkg_dict = h.get_package_dict(harvest_object, package_dict_defaults,
source_config, existing_pkg)
pkg_dict['extras'] = sorted(pkg_dict['extras'])
pprint(pkg_dict)
assert_equal(pkg_dict, {
'name': 'test-dataset-co',
'title': u'Test dataset',
'notes': u'Test description',
'license_id': 'uk-ogl',
'state': 'active',
'tags': [],
'resources': [{'description': u'Some file - Download',
'format': 'CSV',
'resource_type': 'documentation',
'url': u'http://test.com/file.xls',
'schema-url': 'http://schemas.opendata.esd.org.uk/publictoilets/PublicToilets.xml?v=0.24',
'schema-type': 'csvlint',
}],
'extras': [{'key': 'harvest_source_reference', 'value': 'testguid'},
{'key': 'inventory_identifier', 'value': 'payments_over_500'},
{'key': 'la_function', 'value': ''},
{'key': 'la_service', 'value': ''}],
})
def test_harvest_basic(self):
# Create source
source_fixture = {
'title': 'Test Source',
'name': 'test-source',
'url': u'http://127.0.0.1:8999/esdInventory_live_truncated.xml',
'type': u'inventory',
}
source, job = self._create_source_and_job(source_fixture)
# Gather
harvester = InventoryHarvester()
# mock boundary stuff to avoid needing PostGIS - it is not tested here
# and that allows this test to run on sqlite
with patch('ckanext.dgulocal.harvester.get_boundary') as get_boundary:
get_boundary.return_value = None
object_ids = harvester.gather_stage(job)
assert_equal(len(object_ids), 3)
assert len(job.gather_errors) == 0
# Fetch
for object_id in object_ids:
harvest_object = HarvestObject.get(object_id)
assert harvest_object
success = harvester.fetch_stage(harvest_object)
assert_equal(success, True)
assert not harvest_object.errors
# Import
objects = []
for object_id in object_ids:
obj = HarvestObject.get(object_id)
assert obj
objects.append(obj)
harvester.import_stage(obj)
assert not harvest_object.errors
pkgs = Session.query(Package).filter(Package.type!=u'harvest_source').all()
assert_equal(len(pkgs), 3)
pkg_ids = [pkg.id for pkg in pkgs]
for obj in objects:
assert obj.current == True
assert obj.package_id in pkg_ids
def test_hard(self):
assert_equal(
InventoryHarvester._get_publisher_abbreviation(
MockObject(
extras={},
title='Department for Environment, Food & Rural Affairs')),
'DEFRA')
def test_get_package_dict(self):
h = InventoryHarvester()
harvest_object = self._get_test_harvest_object()
package_dict_defaults = PackageDictDefaults()
source_config = {}
existing_pkg = MockObject(resources=[])
pkg_dict = h.get_package_dict(harvest_object, package_dict_defaults,
source_config, existing_pkg)
pkg_dict['extras'] = sorted(pkg_dict['extras'])
pprint(pkg_dict)
assert_equal(
pkg_dict, {
'name':
'test-dataset-co',
'title':
u'Test dataset',
'notes':
u'Test description',
'license_id':
'uk-ogl',
'state':
'active',
'tags': [],
'resources': [{
'description': u'Some file - Download',
'format': 'CSV',
'resource_type': 'documentation',
'url': u'http://test.com/file.xls',
'schema-url':
'http://schemas.opendata.esd.org.uk/publictoilets/PublicToilets.xml?v=0.24',
'schema-type': 'csvlint',
}],
'extras': [{
'key': 'harvest_source_reference',
'value': 'testguid'
}, {
'key': 'inventory_identifier',
'value': 'payments_over_500'
}, {
'key': 'la_function',
'value': ''
}, {
'key': 'la_service',
'value': ''
}],
})
def test_harvest_basic(self):
# Create source
source_fixture = {
'title': 'Test Source',
'name': 'test-source',
'url': u'http://127.0.0.1:8999/esdInventory_live_truncated.xml',
'type': u'inventory',
}
source, job = self._create_source_and_job(source_fixture)
# Gather
harvester = InventoryHarvester()
# mock boundary stuff to avoid needing PostGIS - it is not tested here
# and that allows this test to run on sqlite
with patch('ckanext.dgulocal.harvester.get_boundary') as get_boundary:
get_boundary.return_value = None
object_ids = harvester.gather_stage(job)
assert_equal(len(object_ids), 3)
assert len(job.gather_errors) == 0
# Fetch
for object_id in object_ids:
harvest_object = HarvestObject.get(object_id)
assert harvest_object
success = harvester.fetch_stage(harvest_object)
assert_equal(success, True)
assert not harvest_object.errors
# Import
objects = []
for object_id in object_ids:
obj = HarvestObject.get(object_id)
assert obj
objects.append(obj)
harvester.import_stage(obj)
assert not harvest_object.errors
pkgs = Session.query(Package).filter(
Package.type != u'harvest_source').all()
assert_equal(len(pkgs), 3)
pkg_ids = [pkg.id for pkg in pkgs]
for obj in objects:
assert obj.current == True
assert obj.package_id in pkg_ids
def test_hard(self):
assert_equal(InventoryHarvester._get_publisher_abbreviation(MockObject(extras={}, title='Department for Environment, Food & Rural Affairs')), 'DEFRA')
def test_simple(self):
assert_equal(InventoryHarvester._get_publisher_abbreviation(MockObject(extras={}, title='Cabinet Office')), 'CO')
def test_simple(self):
assert_equal(
InventoryHarvester._get_publisher_abbreviation(
MockObject(extras={}, title='Cabinet Office')), 'CO')
