예제 #1
0
 def test_get_package_dict(self):
     h = InventoryHarvester()
     harvest_object = self._get_test_harvest_object()
     package_dict_defaults = PackageDictDefaults()
     source_config = {}
     existing_pkg = MockObject(resources=[])
     pkg_dict = h.get_package_dict(harvest_object, package_dict_defaults,
                                   source_config, existing_pkg)
     pkg_dict['extras'] = sorted(pkg_dict['extras'])
     pprint(pkg_dict)
     assert_equal(pkg_dict, {
         'name': 'test-dataset-co',
         'title': u'Test dataset',
         'notes': u'Test description',
         'license_id': 'uk-ogl',
         'state': 'active',
         'tags': [],
         'resources': [{'description': u'Some file - Download',
                        'format': 'CSV',
                        'resource_type': 'documentation',
                        'url': u'http://test.com/file.xls',
                        'schema-url': 'http://schemas.opendata.esd.org.uk/publictoilets/PublicToilets.xml?v=0.24',
                        'schema-type': 'csvlint',
                        }],
         'extras': [{'key': 'harvest_source_reference', 'value': 'testguid'},
                    {'key': 'inventory_identifier', 'value': 'payments_over_500'},
                    {'key': 'la_function', 'value': ''},
                    {'key': 'la_service', 'value': ''}],
         })
    def test_harvest_basic(self):

        # Create source
        source_fixture = {
            'title': 'Test Source',
            'name': 'test-source',
            'url': u'http://127.0.0.1:8999/esdInventory_live_truncated.xml',
            'type': u'inventory',
        }
        source, job = self._create_source_and_job(source_fixture)

        # Gather
        harvester = InventoryHarvester()
        # mock boundary stuff to avoid needing PostGIS - it is not tested here
        # and that allows this test to run on sqlite
        with patch('ckanext.dgulocal.harvester.get_boundary') as get_boundary:
            get_boundary.return_value = None
            object_ids = harvester.gather_stage(job)

        assert_equal(len(object_ids), 3)
        assert len(job.gather_errors) == 0

        # Fetch
        for object_id in object_ids:
            harvest_object = HarvestObject.get(object_id)
            assert harvest_object
            success = harvester.fetch_stage(harvest_object)
            assert_equal(success, True)
            assert not harvest_object.errors

        # Import
        objects = []
        for object_id in object_ids:
            obj = HarvestObject.get(object_id)
            assert obj
            objects.append(obj)
            harvester.import_stage(obj)
            assert not harvest_object.errors

        pkgs = Session.query(Package).filter(Package.type!=u'harvest_source').all()

        assert_equal(len(pkgs), 3)

        pkg_ids = [pkg.id for pkg in pkgs]

        for obj in objects:
            assert obj.current == True
            assert obj.package_id in pkg_ids
예제 #3
0
 def test_hard(self):
     assert_equal(
         InventoryHarvester._get_publisher_abbreviation(
             MockObject(
                 extras={},
                 title='Department for Environment, Food & Rural Affairs')),
         'DEFRA')
 def test_get_package_dict(self):
     h = InventoryHarvester()
     harvest_object = self._get_test_harvest_object()
     package_dict_defaults = PackageDictDefaults()
     source_config = {}
     existing_pkg = MockObject(resources=[])
     pkg_dict = h.get_package_dict(harvest_object, package_dict_defaults,
                                   source_config, existing_pkg)
     pkg_dict['extras'] = sorted(pkg_dict['extras'])
     pprint(pkg_dict)
     assert_equal(
         pkg_dict, {
             'name':
             'test-dataset-co',
             'title':
             u'Test dataset',
             'notes':
             u'Test description',
             'license_id':
             'uk-ogl',
             'state':
             'active',
             'tags': [],
             'resources': [{
                 'description': u'Some file - Download',
                 'format': 'CSV',
                 'resource_type': 'documentation',
                 'url': u'http://test.com/file.xls',
                 'schema-url':
                 'http://schemas.opendata.esd.org.uk/publictoilets/PublicToilets.xml?v=0.24',
                 'schema-type': 'csvlint',
             }],
             'extras': [{
                 'key': 'harvest_source_reference',
                 'value': 'testguid'
             }, {
                 'key': 'inventory_identifier',
                 'value': 'payments_over_500'
             }, {
                 'key': 'la_function',
                 'value': ''
             }, {
                 'key': 'la_service',
                 'value': ''
             }],
         })
예제 #5
0
    def test_harvest_basic(self):

        # Create source
        source_fixture = {
            'title': 'Test Source',
            'name': 'test-source',
            'url': u'http://127.0.0.1:8999/esdInventory_live_truncated.xml',
            'type': u'inventory',
        }
        source, job = self._create_source_and_job(source_fixture)

        # Gather
        harvester = InventoryHarvester()
        # mock boundary stuff to avoid needing PostGIS - it is not tested here
        # and that allows this test to run on sqlite
        with patch('ckanext.dgulocal.harvester.get_boundary') as get_boundary:
            get_boundary.return_value = None
            object_ids = harvester.gather_stage(job)

        assert_equal(len(object_ids), 3)
        assert len(job.gather_errors) == 0

        # Fetch
        for object_id in object_ids:
            harvest_object = HarvestObject.get(object_id)
            assert harvest_object
            success = harvester.fetch_stage(harvest_object)
            assert_equal(success, True)
            assert not harvest_object.errors

        # Import
        objects = []
        for object_id in object_ids:
            obj = HarvestObject.get(object_id)
            assert obj
            objects.append(obj)
            harvester.import_stage(obj)
            assert not harvest_object.errors

        pkgs = Session.query(Package).filter(
            Package.type != u'harvest_source').all()

        assert_equal(len(pkgs), 3)

        pkg_ids = [pkg.id for pkg in pkgs]

        for obj in objects:
            assert obj.current == True
            assert obj.package_id in pkg_ids
 def test_hard(self):
     assert_equal(InventoryHarvester._get_publisher_abbreviation(MockObject(extras={}, title='Department for Environment, Food & Rural Affairs')), 'DEFRA')
 def test_simple(self):
     assert_equal(InventoryHarvester._get_publisher_abbreviation(MockObject(extras={}, title='Cabinet Office')), 'CO')
예제 #8
0
 def test_simple(self):
     assert_equal(
         InventoryHarvester._get_publisher_abbreviation(
             MockObject(extras={}, title='Cabinet Office')), 'CO')