class TestOnsLoadBasic(OnsLoaderBase): lots_of_publishers = True @classmethod def setup_class(self): super(TestOnsLoadBasic, self).setup_class() try: user = model.User.by_name(u'annafan') assert user test_ckan_client = WsgiCkanClient(self.app, api_key=user.apikey) importer_ = importer.OnsImporter(sample_filepath(''), test_ckan_client) self.pkg_dicts = [pkg_dict for pkg_dict in importer_.pkg_dict()] self.loader = OnsLoader(test_ckan_client) self.res = self.loader.load_packages(self.pkg_dicts) assert self.res['num_errors'] == 0, self.res except Exception: # ensure that mock_drupal is destroyed MockDrupalCase.teardown_class() model.repo.rebuild_db() raise def test_0_search_options(self): field_keys = ['title', 'groups'] pkg_dict = {'title':'titleA', 'groups':['Department for Children, Schools and Families']} opts = self.loader._get_search_options(field_keys, pkg_dict) assert_equal(opts, [{'groups': 'Department for Children, Schools and Families', 'title': 'titleA'}]) def test_1_hub_id_extraction(self): def assert_id(hub_id_value, expected_id): resource = {'description':'Some description', 'hub-id':hub_id_value} result = self.loader._get_hub_id(resource) assert_equal(result, expected_id) assert_id("119-46440", "119-46440") def test_2_date_choose(self): def assert_id(date1, date2, earlier_or_later, expected_date_index): dates = (date1, date2) pkg0 = {'extras':{'date': date1}} result = self.loader._choose_date(pkg0, date2, earlier_or_later, 'date') if not expected_date_index: assert_equal(result, expected_date_index) else: assert_equal(result, dates[expected_date_index - 1]) assert_id('2010-12-01', '2010-12-02', 'earlier', 1) assert_id('2010-12-01', '2010-12-02', 'later', 2) assert_id('2010-12-02', '2010-12-01', 'earlier', 2) assert_id('2010-12-02', '2010-12-01', 'later', 1) assert_id('', '2010-12-02', 'earlier', 2) assert_id('2010-12-01', '', 'later', 1) assert_id('', '', 'earlier', None) def test_fields(self): q = model.Session.query(model.Package) names = [pkg.name for pkg in q.all()] pkg1 = model.Package.by_name(u'uk_official_holdings_of_international_reserves') cereals = model.Package.by_name(u'cereals_and_oilseeds_production_harvest') custody = model.Package.by_name(u'end_of_custody_licence_release_and_recalls') probation = model.Package.by_name(u'probation_statistics_brief') assert pkg1, names assert cereals, names assert custody, names assert probation, names assert pkg1.title == 'UK Official Holdings of International Reserves', pkg1.title assert pkg1.notes.startswith("Monthly breakdown for government's net reserves, detailing gross reserves and gross liabilities."), pkg1.notes assert len(pkg1.resources) == 1, pkg1.resources assert pkg1.resources[0].url == 'http://www.hm-treasury.gov.uk/national_statistics.htm', pkg1.resources[0] assert_equal(pkg1.resources[0].description, 'December 2009') assert_equal(pkg1.resources[0].extras['hub-id'], '119-36345') assert len(custody.resources) == 2, custody.resources assert custody.resources[0].url == 'http://www.justice.gov.uk/publications/endofcustodylicence.htm', custody.resources[0] assert_equal(custody.resources[0].description, 'November 2009') assert_equal(custody.resources[0].extras['hub-id'], '119-36836') assert custody.resources[1].url == 'http://www.justice.gov.uk/publications/endofcustodylicence.htm', custody.resources[0] assert_equal(custody.resources[1].description, 'December 2009') assert_equal(custody.resources[1].extras['hub-id'], '119-36838') assert pkg1.extras['date_released'] == u'2010-01-06', pkg1.extras['date_released'] assert probation.extras['date_released'] == u'2010-01-04', probation.extras['date_released'] assert_equal(group_names(pkg1), [u"her-majestys-treasury"]) assert_equal(group_names(cereals), [u"department-for-environment-food-and-rural-affairs"]) assert_equal(group_names(custody), [u"department-of-justice"]) assert u"Source agency: HM Treasury" in pkg1.notes, pkg1.notes assert pkg1.extras['categories'] == 'Economy', pkg1.extras['category'] assert_equal(pkg1.extras['geographic_coverage'], '111100: United Kingdom (England, Scotland, Wales, Northern Ireland)') assert pkg1.extras['national_statistic'] == 'no', pkg1.extras['national_statistic'] assert cereals.extras['national_statistic'] == 'yes', cereals.extras['national_statistic'] assert custody.extras['national_statistic'] == 'no', custody.extras['national_statistic'] assert 'Designation: Official Statistics not designated as National Statistics' in custody.notes assert_equal(pkg1.extras['geographic_granularity'], 'UK and GB') assert 'Language: English' in pkg1.notes, pkg1.notes def check_tags(pkg, tags_list): pkg_tags = [tag.name for tag in pkg.get_tags()] for tag in tags_list: assert tag in pkg_tags, "Couldn't find tag '%s' in tags: %s" % (tag, pkg_tags) check_tags(pkg1, ('economics-and-finance', 'reserves', 'currency', 'assets', 'liabilities', 'gold', 'economy', 'government-receipts-and-expenditure', 'public-sector-finance')) check_tags(cereals, ('environment', 'farming')) check_tags(custody, ('public-order-justice-and-rights', 'justice-system', 'prisons')) assert 'Alternative title: UK Reserves' in pkg1.notes, pkg1.notes assert pkg1.extras['external_reference'] == u'ONSHUB', pkg1.extras['external_reference'] assert 'Open Government Licence' in pkg.license.title, pkg.license.title assert pkg1.extras['update_frequency'] == u'monthly', pkg1.extras['update_frequency'] assert custody.extras['update_frequency'] == u'monthly', custody.extras['update_frequency'] for pkg in (pkg1, cereals, custody): assert pkg.extras['import_source'].startswith('ONS'), '%s %s' % (pkg.name, pkg.extras['import_source'])