class TestOnsLoadBasic(OnsLoaderBase):
lots_of_publishers = True
@classmethod
def setup_class(self):
super(TestOnsLoadBasic, self).setup_class()
try:
user = model.User.by_name(u'annafan')
assert user
test_ckan_client = WsgiCkanClient(self.app, api_key=user.apikey)
importer_ = importer.OnsImporter(sample_filepath(''), test_ckan_client)
self.pkg_dicts = [pkg_dict for pkg_dict in importer_.pkg_dict()]
self.loader = OnsLoader(test_ckan_client)
self.res = self.loader.load_packages(self.pkg_dicts)
assert self.res['num_errors'] == 0, self.res
except Exception:
# ensure that mock_drupal is destroyed
MockDrupalCase.teardown_class()
model.repo.rebuild_db()
raise
def test_0_search_options(self):
field_keys = ['title', 'groups']
pkg_dict = {'title':'titleA',
'groups':['Department for Children, Schools and Families']}
opts = self.loader._get_search_options(field_keys, pkg_dict)
assert_equal(opts, [{'groups': 'Department for Children, Schools and Families', 'title': 'titleA'}])
def test_1_hub_id_extraction(self):
def assert_id(hub_id_value, expected_id):
resource = {'description':'Some description',
'hub-id':hub_id_value}
result = self.loader._get_hub_id(resource)
assert_equal(result, expected_id)
assert_id("119-46440",
"119-46440")
def test_2_date_choose(self):
def assert_id(date1, date2, earlier_or_later, expected_date_index):
dates = (date1, date2)
pkg0 = {'extras':{'date': date1}}
result = self.loader._choose_date(pkg0, date2,
earlier_or_later,
'date')
if not expected_date_index:
assert_equal(result, expected_date_index)
else:
assert_equal(result, dates[expected_date_index - 1])
assert_id('2010-12-01', '2010-12-02', 'earlier', 1)
assert_id('2010-12-01', '2010-12-02', 'later', 2)
assert_id('2010-12-02', '2010-12-01', 'earlier', 2)
assert_id('2010-12-02', '2010-12-01', 'later', 1)
assert_id('', '2010-12-02', 'earlier', 2)
assert_id('2010-12-01', '', 'later', 1)
assert_id('', '', 'earlier', None)
def test_fields(self):
q = model.Session.query(model.Package)
names = [pkg.name for pkg in q.all()]
pkg1 = model.Package.by_name(u'uk_official_holdings_of_international_reserves')
cereals = model.Package.by_name(u'cereals_and_oilseeds_production_harvest')
custody = model.Package.by_name(u'end_of_custody_licence_release_and_recalls')
probation = model.Package.by_name(u'probation_statistics_brief')
assert pkg1, names
assert cereals, names
assert custody, names
assert probation, names
assert pkg1.title == 'UK Official Holdings of International Reserves', pkg1.title
assert pkg1.notes.startswith("Monthly breakdown for government's net reserves, detailing gross reserves and gross liabilities."), pkg1.notes
assert len(pkg1.resources) == 1, pkg1.resources
assert pkg1.resources[0].url == 'http://www.hm-treasury.gov.uk/national_statistics.htm', pkg1.resources[0]
assert_equal(pkg1.resources[0].description, 'December 2009')
assert_equal(pkg1.resources[0].extras['hub-id'], '119-36345')
assert len(custody.resources) == 2, custody.resources
assert custody.resources[0].url == 'http://www.justice.gov.uk/publications/endofcustodylicence.htm', custody.resources[0]
assert_equal(custody.resources[0].description, 'November 2009')
assert_equal(custody.resources[0].extras['hub-id'], '119-36836')
assert custody.resources[1].url == 'http://www.justice.gov.uk/publications/endofcustodylicence.htm', custody.resources[0]
assert_equal(custody.resources[1].description, 'December 2009')
assert_equal(custody.resources[1].extras['hub-id'], '119-36838')
assert pkg1.extras['date_released'] == u'2010-01-06', pkg1.extras['date_released']
assert probation.extras['date_released'] == u'2010-01-04', probation.extras['date_released']
assert_equal(group_names(pkg1), [u"her-majestys-treasury"])
assert_equal(group_names(cereals), [u"department-for-environment-food-and-rural-affairs"])
assert_equal(group_names(custody), [u"department-of-justice"])
assert u"Source agency: HM Treasury" in pkg1.notes, pkg1.notes
assert pkg1.extras['categories'] == 'Economy', pkg1.extras['category']
assert_equal(pkg1.extras['geographic_coverage'], '111100: United Kingdom (England, Scotland, Wales, Northern Ireland)')
assert pkg1.extras['national_statistic'] == 'no', pkg1.extras['national_statistic']
assert cereals.extras['national_statistic'] == 'yes', cereals.extras['national_statistic']
assert custody.extras['national_statistic'] == 'no', custody.extras['national_statistic']
assert 'Designation: Official Statistics not designated as National Statistics' in custody.notes
assert_equal(pkg1.extras['geographic_granularity'], 'UK and GB')
assert 'Language: English' in pkg1.notes, pkg1.notes
def check_tags(pkg, tags_list):
pkg_tags = [tag.name for tag in pkg.get_tags()]
for tag in tags_list:
assert tag in pkg_tags, "Couldn't find tag '%s' in tags: %s" % (tag, pkg_tags)
check_tags(pkg1, ('economics-and-finance', 'reserves', 'currency', 'assets', 'liabilities', 'gold', 'economy', 'government-receipts-and-expenditure', 'public-sector-finance'))
check_tags(cereals, ('environment', 'farming'))
check_tags(custody, ('public-order-justice-and-rights', 'justice-system', 'prisons'))
assert 'Alternative title: UK Reserves' in pkg1.notes, pkg1.notes
assert pkg1.extras['external_reference'] == u'ONSHUB', pkg1.extras['external_reference']
assert 'Open Government Licence' in pkg.license.title, pkg.license.title
assert pkg1.extras['update_frequency'] == u'monthly', pkg1.extras['update_frequency']
assert custody.extras['update_frequency'] == u'monthly', custody.extras['update_frequency']
for pkg in (pkg1, cereals, custody):
assert pkg.extras['import_source'].startswith('ONS'), '%s %s' % (pkg.name, pkg.extras['import_source'])
class TestOnsLoadBasic(OnsLoaderBase):
lots_of_publishers = True
@classmethod
def setup_class(self):
super(TestOnsLoadBasic, self).setup_class()
try:
user = model.User.by_name(u'annafan')
assert user
test_ckan_client = WsgiCkanClient(self.app, api_key=user.apikey)
importer_ = importer.OnsImporter(sample_filepath(''), test_ckan_client)
self.pkg_dicts = [pkg_dict for pkg_dict in importer_.pkg_dict()]
self.loader = OnsLoader(test_ckan_client)
self.res = self.loader.load_packages(self.pkg_dicts)
assert self.res['num_errors'] == 0, self.res
except Exception:
# ensure that mock_drupal is destroyed
MockDrupalCase.teardown_class()
model.repo.rebuild_db()
raise
def test_0_search_options(self):
field_keys = ['title', 'groups']
pkg_dict = {'title':'titleA',
'groups':['Department for Children, Schools and Families']}
opts = self.loader._get_search_options(field_keys, pkg_dict)
assert_equal(opts, [{'groups': 'Department for Children, Schools and Families', 'title': 'titleA'}])
def test_1_hub_id_extraction(self):
def assert_id(hub_id_value, expected_id):
resource = {'description':'Some description',
'hub-id':hub_id_value}
result = self.loader._get_hub_id(resource)
assert_equal(result, expected_id)
assert_id("119-46440",
"119-46440")
def test_2_date_choose(self):
def assert_id(date1, date2, earlier_or_later, expected_date_index):
dates = (date1, date2)
pkg0 = {'extras':{'date': date1}}
result = self.loader._choose_date(pkg0, date2,
earlier_or_later,
'date')
if not expected_date_index:
assert_equal(result, expected_date_index)
else:
assert_equal(result, dates[expected_date_index - 1])
assert_id('2010-12-01', '2010-12-02', 'earlier', 1)
assert_id('2010-12-01', '2010-12-02', 'later', 2)
assert_id('2010-12-02', '2010-12-01', 'earlier', 2)
assert_id('2010-12-02', '2010-12-01', 'later', 1)
assert_id('', '2010-12-02', 'earlier', 2)
assert_id('2010-12-01', '', 'later', 1)
assert_id('', '', 'earlier', None)
def test_fields(self):
q = model.Session.query(model.Package)
names = [pkg.name for pkg in q.all()]
pkg1 = model.Package.by_name(u'uk_official_holdings_of_international_reserves')
cereals = model.Package.by_name(u'cereals_and_oilseeds_production_harvest')
custody = model.Package.by_name(u'end_of_custody_licence_release_and_recalls')
probation = model.Package.by_name(u'probation_statistics_brief')
assert pkg1, names
assert cereals, names
assert custody, names
assert probation, names
assert pkg1.title == 'UK Official Holdings of International Reserves', pkg1.title
assert pkg1.notes.startswith("Monthly breakdown for government's net reserves, detailing gross reserves and gross liabilities."), pkg1.notes
assert len(pkg1.resources) == 1, pkg1.resources
assert pkg1.resources[0].url == 'http://www.hm-treasury.gov.uk/national_statistics.htm', pkg1.resources[0]
assert_equal(pkg1.resources[0].description, 'December 2009')
assert_equal(pkg1.resources[0].extras['hub-id'], '119-36345')
assert len(custody.resources) == 2, custody.resources
assert custody.resources[0].url == 'http://www.justice.gov.uk/publications/endofcustodylicence.htm', custody.resources[0]
assert_equal(custody.resources[0].description, 'November 2009')
assert_equal(custody.resources[0].extras['hub-id'], '119-36836')
assert custody.resources[1].url == 'http://www.justice.gov.uk/publications/endofcustodylicence.htm', custody.resources[0]
assert_equal(custody.resources[1].description, 'December 2009')
assert_equal(custody.resources[1].extras['hub-id'], '119-36838')
assert pkg1.extras['date_released'] == u'2010-01-06', pkg1.extras['date_released']
assert probation.extras['date_released'] == u'2010-01-04', probation.extras['date_released']
assert_equal(group_names(pkg1), [u"her-majestys-treasury"])
assert_equal(group_names(cereals), [u"department-for-environment-food-and-rural-affairs"])
assert_equal(group_names(custody), [u"department-of-justice"])
assert u"Source agency: HM Treasury" in pkg1.notes, pkg1.notes
assert pkg1.extras['categories'] == 'Economy', pkg1.extras['category']
assert_equal(pkg1.extras['geographic_coverage'], '111100: United Kingdom (England, Scotland, Wales, Northern Ireland)')
assert pkg1.extras['national_statistic'] == 'no', pkg1.extras['national_statistic']
assert cereals.extras['national_statistic'] == 'yes', cereals.extras['national_statistic']
assert custody.extras['national_statistic'] == 'no', custody.extras['national_statistic']
assert 'Designation: Official Statistics not designated as National Statistics' in custody.notes
assert_equal(pkg1.extras['geographic_granularity'], 'UK and GB')
assert 'Language: English' in pkg1.notes, pkg1.notes
def check_tags(pkg, tags_list):
pkg_tags = [tag.name for tag in pkg.get_tags()]
for tag in tags_list:
assert tag in pkg_tags, "Couldn't find tag '%s' in tags: %s" % (tag, pkg_tags)
check_tags(pkg1, ('economics-and-finance', 'reserves', 'currency', 'assets', 'liabilities', 'gold', 'economy', 'government-receipts-and-expenditure', 'public-sector-finance'))
check_tags(cereals, ('environment', 'farming'))
check_tags(custody, ('public-order-justice-and-rights', 'justice-system', 'prisons'))
assert 'Alternative title: UK Reserves' in pkg1.notes, pkg1.notes
assert pkg1.extras['external_reference'] == u'ONSHUB', pkg1.extras['external_reference']
assert 'Open Government Licence' in pkg.license.title, pkg.license.title
assert pkg1.extras['update_frequency'] == u'monthly', pkg1.extras['update_frequency']
assert custody.extras['update_frequency'] == u'monthly', custody.extras['update_frequency']
for pkg in (pkg1, cereals, custody):
assert pkg.extras['import_source'].startswith('ONS'), '%s %s' % (pkg.name, pkg.extras['import_source'])
