def setup_class(self):
self.tsi = TestSearchIndexer()
super(TestLoaderInsertingResources, self).setup_class()
self.loader = ResourceSeriesLoader(
self.testclient, ['title', 'department'],
'ons/id/',
field_keys_to_expect_invariant=['country'])
def setup_class(self):
model.repo.init_db()
q = model.Session.query(model.UserObjectRole).filter(model.UserObjectRole.role==Role.EDITOR)
q = q.filter(model.UserObjectRole.user==model.User.by_name(u"visitor"))
for role in q:
model.Session.delete(role)
model.repo.commit_and_remove()
indexer = TestSearchIndexer()
self._create_test_data()
model.Session.remove()
indexer.index()
def setup_class(cls):
search.clear()
cls.tsi = TestSearchIndexer()
DguCreateTestData.create_dgu_test_data()
DguCreateTestData.create_arbitrary({'name': 'latest',
'notes': '<b>Latest</b> dataset.',
'groups': ['national-health-service']})
cls.tsi.index()
def setup_class(self):
self.tsi = TestSearchIndexer()
super(TestLoaderInsertingResources, self).setup_class()
self.loader = ResourceSeriesLoader(
self.testclient,
['title', 'department'],
'ons/id/',
field_keys_to_expect_invariant=['country'])
def setup_class(self):
model.repo.init_db()
q = model.Session.query(model.UserObjectRole) \
.filter(sa.or_(model.UserObjectRole.role==model.Role.EDITOR,
model.UserObjectRole.role==model.Role.ANON_EDITOR)) \
.filter(model.UserObjectRole.user==model.User.by_name(u"visitor"))
for role in q:
model.Session.delete(role)
q = model.Session.query(model.RoleAction).filter(model.RoleAction.role==model.Role.READER)
for role_action in q:
model.Session.delete(role_action)
model.repo.commit_and_remove()
indexer = TestSearchIndexer()
TestUsage._create_test_data()
model.Session.remove()
indexer.index()
self.user_name = TestUsage.mrloggedin.name.encode('utf-8')
class TestLoaderInsertingResourcesWithSynonym(TestLoaderBase):
@classmethod
def setup_class(self):
self.tsi = TestSearchIndexer()
super(TestLoaderInsertingResourcesWithSynonym, self).setup_class()
self.loader = ResourceSeriesLoader(
self.testclient,
['title', 'department'],
'ons/id/',
field_keys_to_expect_invariant=['country'],
synonyms={'department': [('air', 'sky')]}
)
# teardown is in the base class
def test_0_search_options(self):
loader = ResourceSeriesLoader(
self.testclient,
['title', 'department'],
'ons/id/',
field_keys_to_expect_invariant=['country'],
synonyms={'department': [('dept1', 'dept2', 'dept3')],
'title': [('titleA', 'titleB', 'titleC')]}
)
field_keys = ['title', 'department']
pkg_dict = {'title':'titleA',
'extras':{'department':'dept1'}}
opts = loader._get_search_options(field_keys, pkg_dict)
self.assert_equal(opts, [{'department': 'dept1', 'title': 'titleA'}, {'department': 'dept2', 'title': 'titleA'}, {'department': 'dept3', 'title': 'titleA'}, {'department': 'dept1', 'title': 'titleB'}, {'department': 'dept1', 'title': 'titleC'}, {'department': 'dept2', 'title': 'titleB'}, {'department': 'dept2', 'title': 'titleC'}, {'department': 'dept3', 'title': 'titleB'}, {'department': 'dept3', 'title': 'titleC'}])
def test_1_reload(self):
# create initial package
num_pkgs = count_pkgs()
pkg_dict = {'name':u'pollution',
'title':u'Pollution',
'extras':{u'department':'air',
u'country':'UK', #invariant
u'last_updated':'Monday', #variant
},
'resources':[{'url':'pollution.com/1',
'description':'ons/id/1'}],
}
bogus_dict = {'name':u'bogus',
'title':u'Pollution',
'extras':{u'department':'water',
u'country':'UK',
u'last_updated':'Monday',
},
'resources':[{'url':'pollution.com/2',
'description':'ons/id/2'}],
}
assert not model.Package.by_name(pkg_dict['name'])
assert not model.Package.by_name(bogus_dict['name'])
CreateTestData.create_arbitrary([pkg_dict, bogus_dict])
self.tsi.index()
pkg = model.Package.by_name(pkg_dict['name'])
assert pkg
assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs)
assert len(pkg.resources) == 1, pkg.resources
# load the similar package: same title, updated resource,
# BUT synonym department
pkg_dict = {'name':u'pollution',
'title':u'Pollution',
'extras':{u'department':'sky',
u'country':'UK', #invariant
u'last_updated':'Tuesday', #variant
},
'resources':[{'url':'pollution.com/id/1',
'description':'ons/id/1'}],
}
self.loader.load_package(pkg_dict)
pkg = model.Package.by_name(pkg_dict['name'])
assert pkg
assert pkg.name == pkg_dict['name']
assert pkg.title == pkg_dict['title']
assert pkg.extras['country'] == pkg_dict['extras']['country']
assert pkg.extras['last_updated'] == pkg_dict['extras']['last_updated']
assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs)
assert len(pkg.resources) == 1, pkg.resources
assert pkg.resources[0].url == pkg_dict['resources'][0]['url'], pkg.resources[0].url
assert pkg.resources[0].description == pkg_dict['resources'][0]['description'], pkg.resources[0]['description']
# load the different package: because of different department
pkg_dict3 = {'name':u'pollution',
'title':u'Pollution',
'extras':{u'department':'river',
u'country':'UK', #invariant
u'last_updated':'Tuesday', #variant
},
'resources':[{'url':'pollution.com/id/3',
'description':'Lots of pollution | ons/id/3'}],
}
self.loader.load_package(pkg_dict3)
CreateTestData.flag_for_deletion('pollution_')
assert count_pkgs() == num_pkgs + 3, (count_pkgs() - num_pkgs)
pkg_names = [pkg.name for pkg in model.Session.query(model.Package).all()]
pkg = model.Package.by_name(u'pollution_')
assert pkg
assert pkg.extras['department'] == pkg_dict3['extras']['department']
class TestLoaderInsertingResources(TestLoaderBase):
@classmethod
def setup_class(self):
self.tsi = TestSearchIndexer()
super(TestLoaderInsertingResources, self).setup_class()
self.loader = ResourceSeriesLoader(
self.testclient,
['title', 'department'],
'ons/id/',
field_keys_to_expect_invariant=['country'])
# teardown is in the base class
def test_0_reload(self):
# create initial package
num_pkgs = count_pkgs()
pkg_dict = {'name':u'pollution',
'title':u'Pollution',
'extras':{u'department':'air',
u'country':'UK', #invariant
u'last_updated':'Monday', #variant
},
'resources':[{'url':'pollution.com/1',
'description':'ons/id/1'}],
}
bogus_dict = {'name':u'bogus',
'title':u'Pollution',
'extras':{u'department':'water',
u'country':'UK',
u'last_updated':'Monday',
},
'resources':[{'url':'pollution.com/2',
'description':'ons/id/2'}],
}
assert not model.Package.by_name(pkg_dict['name'])
assert not model.Package.by_name(bogus_dict['name'])
CreateTestData.create_arbitrary([pkg_dict, bogus_dict])
self.tsi.index()
pkg = model.Package.by_name(pkg_dict['name'])
assert pkg
assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs)
assert len(pkg.resources) == 1, pkg.resources
# load the same package: same title, department, updated resource
pkg_dict = {'name':u'pollution',
'title':u'Pollution',
'extras':{u'department':'air',
u'country':'UK', #invariant
u'last_updated':'Tuesday', #variant
},
'resources':[{'url':'pollution.com/id/1',
'description':'ons/id/1'}],
}
self.loader.load_package(pkg_dict)
pkg = model.Package.by_name(pkg_dict['name'])
assert pkg
assert pkg.name == pkg_dict['name']
assert pkg.title == pkg_dict['title']
assert pkg.extras['country'] == pkg_dict['extras']['country']
assert pkg.extras['last_updated'] == pkg_dict['extras']['last_updated']
assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs)
assert len(pkg.resources) == 1, pkg.resources
assert pkg.resources[0].url == pkg_dict['resources'][0]['url'], pkg.resources[0].url
assert pkg.resources[0].description == pkg_dict['resources'][0]['description'], pkg.resources[0]['description']
# load the same package: same title, department, new resource
pkg_dict2 = {'name':u'pollution',
'title':u'Pollution',
'extras':{u'department':'air',
u'country':'UK', #invariant
u'last_updated':'Tuesday', #variant
},
'resources':[{'url':'pollution.com/id/3',
'description':'ons/id/3'}],
}
self.loader.load_package(pkg_dict2)
pkg = model.Package.by_name(pkg_dict2['name'])
assert pkg
assert pkg.name == pkg_dict2['name']
assert pkg.title == pkg_dict2['title']
assert pkg.extras['country'] == pkg_dict2['extras']['country']
assert pkg.extras['last_updated'] == pkg_dict2['extras']['last_updated']
assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs)
assert len(pkg.resources) == 2, pkg.resources
print pkg.resources
assert_equal(pkg.resources[0].url, pkg_dict['resources'][0]['url'])
assert pkg.resources[0].description == pkg_dict['resources'][0]['description'], pkg.resources[0]['description']
assert pkg.resources[1].url == pkg_dict2['resources'][0]['url'], pkg.resources[1].url
assert pkg.resources[1].description == pkg_dict2['resources'][0]['description'], pkg.resources[1]['description']
# load the different package: because of different department
pkg_dict3 = {'name':u'pollution',
'title':u'Pollution',
'extras':{u'department':'river',
u'country':'UK', #invariant
u'last_updated':'Tuesday', #variant
},
'resources':[{'url':'pollution.com/id/3',
'description':'Lots of pollution | ons/id/3'}],
}
self.loader.load_package(pkg_dict3)
CreateTestData.flag_for_deletion('pollution_')
assert count_pkgs() == num_pkgs + 3, (count_pkgs() - num_pkgs)
pkg_names = [pkg.name for pkg in model.Session.query(model.Package).all()]
pkg = model.Package.by_name(u'pollution_')
assert pkg
assert pkg.extras['department'] == pkg_dict3['extras']['department']
# load the same package: but with different country
# should just get a warning
pkg_dict4 = {'name':u'pollution',
'title':u'Pollution',
'extras':OrderedDict([
(u'department', 'air'),
(u'country', 'UK and France'), #invariant
(u'last_updated', 'Tuesday'), #variant
]),
'resources':[OrderedDict([
('url', 'pollution.com/id/3'),
('description', 'Lots of pollution | ons/id/3'),
])],
}
self.loader.load_package(pkg_dict4)
pkg = model.Package.by_name(pkg_dict4['name'])
assert pkg
assert pkg.name == pkg_dict4['name']
assert pkg.title == pkg_dict4['title']
assert pkg.extras['country'] == pkg_dict4['extras']['country']
assert pkg.extras['last_updated'] == pkg_dict4['extras']['last_updated']
assert count_pkgs() == num_pkgs + 3, (count_pkgs() - num_pkgs)
assert len(pkg.resources) == 2, pkg.resources
assert pkg.resources[0].url == pkg_dict['resources'][0]['url'], pkg.resources[0].url
assert pkg.resources[0].description == pkg_dict['resources'][0]['description'], pkg.resources[0]['description']
assert pkg.resources[1].url == pkg_dict4['resources'][0]['url'], pkg.resources[1].url
assert pkg.resources[1].description == pkg_dict4['resources'][0]['description'], pkg.resources[1]['description']
def setup_class(self):
self.tsi = TestSearchIndexer()
super(TestLoaderUsingUniqueFields, self).setup_class()
self.loader = ReplaceByExtraFieldLoader(self.testclient, 'ref')
class TestLoaderUsingUniqueFields(TestLoaderBase):
@classmethod
def setup_class(self):
self.tsi = TestSearchIndexer()
super(TestLoaderUsingUniqueFields, self).setup_class()
self.loader = ReplaceByExtraFieldLoader(self.testclient, 'ref')
# teardown is in the base class
def test_0_reload(self):
# create initial package
num_pkgs = count_pkgs()
pkg_dict = {'name':u'pkgname0',
'title':u'Boris',
'extras':{u'ref':'boris'}}
assert not model.Package.by_name(pkg_dict['name'])
CreateTestData.create_arbitrary([pkg_dict])
self.tsi.index()
pkg = model.Package.by_name(pkg_dict['name'])
assert pkg
assert count_pkgs() == num_pkgs + 1, (count_pkgs() - num_pkgs)
# load the package with same name and ref
pkg_dict = {'name':u'pkgname0',
'title':u'Boris 2',
'extras':{u'ref':'boris'}}
self.loader.load_package(pkg_dict)
pkg = model.Package.by_name(pkg_dict['name'])
assert pkg
assert pkg.name == pkg_dict['name']
assert pkg.title == pkg_dict['title']
assert count_pkgs() == num_pkgs + 1, (count_pkgs() - num_pkgs)
# load the package with different name, same ref
pkg_dict = {'name':u'pkgname0changed',
'title':u'Boris 3',
'extras':{u'ref':'boris'}}
CreateTestData.flag_for_deletion(pkg_names=[pkg_dict['name']])
self.loader.load_package(pkg_dict)
assert count_pkgs() == num_pkgs + 1, (count_pkgs() - num_pkgs)
# for now we do not support renaming
pkg = model.Package.by_name(pkg_dict['name'])
assert pkg is None, pkg
pkg = model.Package.by_name(u'pkgname0')
assert pkg
assert pkg.title == pkg_dict['title']
# load the package with same name, different ref - new package
other_pkg_dict = pkg_dict
pkg_dict = {'name':u'pkgname0',
'title':u'Boris 4',
'extras':{u'ref':'boris-4'}}
CreateTestData.flag_for_deletion(pkg_names=[pkg_dict['name']])
self.loader.load_package(pkg_dict)
assert pkg_dict['name'] == 'pkgname0_'
orig_pkg = model.Package.by_name(u'pkgname0')
assert orig_pkg
assert orig_pkg.title == u'Boris 3'
pkg = model.Package.by_name(pkg_dict['name'])
assert pkg
assert pkg.name == pkg_dict['name']
assert pkg.title == pkg_dict['title']
assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs)
def test_1_avoid_long_name_clash(self):
# load the package once
num_pkgs = count_pkgs()
pkg_dict = {'name':u'a'*99,
'title':u'99 char name',
'extras':{u'ref':'aaa'}}
assert not model.Package.by_name(pkg_dict['name'])
CreateTestData.flag_for_deletion(pkg_names=[pkg_dict['name']])
self.loader.load_package(pkg_dict)
pkg = model.Package.by_name(pkg_dict['name'])
assert pkg
assert count_pkgs() == num_pkgs + 1, (count_pkgs() - num_pkgs)
# load a clashing package - name appended '_'
orig_pkg = pkg_dict
pkg_dict = {'name':orig_pkg['name'],
'title':u'bbb',
'extras':{u'ref':'bbb'}}
self.loader.load_package(pkg_dict)
clash_name = u'a'*99 + u'_'
pkg = model.Package.by_name(clash_name)
assert pkg
assert pkg.title == pkg_dict['title'], pkg.title
assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs)
# load another clashing package - name over 100 chars so shortened
# and finishes '__'
orig_pkg = pkg_dict
pkg_dict = {'name':orig_pkg['name'],
'title':u'ccc',
'extras':{u'ref':'ccc'}}
self.loader.load_package(pkg_dict)
clash_name = u'a'*98 + u'__'
assert pkg_dict['name'] == clash_name, (pkg_dict['name'], clash_name)
pkg = model.Package.by_name(clash_name)
assert pkg
assert pkg.title == pkg_dict['title'], pkg.title
assert count_pkgs() == num_pkgs + 3, (count_pkgs() - num_pkgs)
class TestLoaderInsertingResourcesWithSynonym(TestLoaderBase):
@classmethod
def setup_class(self):
self.tsi = TestSearchIndexer()
super(TestLoaderInsertingResourcesWithSynonym, self).setup_class()
self.loader = ResourceSeriesLoader(
self.testclient, ['title', 'department'],
'ons/id/',
field_keys_to_expect_invariant=['country'],
synonyms={'department': [('air', 'sky')]})
# teardown is in the base class
def test_0_search_options(self):
loader = ResourceSeriesLoader(
self.testclient, ['title', 'department'],
'ons/id/',
field_keys_to_expect_invariant=['country'],
synonyms={
'department': [('dept1', 'dept2', 'dept3')],
'title': [('titleA', 'titleB', 'titleC')]
})
field_keys = ['title', 'department']
pkg_dict = {'title': 'titleA', 'extras': {'department': 'dept1'}}
opts = loader._get_search_options(field_keys, pkg_dict)
self.assert_equal(opts, [{
'department': 'dept1',
'title': 'titleA'
}, {
'department': 'dept2',
'title': 'titleA'
}, {
'department': 'dept3',
'title': 'titleA'
}, {
'department': 'dept1',
'title': 'titleB'
}, {
'department': 'dept1',
'title': 'titleC'
}, {
'department': 'dept2',
'title': 'titleB'
}, {
'department': 'dept2',
'title': 'titleC'
}, {
'department': 'dept3',
'title': 'titleB'
}, {
'department': 'dept3',
'title': 'titleC'
}])
def test_1_reload(self):
# create initial package
num_pkgs = count_pkgs()
pkg_dict = {
'name': u'pollution',
'title': u'Pollution',
'extras': {
u'department': 'air',
u'country': 'UK', #invariant
u'last_updated': 'Monday', #variant
},
'resources': [{
'url': 'pollution.com/1',
'description': 'ons/id/1'
}],
}
bogus_dict = {
'name': u'bogus',
'title': u'Pollution',
'extras': {
u'department': 'water',
u'country': 'UK',
u'last_updated': 'Monday',
},
'resources': [{
'url': 'pollution.com/2',
'description': 'ons/id/2'
}],
}
assert not model.Package.by_name(pkg_dict['name'])
assert not model.Package.by_name(bogus_dict['name'])
CreateTestData.create_arbitrary([pkg_dict, bogus_dict])
self.tsi.index()
pkg = model.Package.by_name(pkg_dict['name'])
assert pkg
assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs)
assert len(pkg.resources) == 1, pkg.resources
# load the similar package: same title, updated resource,
# BUT synonym department
pkg_dict = {
'name':
u'pollution',
'title':
u'Pollution',
'extras': {
u'department': 'sky',
u'country': 'UK', #invariant
u'last_updated': 'Tuesday', #variant
},
'resources': [{
'url': 'pollution.com/id/1',
'description': 'ons/id/1'
}],
}
self.loader.load_package(pkg_dict)
pkg = model.Package.by_name(pkg_dict['name'])
assert pkg
assert pkg.name == pkg_dict['name']
assert pkg.title == pkg_dict['title']
assert pkg.extras['country'] == pkg_dict['extras']['country']
assert pkg.extras['last_updated'] == pkg_dict['extras']['last_updated']
assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs)
assert len(pkg.resources) == 1, pkg.resources
assert pkg.resources[0].url == pkg_dict['resources'][0][
'url'], pkg.resources[0].url
assert pkg.resources[0].description == pkg_dict['resources'][0][
'description'], pkg.resources[0]['description']
# load the different package: because of different department
pkg_dict3 = {
'name':
u'pollution',
'title':
u'Pollution',
'extras': {
u'department': 'river',
u'country': 'UK', #invariant
u'last_updated': 'Tuesday', #variant
},
'resources': [{
'url': 'pollution.com/id/3',
'description': 'Lots of pollution | ons/id/3'
}],
}
self.loader.load_package(pkg_dict3)
CreateTestData.flag_for_deletion('pollution_')
assert count_pkgs() == num_pkgs + 3, (count_pkgs() - num_pkgs)
pkg_names = [
pkg.name for pkg in model.Session.query(model.Package).all()
]
pkg = model.Package.by_name(u'pollution_')
assert pkg
assert pkg.extras['department'] == pkg_dict3['extras']['department']
class TestLoaderInsertingResources(TestLoaderBase):
@classmethod
def setup_class(self):
self.tsi = TestSearchIndexer()
super(TestLoaderInsertingResources, self).setup_class()
self.loader = ResourceSeriesLoader(
self.testclient, ['title', 'department'],
'ons/id/',
field_keys_to_expect_invariant=['country'])
# teardown is in the base class
def test_0_reload(self):
# create initial package
num_pkgs = count_pkgs()
pkg_dict = {
'name': u'pollution',
'title': u'Pollution',
'extras': {
u'department': 'air',
u'country': 'UK', #invariant
u'last_updated': 'Monday', #variant
},
'resources': [{
'url': 'pollution.com/1',
'description': 'ons/id/1'
}],
}
bogus_dict = {
'name': u'bogus',
'title': u'Pollution',
'extras': {
u'department': 'water',
u'country': 'UK',
u'last_updated': 'Monday',
},
'resources': [{
'url': 'pollution.com/2',
'description': 'ons/id/2'
}],
}
assert not model.Package.by_name(pkg_dict['name'])
assert not model.Package.by_name(bogus_dict['name'])
CreateTestData.create_arbitrary([pkg_dict, bogus_dict])
self.tsi.index()
pkg = model.Package.by_name(pkg_dict['name'])
assert pkg
assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs)
assert len(pkg.resources) == 1, pkg.resources
# load the same package: same title, department, updated resource
pkg_dict = {
'name':
u'pollution',
'title':
u'Pollution',
'extras': {
u'department': 'air',
u'country': 'UK', #invariant
u'last_updated': 'Tuesday', #variant
},
'resources': [{
'url': 'pollution.com/id/1',
'description': 'ons/id/1'
}],
}
self.loader.load_package(pkg_dict)
pkg = model.Package.by_name(pkg_dict['name'])
assert pkg
assert pkg.name == pkg_dict['name']
assert pkg.title == pkg_dict['title']
assert pkg.extras['country'] == pkg_dict['extras']['country']
assert pkg.extras['last_updated'] == pkg_dict['extras']['last_updated']
assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs)
assert len(pkg.resources) == 1, pkg.resources
assert pkg.resources[0].url == pkg_dict['resources'][0][
'url'], pkg.resources[0].url
assert pkg.resources[0].description == pkg_dict['resources'][0][
'description'], pkg.resources[0]['description']
# load the same package: same title, department, new resource
pkg_dict2 = {
'name':
u'pollution',
'title':
u'Pollution',
'extras': {
u'department': 'air',
u'country': 'UK', #invariant
u'last_updated': 'Tuesday', #variant
},
'resources': [{
'url': 'pollution.com/id/3',
'description': 'ons/id/3'
}],
}
self.loader.load_package(pkg_dict2)
pkg = model.Package.by_name(pkg_dict2['name'])
assert pkg
assert pkg.name == pkg_dict2['name']
assert pkg.title == pkg_dict2['title']
assert pkg.extras['country'] == pkg_dict2['extras']['country']
assert pkg.extras['last_updated'] == pkg_dict2['extras'][
'last_updated']
assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs)
assert len(pkg.resources) == 2, pkg.resources
print pkg.resources
assert_equal(pkg.resources[0].url, pkg_dict['resources'][0]['url'])
assert pkg.resources[0].description == pkg_dict['resources'][0][
'description'], pkg.resources[0]['description']
assert pkg.resources[1].url == pkg_dict2['resources'][0][
'url'], pkg.resources[1].url
assert pkg.resources[1].description == pkg_dict2['resources'][0][
'description'], pkg.resources[1]['description']
# load the different package: because of different department
pkg_dict3 = {
'name':
u'pollution',
'title':
u'Pollution',
'extras': {
u'department': 'river',
u'country': 'UK', #invariant
u'last_updated': 'Tuesday', #variant
},
'resources': [{
'url': 'pollution.com/id/3',
'description': 'Lots of pollution | ons/id/3'
}],
}
self.loader.load_package(pkg_dict3)
CreateTestData.flag_for_deletion('pollution_')
assert count_pkgs() == num_pkgs + 3, (count_pkgs() - num_pkgs)
pkg_names = [
pkg.name for pkg in model.Session.query(model.Package).all()
]
pkg = model.Package.by_name(u'pollution_')
assert pkg
assert pkg.extras['department'] == pkg_dict3['extras']['department']
# load the same package: but with different country
# should just get a warning
pkg_dict4 = {
'name':
u'pollution',
'title':
u'Pollution',
'extras':
OrderedDict([
(u'department', 'air'),
(u'country', 'UK and France'), #invariant
(u'last_updated', 'Tuesday'), #variant
]),
'resources': [
OrderedDict([
('url', 'pollution.com/id/3'),
('description', 'Lots of pollution | ons/id/3'),
])
],
}
self.loader.load_package(pkg_dict4)
pkg = model.Package.by_name(pkg_dict4['name'])
assert pkg
assert pkg.name == pkg_dict4['name']
assert pkg.title == pkg_dict4['title']
assert pkg.extras['country'] == pkg_dict4['extras']['country']
assert pkg.extras['last_updated'] == pkg_dict4['extras'][
'last_updated']
assert count_pkgs() == num_pkgs + 3, (count_pkgs() - num_pkgs)
assert len(pkg.resources) == 2, pkg.resources
assert pkg.resources[0].url == pkg_dict['resources'][0][
'url'], pkg.resources[0].url
assert pkg.resources[0].description == pkg_dict['resources'][0][
'description'], pkg.resources[0]['description']
assert pkg.resources[1].url == pkg_dict4['resources'][0][
'url'], pkg.resources[1].url
assert pkg.resources[1].description == pkg_dict4['resources'][0][
'description'], pkg.resources[1]['description']
def setup_class(self):
self.tsi = TestSearchIndexer()
super(TestLoaderUsingUniqueFields, self).setup_class()
self.loader = ReplaceByExtraFieldLoader(self.testclient, 'ref')
class TestLoaderUsingUniqueFields(TestLoaderBase):
@classmethod
def setup_class(self):
self.tsi = TestSearchIndexer()
super(TestLoaderUsingUniqueFields, self).setup_class()
self.loader = ReplaceByExtraFieldLoader(self.testclient, 'ref')
# teardown is in the base class
def test_0_reload(self):
# create initial package
num_pkgs = count_pkgs()
pkg_dict = {
'name': u'pkgname0',
'title': u'Boris',
'extras': {
u'ref': 'boris'
}
}
assert not model.Package.by_name(pkg_dict['name'])
CreateTestData.create_arbitrary([pkg_dict])
self.tsi.index()
pkg = model.Package.by_name(pkg_dict['name'])
assert pkg
assert count_pkgs() == num_pkgs + 1, (count_pkgs() - num_pkgs)
# load the package with same name and ref
pkg_dict = {
'name': u'pkgname0',
'title': u'Boris 2',
'extras': {
u'ref': 'boris'
}
}
self.loader.load_package(pkg_dict)
pkg = model.Package.by_name(pkg_dict['name'])
assert pkg
assert pkg.name == pkg_dict['name']
assert pkg.title == pkg_dict['title']
assert count_pkgs() == num_pkgs + 1, (count_pkgs() - num_pkgs)
# load the package with different name, same ref
pkg_dict = {
'name': u'pkgname0changed',
'title': u'Boris 3',
'extras': {
u'ref': 'boris'
}
}
CreateTestData.flag_for_deletion(pkg_names=[pkg_dict['name']])
self.loader.load_package(pkg_dict)
assert count_pkgs() == num_pkgs + 1, (count_pkgs() - num_pkgs)
# for now we do not support renaming
pkg = model.Package.by_name(pkg_dict['name'])
assert pkg is None, pkg
pkg = model.Package.by_name(u'pkgname0')
assert pkg
assert pkg.title == pkg_dict['title']
# load the package with same name, different ref - new package
other_pkg_dict = pkg_dict
pkg_dict = {
'name': u'pkgname0',
'title': u'Boris 4',
'extras': {
u'ref': 'boris-4'
}
}
CreateTestData.flag_for_deletion(pkg_names=[pkg_dict['name']])
self.loader.load_package(pkg_dict)
assert pkg_dict['name'] == 'pkgname0_'
orig_pkg = model.Package.by_name(u'pkgname0')
assert orig_pkg
assert orig_pkg.title == u'Boris 3'
pkg = model.Package.by_name(pkg_dict['name'])
assert pkg
assert pkg.name == pkg_dict['name']
assert pkg.title == pkg_dict['title']
assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs)
def test_1_avoid_long_name_clash(self):
# load the package once
num_pkgs = count_pkgs()
pkg_dict = {
'name': u'a' * 99,
'title': u'99 char name',
'extras': {
u'ref': 'aaa'
}
}
assert not model.Package.by_name(pkg_dict['name'])
CreateTestData.flag_for_deletion(pkg_names=[pkg_dict['name']])
self.loader.load_package(pkg_dict)
pkg = model.Package.by_name(pkg_dict['name'])
assert pkg
assert count_pkgs() == num_pkgs + 1, (count_pkgs() - num_pkgs)
# load a clashing package - name appended '_'
orig_pkg = pkg_dict
pkg_dict = {
'name': orig_pkg['name'],
'title': u'bbb',
'extras': {
u'ref': 'bbb'
}
}
self.loader.load_package(pkg_dict)
clash_name = u'a' * 99 + u'_'
pkg = model.Package.by_name(clash_name)
assert pkg
assert pkg.title == pkg_dict['title'], pkg.title
assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs)
# load another clashing package - name over 100 chars so shortened
# and finishes '__'
orig_pkg = pkg_dict
pkg_dict = {
'name': orig_pkg['name'],
'title': u'ccc',
'extras': {
u'ref': 'ccc'
}
}
self.loader.load_package(pkg_dict)
clash_name = u'a' * 98 + u'__'
assert pkg_dict['name'] == clash_name, (pkg_dict['name'], clash_name)
pkg = model.Package.by_name(clash_name)
assert pkg
assert pkg.title == pkg_dict['title'], pkg.title
assert count_pkgs() == num_pkgs + 3, (count_pkgs() - num_pkgs)
def setup_class(self):
super(TestDeletedDecoyWhenAdmin, self).setup_class()
self.tsi = TestSearchIndexer()
self.orig_pkg_dict = {
"name":
u"quarterly_epidemiological_commentary",
"title":
"Quarterly Epidemiological Commentary",
"version":
None,
"url":
None,
"author":
None,
"author_email":
None,
"maintainer":
None,
"maintainer_email":
None,
"notes":
"Epidemiological analyses of Mandatory surveillance data on MRSA bacteraemia and C. difficile infection covering at least nine quarters\r\nSource agency: Health Protection Agency\r\nDesignation: Official Statistics not designated as National Statistics\r\nLanguage: English\r\nAlternative title: Quarterly Epi Commentary",
"license_id":
"uk-ogl",
"tags": [
"conditions-and-diseases", "health", "health-and-social-care",
"health-of-the-population",
"nhs-trust-hcai-pct-mrsa-mrsa-bacteraemia-c-difficile-c-diff-clostridium-difficile-healthcare-associa",
"well-being-and-care"
],
"groups": [],
"extras": {
"geographic_coverage": "100000: England",
"geographical_granularity": "Other",
"external_reference": "ONSHUB",
"temporal_coverage-from": "",
"temporal_granularity": "",
"date_updated": "",
"precision": "",
"geographic_granularity": "",
"temporal_coverage_to": "",
"temporal_coverage_from": "",
"taxonomy_url": "",
"import_source": "ONS-ons_data_7_days_to_2010-06-23",
"date_released": "2010-06-18",
"temporal_coverage-to": "",
"published_by": "Department of Health",
"published_via": "Health Protection Agency",
"update_frequency": "quarterly",
"national_statistic": "no",
"categories": "Health and Social Care"
},
"resources": []
}
self.deleted_decoy_pkg_dict = {
"name": u"quarterly_epidemiological_commentary_-_none",
"title": "Quarterly Epidemiological Commentary",
"extras": {
"published_via": "Health Protection Agency",
"published_by": "Department of Health",
},
}
CreateTestData.create_arbitrary([self.orig_pkg_dict])
CreateTestData.create_arbitrary([self.deleted_decoy_pkg_dict],
extra_user_names=[u'testsysadmin'])
# make a sysadmin user
rev = model.repo.new_revision()
testsysadmin = model.User.by_name(u'testsysadmin')
model.add_user_to_role(testsysadmin, model.Role.ADMIN, model.System())
# delete decoy
decoy_pkg = model.Package.by_name(self.deleted_decoy_pkg_dict['name'])
assert decoy_pkg
decoy_pkg.delete()
model.repo.commit_and_remove()
self.tsi.index()
# same data is imported, but should find record and add department
importer_ = importer.OnsImporter(sample_filepath(9))
self.pkg_dict = [pkg_dict for pkg_dict in importer_.pkg_dict()][0]
def setup_class(self):
indexer = TestSearchIndexer()
self._create_test_data()
model.Session.remove()
indexer.index()
def setup_class(self):
super(TestDeletedDecoyWhenAdmin, self).setup_class()
self.tsi = TestSearchIndexer()
self.orig_pkg_dict = {
"name": u"quarterly_epidemiological_commentary",
"title": "Quarterly Epidemiological Commentary",
"version": None, "url": None, "author": None, "author_email": None, "maintainer": None, "maintainer_email": None,
"notes": "Epidemiological analyses of Mandatory surveillance data on MRSA bacteraemia and C. difficile infection covering at least nine quarters\r\nSource agency: Health Protection Agency\r\nDesignation: Official Statistics not designated as National Statistics\r\nLanguage: English\r\nAlternative title: Quarterly Epi Commentary",
"license_id": "uk-ogl",
"tags": ["conditions-and-diseases", "health", "health-and-social-care", "health-of-the-population", "nhs-trust-hcai-pct-mrsa-mrsa-bacteraemia-c-difficile-c-diff-clostridium-difficile-healthcare-associa", "well-being-and-care"],
"groups": [],
"extras": {
"geographic_coverage": "100000: England",
"geographical_granularity": "Other",
"external_reference": "ONSHUB",
"temporal_coverage-from": "",
"temporal_granularity": "",
"date_updated": "",
"precision": "",
"geographic_granularity": "",
"temporal_coverage_to": "",
"temporal_coverage_from": "",
"taxonomy_url": "",
"import_source": "ONS-ons_data_7_days_to_2010-06-23",
"date_released": "2010-06-18",
"temporal_coverage-to": "",
"published_by": "Department of Health",
"published_via": "Health Protection Agency",
"update_frequency": "quarterly",
"national_statistic": "no",
"categories": "Health and Social Care"
},
"resources": []
}
self.deleted_decoy_pkg_dict = {
"name": u"quarterly_epidemiological_commentary_-_none",
"title": "Quarterly Epidemiological Commentary",
"extras": {
"published_via": "Health Protection Agency",
"published_by": "Department of Health",
},
}
CreateTestData.create_arbitrary([self.orig_pkg_dict])
CreateTestData.create_arbitrary([self.deleted_decoy_pkg_dict],
extra_user_names=[u'testsysadmin'])
# make a sysadmin user
rev = model.repo.new_revision()
testsysadmin = model.User.by_name(u'testsysadmin')
model.add_user_to_role(testsysadmin, model.Role.ADMIN, model.System())
# delete decoy
decoy_pkg = model.Package.by_name(self.deleted_decoy_pkg_dict['name'])
assert decoy_pkg
decoy_pkg.delete()
model.repo.commit_and_remove()
self.tsi.index()
# same data is imported, but should find record and add department
importer_ = importer.OnsImporter(sample_filepath(9))
self.pkg_dict = [pkg_dict for pkg_dict in importer_.pkg_dict()][0]
