def setup_class(self):
self.tsi = TestSearchIndexer()
super(TestLoaderInsertingResources, self).setup_class()
self.loader = ResourceSeriesLoader(
self.testclient, ['title', 'department'],
'ons/id/',
field_keys_to_expect_invariant=['country'])
def test_0_search_options(self):
loader = ResourceSeriesLoader(
self.testclient,
['title', 'department'],
'ons/id/',
field_keys_to_expect_invariant=['country'],
synonyms={'department': [('dept1', 'dept2', 'dept3')],
'title': [('titleA', 'titleB', 'titleC')]}
)
field_keys = ['title', 'department']
pkg_dict = {'title':'titleA',
'extras':{'department':'dept1'}}
opts = loader._get_search_options(field_keys, pkg_dict)
self.assert_equal(opts, [{'department': 'dept1', 'title': 'titleA'}, {'department': 'dept2', 'title': 'titleA'}, {'department': 'dept3', 'title': 'titleA'}, {'department': 'dept1', 'title': 'titleB'}, {'department': 'dept1', 'title': 'titleC'}, {'department': 'dept2', 'title': 'titleB'}, {'department': 'dept2', 'title': 'titleC'}, {'department': 'dept3', 'title': 'titleB'}, {'department': 'dept3', 'title': 'titleC'}])
def setup_class(self):
self.tsi = TestSearchIndexer()
super(TestLoaderInsertingResources, self).setup_class()
self.loader = ResourceSeriesLoader(
self.testclient,
['title', 'department'],
'ons/id/',
field_keys_to_expect_invariant=['country'])
def test_0_search_options(self):
loader = ResourceSeriesLoader(
self.testclient, ['title', 'department'],
'ons/id/',
field_keys_to_expect_invariant=['country'],
synonyms={
'department': [('dept1', 'dept2', 'dept3')],
'title': [('titleA', 'titleB', 'titleC')]
})
field_keys = ['title', 'department']
pkg_dict = {'title': 'titleA', 'extras': {'department': 'dept1'}}
opts = loader._get_search_options(field_keys, pkg_dict)
self.assert_equal(opts, [{
'department': 'dept1',
'title': 'titleA'
}, {
'department': 'dept2',
'title': 'titleA'
}, {
'department': 'dept3',
'title': 'titleA'
}, {
'department': 'dept1',
'title': 'titleB'
}, {
'department': 'dept1',
'title': 'titleC'
}, {
'department': 'dept2',
'title': 'titleB'
}, {
'department': 'dept2',
'title': 'titleC'
}, {
'department': 'dept3',
'title': 'titleB'
}, {
'department': 'dept3',
'title': 'titleC'
}])
class TestLoaderInsertingResourcesWithSynonym(TestLoaderBase):
@classmethod
def setup_class(self):
self.tsi = TestSearchIndexer()
super(TestLoaderInsertingResourcesWithSynonym, self).setup_class()
self.loader = ResourceSeriesLoader(
self.testclient,
['title', 'department'],
'ons/id/',
field_keys_to_expect_invariant=['country'],
synonyms={'department': [('air', 'sky')]}
)
# teardown is in the base class
def test_0_search_options(self):
loader = ResourceSeriesLoader(
self.testclient,
['title', 'department'],
'ons/id/',
field_keys_to_expect_invariant=['country'],
synonyms={'department': [('dept1', 'dept2', 'dept3')],
'title': [('titleA', 'titleB', 'titleC')]}
)
field_keys = ['title', 'department']
pkg_dict = {'title':'titleA',
'extras':{'department':'dept1'}}
opts = loader._get_search_options(field_keys, pkg_dict)
self.assert_equal(opts, [{'department': 'dept1', 'title': 'titleA'}, {'department': 'dept2', 'title': 'titleA'}, {'department': 'dept3', 'title': 'titleA'}, {'department': 'dept1', 'title': 'titleB'}, {'department': 'dept1', 'title': 'titleC'}, {'department': 'dept2', 'title': 'titleB'}, {'department': 'dept2', 'title': 'titleC'}, {'department': 'dept3', 'title': 'titleB'}, {'department': 'dept3', 'title': 'titleC'}])
def test_1_reload(self):
# create initial package
num_pkgs = count_pkgs()
pkg_dict = {'name':u'pollution',
'title':u'Pollution',
'extras':{u'department':'air',
u'country':'UK', #invariant
u'last_updated':'Monday', #variant
},
'resources':[{'url':'pollution.com/1',
'description':'ons/id/1'}],
}
bogus_dict = {'name':u'bogus',
'title':u'Pollution',
'extras':{u'department':'water',
u'country':'UK',
u'last_updated':'Monday',
},
'resources':[{'url':'pollution.com/2',
'description':'ons/id/2'}],
}
assert not model.Package.by_name(pkg_dict['name'])
assert not model.Package.by_name(bogus_dict['name'])
CreateTestData.create_arbitrary([pkg_dict, bogus_dict])
self.tsi.index()
pkg = model.Package.by_name(pkg_dict['name'])
assert pkg
assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs)
assert len(pkg.resources) == 1, pkg.resources
# load the similar package: same title, updated resource,
# BUT synonym department
pkg_dict = {'name':u'pollution',
'title':u'Pollution',
'extras':{u'department':'sky',
u'country':'UK', #invariant
u'last_updated':'Tuesday', #variant
},
'resources':[{'url':'pollution.com/id/1',
'description':'ons/id/1'}],
}
self.loader.load_package(pkg_dict)
pkg = model.Package.by_name(pkg_dict['name'])
assert pkg
assert pkg.name == pkg_dict['name']
assert pkg.title == pkg_dict['title']
assert pkg.extras['country'] == pkg_dict['extras']['country']
assert pkg.extras['last_updated'] == pkg_dict['extras']['last_updated']
assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs)
assert len(pkg.resources) == 1, pkg.resources
assert pkg.resources[0].url == pkg_dict['resources'][0]['url'], pkg.resources[0].url
assert pkg.resources[0].description == pkg_dict['resources'][0]['description'], pkg.resources[0]['description']
# load the different package: because of different department
pkg_dict3 = {'name':u'pollution',
'title':u'Pollution',
'extras':{u'department':'river',
u'country':'UK', #invariant
u'last_updated':'Tuesday', #variant
},
'resources':[{'url':'pollution.com/id/3',
'description':'Lots of pollution | ons/id/3'}],
}
self.loader.load_package(pkg_dict3)
CreateTestData.flag_for_deletion('pollution_')
assert count_pkgs() == num_pkgs + 3, (count_pkgs() - num_pkgs)
pkg_names = [pkg.name for pkg in model.Session.query(model.Package).all()]
pkg = model.Package.by_name(u'pollution_')
assert pkg
assert pkg.extras['department'] == pkg_dict3['extras']['department']
class TestLoaderInsertingResources(TestLoaderBase):
@classmethod
def setup_class(self):
self.tsi = TestSearchIndexer()
super(TestLoaderInsertingResources, self).setup_class()
self.loader = ResourceSeriesLoader(
self.testclient,
['title', 'department'],
'ons/id/',
field_keys_to_expect_invariant=['country'])
# teardown is in the base class
def test_0_reload(self):
# create initial package
num_pkgs = count_pkgs()
pkg_dict = {'name':u'pollution',
'title':u'Pollution',
'extras':{u'department':'air',
u'country':'UK', #invariant
u'last_updated':'Monday', #variant
},
'resources':[{'url':'pollution.com/1',
'description':'ons/id/1'}],
}
bogus_dict = {'name':u'bogus',
'title':u'Pollution',
'extras':{u'department':'water',
u'country':'UK',
u'last_updated':'Monday',
},
'resources':[{'url':'pollution.com/2',
'description':'ons/id/2'}],
}
assert not model.Package.by_name(pkg_dict['name'])
assert not model.Package.by_name(bogus_dict['name'])
CreateTestData.create_arbitrary([pkg_dict, bogus_dict])
self.tsi.index()
pkg = model.Package.by_name(pkg_dict['name'])
assert pkg
assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs)
assert len(pkg.resources) == 1, pkg.resources
# load the same package: same title, department, updated resource
pkg_dict = {'name':u'pollution',
'title':u'Pollution',
'extras':{u'department':'air',
u'country':'UK', #invariant
u'last_updated':'Tuesday', #variant
},
'resources':[{'url':'pollution.com/id/1',
'description':'ons/id/1'}],
}
self.loader.load_package(pkg_dict)
pkg = model.Package.by_name(pkg_dict['name'])
assert pkg
assert pkg.name == pkg_dict['name']
assert pkg.title == pkg_dict['title']
assert pkg.extras['country'] == pkg_dict['extras']['country']
assert pkg.extras['last_updated'] == pkg_dict['extras']['last_updated']
assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs)
assert len(pkg.resources) == 1, pkg.resources
assert pkg.resources[0].url == pkg_dict['resources'][0]['url'], pkg.resources[0].url
assert pkg.resources[0].description == pkg_dict['resources'][0]['description'], pkg.resources[0]['description']
# load the same package: same title, department, new resource
pkg_dict2 = {'name':u'pollution',
'title':u'Pollution',
'extras':{u'department':'air',
u'country':'UK', #invariant
u'last_updated':'Tuesday', #variant
},
'resources':[{'url':'pollution.com/id/3',
'description':'ons/id/3'}],
}
self.loader.load_package(pkg_dict2)
pkg = model.Package.by_name(pkg_dict2['name'])
assert pkg
assert pkg.name == pkg_dict2['name']
assert pkg.title == pkg_dict2['title']
assert pkg.extras['country'] == pkg_dict2['extras']['country']
assert pkg.extras['last_updated'] == pkg_dict2['extras']['last_updated']
assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs)
assert len(pkg.resources) == 2, pkg.resources
print pkg.resources
assert_equal(pkg.resources[0].url, pkg_dict['resources'][0]['url'])
assert pkg.resources[0].description == pkg_dict['resources'][0]['description'], pkg.resources[0]['description']
assert pkg.resources[1].url == pkg_dict2['resources'][0]['url'], pkg.resources[1].url
assert pkg.resources[1].description == pkg_dict2['resources'][0]['description'], pkg.resources[1]['description']
# load the different package: because of different department
pkg_dict3 = {'name':u'pollution',
'title':u'Pollution',
'extras':{u'department':'river',
u'country':'UK', #invariant
u'last_updated':'Tuesday', #variant
},
'resources':[{'url':'pollution.com/id/3',
'description':'Lots of pollution | ons/id/3'}],
}
self.loader.load_package(pkg_dict3)
CreateTestData.flag_for_deletion('pollution_')
assert count_pkgs() == num_pkgs + 3, (count_pkgs() - num_pkgs)
pkg_names = [pkg.name for pkg in model.Session.query(model.Package).all()]
pkg = model.Package.by_name(u'pollution_')
assert pkg
assert pkg.extras['department'] == pkg_dict3['extras']['department']
# load the same package: but with different country
# should just get a warning
pkg_dict4 = {'name':u'pollution',
'title':u'Pollution',
'extras':OrderedDict([
(u'department', 'air'),
(u'country', 'UK and France'), #invariant
(u'last_updated', 'Tuesday'), #variant
]),
'resources':[OrderedDict([
('url', 'pollution.com/id/3'),
('description', 'Lots of pollution | ons/id/3'),
])],
}
self.loader.load_package(pkg_dict4)
pkg = model.Package.by_name(pkg_dict4['name'])
assert pkg
assert pkg.name == pkg_dict4['name']
assert pkg.title == pkg_dict4['title']
assert pkg.extras['country'] == pkg_dict4['extras']['country']
assert pkg.extras['last_updated'] == pkg_dict4['extras']['last_updated']
assert count_pkgs() == num_pkgs + 3, (count_pkgs() - num_pkgs)
assert len(pkg.resources) == 2, pkg.resources
assert pkg.resources[0].url == pkg_dict['resources'][0]['url'], pkg.resources[0].url
assert pkg.resources[0].description == pkg_dict['resources'][0]['description'], pkg.resources[0]['description']
assert pkg.resources[1].url == pkg_dict4['resources'][0]['url'], pkg.resources[1].url
assert pkg.resources[1].description == pkg_dict4['resources'][0]['description'], pkg.resources[1]['description']
class TestLoaderInsertingResourcesWithSynonym(TestLoaderBase):
@classmethod
def setup_class(self):
self.tsi = TestSearchIndexer()
super(TestLoaderInsertingResourcesWithSynonym, self).setup_class()
self.loader = ResourceSeriesLoader(
self.testclient, ['title', 'department'],
'ons/id/',
field_keys_to_expect_invariant=['country'],
synonyms={'department': [('air', 'sky')]})
# teardown is in the base class
def test_0_search_options(self):
loader = ResourceSeriesLoader(
self.testclient, ['title', 'department'],
'ons/id/',
field_keys_to_expect_invariant=['country'],
synonyms={
'department': [('dept1', 'dept2', 'dept3')],
'title': [('titleA', 'titleB', 'titleC')]
})
field_keys = ['title', 'department']
pkg_dict = {'title': 'titleA', 'extras': {'department': 'dept1'}}
opts = loader._get_search_options(field_keys, pkg_dict)
self.assert_equal(opts, [{
'department': 'dept1',
'title': 'titleA'
}, {
'department': 'dept2',
'title': 'titleA'
}, {
'department': 'dept3',
'title': 'titleA'
}, {
'department': 'dept1',
'title': 'titleB'
}, {
'department': 'dept1',
'title': 'titleC'
}, {
'department': 'dept2',
'title': 'titleB'
}, {
'department': 'dept2',
'title': 'titleC'
}, {
'department': 'dept3',
'title': 'titleB'
}, {
'department': 'dept3',
'title': 'titleC'
}])
def test_1_reload(self):
# create initial package
num_pkgs = count_pkgs()
pkg_dict = {
'name': u'pollution',
'title': u'Pollution',
'extras': {
u'department': 'air',
u'country': 'UK', #invariant
u'last_updated': 'Monday', #variant
},
'resources': [{
'url': 'pollution.com/1',
'description': 'ons/id/1'
}],
}
bogus_dict = {
'name': u'bogus',
'title': u'Pollution',
'extras': {
u'department': 'water',
u'country': 'UK',
u'last_updated': 'Monday',
},
'resources': [{
'url': 'pollution.com/2',
'description': 'ons/id/2'
}],
}
assert not model.Package.by_name(pkg_dict['name'])
assert not model.Package.by_name(bogus_dict['name'])
CreateTestData.create_arbitrary([pkg_dict, bogus_dict])
self.tsi.index()
pkg = model.Package.by_name(pkg_dict['name'])
assert pkg
assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs)
assert len(pkg.resources) == 1, pkg.resources
# load the similar package: same title, updated resource,
# BUT synonym department
pkg_dict = {
'name':
u'pollution',
'title':
u'Pollution',
'extras': {
u'department': 'sky',
u'country': 'UK', #invariant
u'last_updated': 'Tuesday', #variant
},
'resources': [{
'url': 'pollution.com/id/1',
'description': 'ons/id/1'
}],
}
self.loader.load_package(pkg_dict)
pkg = model.Package.by_name(pkg_dict['name'])
assert pkg
assert pkg.name == pkg_dict['name']
assert pkg.title == pkg_dict['title']
assert pkg.extras['country'] == pkg_dict['extras']['country']
assert pkg.extras['last_updated'] == pkg_dict['extras']['last_updated']
assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs)
assert len(pkg.resources) == 1, pkg.resources
assert pkg.resources[0].url == pkg_dict['resources'][0][
'url'], pkg.resources[0].url
assert pkg.resources[0].description == pkg_dict['resources'][0][
'description'], pkg.resources[0]['description']
# load the different package: because of different department
pkg_dict3 = {
'name':
u'pollution',
'title':
u'Pollution',
'extras': {
u'department': 'river',
u'country': 'UK', #invariant
u'last_updated': 'Tuesday', #variant
},
'resources': [{
'url': 'pollution.com/id/3',
'description': 'Lots of pollution | ons/id/3'
}],
}
self.loader.load_package(pkg_dict3)
CreateTestData.flag_for_deletion('pollution_')
assert count_pkgs() == num_pkgs + 3, (count_pkgs() - num_pkgs)
pkg_names = [
pkg.name for pkg in model.Session.query(model.Package).all()
]
pkg = model.Package.by_name(u'pollution_')
assert pkg
assert pkg.extras['department'] == pkg_dict3['extras']['department']
class TestLoaderInsertingResources(TestLoaderBase):
@classmethod
def setup_class(self):
self.tsi = TestSearchIndexer()
super(TestLoaderInsertingResources, self).setup_class()
self.loader = ResourceSeriesLoader(
self.testclient, ['title', 'department'],
'ons/id/',
field_keys_to_expect_invariant=['country'])
# teardown is in the base class
def test_0_reload(self):
# create initial package
num_pkgs = count_pkgs()
pkg_dict = {
'name': u'pollution',
'title': u'Pollution',
'extras': {
u'department': 'air',
u'country': 'UK', #invariant
u'last_updated': 'Monday', #variant
},
'resources': [{
'url': 'pollution.com/1',
'description': 'ons/id/1'
}],
}
bogus_dict = {
'name': u'bogus',
'title': u'Pollution',
'extras': {
u'department': 'water',
u'country': 'UK',
u'last_updated': 'Monday',
},
'resources': [{
'url': 'pollution.com/2',
'description': 'ons/id/2'
}],
}
assert not model.Package.by_name(pkg_dict['name'])
assert not model.Package.by_name(bogus_dict['name'])
CreateTestData.create_arbitrary([pkg_dict, bogus_dict])
self.tsi.index()
pkg = model.Package.by_name(pkg_dict['name'])
assert pkg
assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs)
assert len(pkg.resources) == 1, pkg.resources
# load the same package: same title, department, updated resource
pkg_dict = {
'name':
u'pollution',
'title':
u'Pollution',
'extras': {
u'department': 'air',
u'country': 'UK', #invariant
u'last_updated': 'Tuesday', #variant
},
'resources': [{
'url': 'pollution.com/id/1',
'description': 'ons/id/1'
}],
}
self.loader.load_package(pkg_dict)
pkg = model.Package.by_name(pkg_dict['name'])
assert pkg
assert pkg.name == pkg_dict['name']
assert pkg.title == pkg_dict['title']
assert pkg.extras['country'] == pkg_dict['extras']['country']
assert pkg.extras['last_updated'] == pkg_dict['extras']['last_updated']
assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs)
assert len(pkg.resources) == 1, pkg.resources
assert pkg.resources[0].url == pkg_dict['resources'][0][
'url'], pkg.resources[0].url
assert pkg.resources[0].description == pkg_dict['resources'][0][
'description'], pkg.resources[0]['description']
# load the same package: same title, department, new resource
pkg_dict2 = {
'name':
u'pollution',
'title':
u'Pollution',
'extras': {
u'department': 'air',
u'country': 'UK', #invariant
u'last_updated': 'Tuesday', #variant
},
'resources': [{
'url': 'pollution.com/id/3',
'description': 'ons/id/3'
}],
}
self.loader.load_package(pkg_dict2)
pkg = model.Package.by_name(pkg_dict2['name'])
assert pkg
assert pkg.name == pkg_dict2['name']
assert pkg.title == pkg_dict2['title']
assert pkg.extras['country'] == pkg_dict2['extras']['country']
assert pkg.extras['last_updated'] == pkg_dict2['extras'][
'last_updated']
assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs)
assert len(pkg.resources) == 2, pkg.resources
print pkg.resources
assert_equal(pkg.resources[0].url, pkg_dict['resources'][0]['url'])
assert pkg.resources[0].description == pkg_dict['resources'][0][
'description'], pkg.resources[0]['description']
assert pkg.resources[1].url == pkg_dict2['resources'][0][
'url'], pkg.resources[1].url
assert pkg.resources[1].description == pkg_dict2['resources'][0][
'description'], pkg.resources[1]['description']
# load the different package: because of different department
pkg_dict3 = {
'name':
u'pollution',
'title':
u'Pollution',
'extras': {
u'department': 'river',
u'country': 'UK', #invariant
u'last_updated': 'Tuesday', #variant
},
'resources': [{
'url': 'pollution.com/id/3',
'description': 'Lots of pollution | ons/id/3'
}],
}
self.loader.load_package(pkg_dict3)
CreateTestData.flag_for_deletion('pollution_')
assert count_pkgs() == num_pkgs + 3, (count_pkgs() - num_pkgs)
pkg_names = [
pkg.name for pkg in model.Session.query(model.Package).all()
]
pkg = model.Package.by_name(u'pollution_')
assert pkg
assert pkg.extras['department'] == pkg_dict3['extras']['department']
# load the same package: but with different country
# should just get a warning
pkg_dict4 = {
'name':
u'pollution',
'title':
u'Pollution',
'extras':
OrderedDict([
(u'department', 'air'),
(u'country', 'UK and France'), #invariant
(u'last_updated', 'Tuesday'), #variant
]),
'resources': [
OrderedDict([
('url', 'pollution.com/id/3'),
('description', 'Lots of pollution | ons/id/3'),
])
],
}
self.loader.load_package(pkg_dict4)
pkg = model.Package.by_name(pkg_dict4['name'])
assert pkg
assert pkg.name == pkg_dict4['name']
assert pkg.title == pkg_dict4['title']
assert pkg.extras['country'] == pkg_dict4['extras']['country']
assert pkg.extras['last_updated'] == pkg_dict4['extras'][
'last_updated']
assert count_pkgs() == num_pkgs + 3, (count_pkgs() - num_pkgs)
assert len(pkg.resources) == 2, pkg.resources
assert pkg.resources[0].url == pkg_dict['resources'][0][
'url'], pkg.resources[0].url
assert pkg.resources[0].description == pkg_dict['resources'][0][
'description'], pkg.resources[0]['description']
assert pkg.resources[1].url == pkg_dict4['resources'][0][
'url'], pkg.resources[1].url
assert pkg.resources[1].description == pkg_dict4['resources'][0][
'description'], pkg.resources[1]['description']
