class TestLoaderInsertingResourcesWithSynonym(TestLoaderBase): @classmethod def setup_class(self): self.tsi = TestSearchIndexer() super(TestLoaderInsertingResourcesWithSynonym, self).setup_class() self.loader = ResourceSeriesLoader( self.testclient, ['title', 'department'], 'ons/id/', field_keys_to_expect_invariant=['country'], synonyms={'department': [('air', 'sky')]} ) # teardown is in the base class def test_0_search_options(self): loader = ResourceSeriesLoader( self.testclient, ['title', 'department'], 'ons/id/', field_keys_to_expect_invariant=['country'], synonyms={'department': [('dept1', 'dept2', 'dept3')], 'title': [('titleA', 'titleB', 'titleC')]} ) field_keys = ['title', 'department'] pkg_dict = {'title':'titleA', 'extras':{'department':'dept1'}} opts = loader._get_search_options(field_keys, pkg_dict) self.assert_equal(opts, [{'department': 'dept1', 'title': 'titleA'}, {'department': 'dept2', 'title': 'titleA'}, {'department': 'dept3', 'title': 'titleA'}, {'department': 'dept1', 'title': 'titleB'}, {'department': 'dept1', 'title': 'titleC'}, {'department': 'dept2', 'title': 'titleB'}, {'department': 'dept2', 'title': 'titleC'}, {'department': 'dept3', 'title': 'titleB'}, {'department': 'dept3', 'title': 'titleC'}]) def test_1_reload(self): # create initial package num_pkgs = count_pkgs() pkg_dict = {'name':u'pollution', 'title':u'Pollution', 'extras':{u'department':'air', u'country':'UK', #invariant u'last_updated':'Monday', #variant }, 'resources':[{'url':'pollution.com/1', 'description':'ons/id/1'}], } bogus_dict = {'name':u'bogus', 'title':u'Pollution', 'extras':{u'department':'water', u'country':'UK', u'last_updated':'Monday', }, 'resources':[{'url':'pollution.com/2', 'description':'ons/id/2'}], } assert not model.Package.by_name(pkg_dict['name']) assert not model.Package.by_name(bogus_dict['name']) CreateTestData.create_arbitrary([pkg_dict, bogus_dict]) self.tsi.index() pkg = model.Package.by_name(pkg_dict['name']) assert pkg assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs) assert len(pkg.resources) == 1, pkg.resources # load the similar package: same title, updated resource, # BUT synonym department pkg_dict = {'name':u'pollution', 'title':u'Pollution', 'extras':{u'department':'sky', u'country':'UK', #invariant u'last_updated':'Tuesday', #variant }, 'resources':[{'url':'pollution.com/id/1', 'description':'ons/id/1'}], } self.loader.load_package(pkg_dict) pkg = model.Package.by_name(pkg_dict['name']) assert pkg assert pkg.name == pkg_dict['name'] assert pkg.title == pkg_dict['title'] assert pkg.extras['country'] == pkg_dict['extras']['country'] assert pkg.extras['last_updated'] == pkg_dict['extras']['last_updated'] assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs) assert len(pkg.resources) == 1, pkg.resources assert pkg.resources[0].url == pkg_dict['resources'][0]['url'], pkg.resources[0].url assert pkg.resources[0].description == pkg_dict['resources'][0]['description'], pkg.resources[0]['description'] # load the different package: because of different department pkg_dict3 = {'name':u'pollution', 'title':u'Pollution', 'extras':{u'department':'river', u'country':'UK', #invariant u'last_updated':'Tuesday', #variant }, 'resources':[{'url':'pollution.com/id/3', 'description':'Lots of pollution | ons/id/3'}], } self.loader.load_package(pkg_dict3) CreateTestData.flag_for_deletion('pollution_') assert count_pkgs() == num_pkgs + 3, (count_pkgs() - num_pkgs) pkg_names = [pkg.name for pkg in model.Session.query(model.Package).all()] pkg = model.Package.by_name(u'pollution_') assert pkg assert pkg.extras['department'] == pkg_dict3['extras']['department']
class TestLoaderInsertingResourcesWithSynonym(TestLoaderBase): @classmethod def setup_class(self): self.tsi = TestSearchIndexer() super(TestLoaderInsertingResourcesWithSynonym, self).setup_class() self.loader = ResourceSeriesLoader( self.testclient, ['title', 'department'], 'ons/id/', field_keys_to_expect_invariant=['country'], synonyms={'department': [('air', 'sky')]}) # teardown is in the base class def test_0_search_options(self): loader = ResourceSeriesLoader( self.testclient, ['title', 'department'], 'ons/id/', field_keys_to_expect_invariant=['country'], synonyms={ 'department': [('dept1', 'dept2', 'dept3')], 'title': [('titleA', 'titleB', 'titleC')] }) field_keys = ['title', 'department'] pkg_dict = {'title': 'titleA', 'extras': {'department': 'dept1'}} opts = loader._get_search_options(field_keys, pkg_dict) self.assert_equal(opts, [{ 'department': 'dept1', 'title': 'titleA' }, { 'department': 'dept2', 'title': 'titleA' }, { 'department': 'dept3', 'title': 'titleA' }, { 'department': 'dept1', 'title': 'titleB' }, { 'department': 'dept1', 'title': 'titleC' }, { 'department': 'dept2', 'title': 'titleB' }, { 'department': 'dept2', 'title': 'titleC' }, { 'department': 'dept3', 'title': 'titleB' }, { 'department': 'dept3', 'title': 'titleC' }]) def test_1_reload(self): # create initial package num_pkgs = count_pkgs() pkg_dict = { 'name': u'pollution', 'title': u'Pollution', 'extras': { u'department': 'air', u'country': 'UK', #invariant u'last_updated': 'Monday', #variant }, 'resources': [{ 'url': 'pollution.com/1', 'description': 'ons/id/1' }], } bogus_dict = { 'name': u'bogus', 'title': u'Pollution', 'extras': { u'department': 'water', u'country': 'UK', u'last_updated': 'Monday', }, 'resources': [{ 'url': 'pollution.com/2', 'description': 'ons/id/2' }], } assert not model.Package.by_name(pkg_dict['name']) assert not model.Package.by_name(bogus_dict['name']) CreateTestData.create_arbitrary([pkg_dict, bogus_dict]) self.tsi.index() pkg = model.Package.by_name(pkg_dict['name']) assert pkg assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs) assert len(pkg.resources) == 1, pkg.resources # load the similar package: same title, updated resource, # BUT synonym department pkg_dict = { 'name': u'pollution', 'title': u'Pollution', 'extras': { u'department': 'sky', u'country': 'UK', #invariant u'last_updated': 'Tuesday', #variant }, 'resources': [{ 'url': 'pollution.com/id/1', 'description': 'ons/id/1' }], } self.loader.load_package(pkg_dict) pkg = model.Package.by_name(pkg_dict['name']) assert pkg assert pkg.name == pkg_dict['name'] assert pkg.title == pkg_dict['title'] assert pkg.extras['country'] == pkg_dict['extras']['country'] assert pkg.extras['last_updated'] == pkg_dict['extras']['last_updated'] assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs) assert len(pkg.resources) == 1, pkg.resources assert pkg.resources[0].url == pkg_dict['resources'][0][ 'url'], pkg.resources[0].url assert pkg.resources[0].description == pkg_dict['resources'][0][ 'description'], pkg.resources[0]['description'] # load the different package: because of different department pkg_dict3 = { 'name': u'pollution', 'title': u'Pollution', 'extras': { u'department': 'river', u'country': 'UK', #invariant u'last_updated': 'Tuesday', #variant }, 'resources': [{ 'url': 'pollution.com/id/3', 'description': 'Lots of pollution | ons/id/3' }], } self.loader.load_package(pkg_dict3) CreateTestData.flag_for_deletion('pollution_') assert count_pkgs() == num_pkgs + 3, (count_pkgs() - num_pkgs) pkg_names = [ pkg.name for pkg in model.Session.query(model.Package).all() ] pkg = model.Package.by_name(u'pollution_') assert pkg assert pkg.extras['department'] == pkg_dict3['extras']['department']
class TestLoaderInsertingResources(TestLoaderBase): @classmethod def setup_class(self): self.tsi = TestSearchIndexer() super(TestLoaderInsertingResources, self).setup_class() self.loader = ResourceSeriesLoader( self.testclient, ['title', 'department'], 'ons/id/', field_keys_to_expect_invariant=['country']) # teardown is in the base class def test_0_reload(self): # create initial package num_pkgs = count_pkgs() pkg_dict = {'name':u'pollution', 'title':u'Pollution', 'extras':{u'department':'air', u'country':'UK', #invariant u'last_updated':'Monday', #variant }, 'resources':[{'url':'pollution.com/1', 'description':'ons/id/1'}], } bogus_dict = {'name':u'bogus', 'title':u'Pollution', 'extras':{u'department':'water', u'country':'UK', u'last_updated':'Monday', }, 'resources':[{'url':'pollution.com/2', 'description':'ons/id/2'}], } assert not model.Package.by_name(pkg_dict['name']) assert not model.Package.by_name(bogus_dict['name']) CreateTestData.create_arbitrary([pkg_dict, bogus_dict]) self.tsi.index() pkg = model.Package.by_name(pkg_dict['name']) assert pkg assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs) assert len(pkg.resources) == 1, pkg.resources # load the same package: same title, department, updated resource pkg_dict = {'name':u'pollution', 'title':u'Pollution', 'extras':{u'department':'air', u'country':'UK', #invariant u'last_updated':'Tuesday', #variant }, 'resources':[{'url':'pollution.com/id/1', 'description':'ons/id/1'}], } self.loader.load_package(pkg_dict) pkg = model.Package.by_name(pkg_dict['name']) assert pkg assert pkg.name == pkg_dict['name'] assert pkg.title == pkg_dict['title'] assert pkg.extras['country'] == pkg_dict['extras']['country'] assert pkg.extras['last_updated'] == pkg_dict['extras']['last_updated'] assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs) assert len(pkg.resources) == 1, pkg.resources assert pkg.resources[0].url == pkg_dict['resources'][0]['url'], pkg.resources[0].url assert pkg.resources[0].description == pkg_dict['resources'][0]['description'], pkg.resources[0]['description'] # load the same package: same title, department, new resource pkg_dict2 = {'name':u'pollution', 'title':u'Pollution', 'extras':{u'department':'air', u'country':'UK', #invariant u'last_updated':'Tuesday', #variant }, 'resources':[{'url':'pollution.com/id/3', 'description':'ons/id/3'}], } self.loader.load_package(pkg_dict2) pkg = model.Package.by_name(pkg_dict2['name']) assert pkg assert pkg.name == pkg_dict2['name'] assert pkg.title == pkg_dict2['title'] assert pkg.extras['country'] == pkg_dict2['extras']['country'] assert pkg.extras['last_updated'] == pkg_dict2['extras']['last_updated'] assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs) assert len(pkg.resources) == 2, pkg.resources print pkg.resources assert_equal(pkg.resources[0].url, pkg_dict['resources'][0]['url']) assert pkg.resources[0].description == pkg_dict['resources'][0]['description'], pkg.resources[0]['description'] assert pkg.resources[1].url == pkg_dict2['resources'][0]['url'], pkg.resources[1].url assert pkg.resources[1].description == pkg_dict2['resources'][0]['description'], pkg.resources[1]['description'] # load the different package: because of different department pkg_dict3 = {'name':u'pollution', 'title':u'Pollution', 'extras':{u'department':'river', u'country':'UK', #invariant u'last_updated':'Tuesday', #variant }, 'resources':[{'url':'pollution.com/id/3', 'description':'Lots of pollution | ons/id/3'}], } self.loader.load_package(pkg_dict3) CreateTestData.flag_for_deletion('pollution_') assert count_pkgs() == num_pkgs + 3, (count_pkgs() - num_pkgs) pkg_names = [pkg.name for pkg in model.Session.query(model.Package).all()] pkg = model.Package.by_name(u'pollution_') assert pkg assert pkg.extras['department'] == pkg_dict3['extras']['department'] # load the same package: but with different country # should just get a warning pkg_dict4 = {'name':u'pollution', 'title':u'Pollution', 'extras':OrderedDict([ (u'department', 'air'), (u'country', 'UK and France'), #invariant (u'last_updated', 'Tuesday'), #variant ]), 'resources':[OrderedDict([ ('url', 'pollution.com/id/3'), ('description', 'Lots of pollution | ons/id/3'), ])], } self.loader.load_package(pkg_dict4) pkg = model.Package.by_name(pkg_dict4['name']) assert pkg assert pkg.name == pkg_dict4['name'] assert pkg.title == pkg_dict4['title'] assert pkg.extras['country'] == pkg_dict4['extras']['country'] assert pkg.extras['last_updated'] == pkg_dict4['extras']['last_updated'] assert count_pkgs() == num_pkgs + 3, (count_pkgs() - num_pkgs) assert len(pkg.resources) == 2, pkg.resources assert pkg.resources[0].url == pkg_dict['resources'][0]['url'], pkg.resources[0].url assert pkg.resources[0].description == pkg_dict['resources'][0]['description'], pkg.resources[0]['description'] assert pkg.resources[1].url == pkg_dict4['resources'][0]['url'], pkg.resources[1].url assert pkg.resources[1].description == pkg_dict4['resources'][0]['description'], pkg.resources[1]['description']
class TestLoaderInsertingResources(TestLoaderBase): @classmethod def setup_class(self): self.tsi = TestSearchIndexer() super(TestLoaderInsertingResources, self).setup_class() self.loader = ResourceSeriesLoader( self.testclient, ['title', 'department'], 'ons/id/', field_keys_to_expect_invariant=['country']) # teardown is in the base class def test_0_reload(self): # create initial package num_pkgs = count_pkgs() pkg_dict = { 'name': u'pollution', 'title': u'Pollution', 'extras': { u'department': 'air', u'country': 'UK', #invariant u'last_updated': 'Monday', #variant }, 'resources': [{ 'url': 'pollution.com/1', 'description': 'ons/id/1' }], } bogus_dict = { 'name': u'bogus', 'title': u'Pollution', 'extras': { u'department': 'water', u'country': 'UK', u'last_updated': 'Monday', }, 'resources': [{ 'url': 'pollution.com/2', 'description': 'ons/id/2' }], } assert not model.Package.by_name(pkg_dict['name']) assert not model.Package.by_name(bogus_dict['name']) CreateTestData.create_arbitrary([pkg_dict, bogus_dict]) self.tsi.index() pkg = model.Package.by_name(pkg_dict['name']) assert pkg assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs) assert len(pkg.resources) == 1, pkg.resources # load the same package: same title, department, updated resource pkg_dict = { 'name': u'pollution', 'title': u'Pollution', 'extras': { u'department': 'air', u'country': 'UK', #invariant u'last_updated': 'Tuesday', #variant }, 'resources': [{ 'url': 'pollution.com/id/1', 'description': 'ons/id/1' }], } self.loader.load_package(pkg_dict) pkg = model.Package.by_name(pkg_dict['name']) assert pkg assert pkg.name == pkg_dict['name'] assert pkg.title == pkg_dict['title'] assert pkg.extras['country'] == pkg_dict['extras']['country'] assert pkg.extras['last_updated'] == pkg_dict['extras']['last_updated'] assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs) assert len(pkg.resources) == 1, pkg.resources assert pkg.resources[0].url == pkg_dict['resources'][0][ 'url'], pkg.resources[0].url assert pkg.resources[0].description == pkg_dict['resources'][0][ 'description'], pkg.resources[0]['description'] # load the same package: same title, department, new resource pkg_dict2 = { 'name': u'pollution', 'title': u'Pollution', 'extras': { u'department': 'air', u'country': 'UK', #invariant u'last_updated': 'Tuesday', #variant }, 'resources': [{ 'url': 'pollution.com/id/3', 'description': 'ons/id/3' }], } self.loader.load_package(pkg_dict2) pkg = model.Package.by_name(pkg_dict2['name']) assert pkg assert pkg.name == pkg_dict2['name'] assert pkg.title == pkg_dict2['title'] assert pkg.extras['country'] == pkg_dict2['extras']['country'] assert pkg.extras['last_updated'] == pkg_dict2['extras'][ 'last_updated'] assert count_pkgs() == num_pkgs + 2, (count_pkgs() - num_pkgs) assert len(pkg.resources) == 2, pkg.resources print pkg.resources assert_equal(pkg.resources[0].url, pkg_dict['resources'][0]['url']) assert pkg.resources[0].description == pkg_dict['resources'][0][ 'description'], pkg.resources[0]['description'] assert pkg.resources[1].url == pkg_dict2['resources'][0][ 'url'], pkg.resources[1].url assert pkg.resources[1].description == pkg_dict2['resources'][0][ 'description'], pkg.resources[1]['description'] # load the different package: because of different department pkg_dict3 = { 'name': u'pollution', 'title': u'Pollution', 'extras': { u'department': 'river', u'country': 'UK', #invariant u'last_updated': 'Tuesday', #variant }, 'resources': [{ 'url': 'pollution.com/id/3', 'description': 'Lots of pollution | ons/id/3' }], } self.loader.load_package(pkg_dict3) CreateTestData.flag_for_deletion('pollution_') assert count_pkgs() == num_pkgs + 3, (count_pkgs() - num_pkgs) pkg_names = [ pkg.name for pkg in model.Session.query(model.Package).all() ] pkg = model.Package.by_name(u'pollution_') assert pkg assert pkg.extras['department'] == pkg_dict3['extras']['department'] # load the same package: but with different country # should just get a warning pkg_dict4 = { 'name': u'pollution', 'title': u'Pollution', 'extras': OrderedDict([ (u'department', 'air'), (u'country', 'UK and France'), #invariant (u'last_updated', 'Tuesday'), #variant ]), 'resources': [ OrderedDict([ ('url', 'pollution.com/id/3'), ('description', 'Lots of pollution | ons/id/3'), ]) ], } self.loader.load_package(pkg_dict4) pkg = model.Package.by_name(pkg_dict4['name']) assert pkg assert pkg.name == pkg_dict4['name'] assert pkg.title == pkg_dict4['title'] assert pkg.extras['country'] == pkg_dict4['extras']['country'] assert pkg.extras['last_updated'] == pkg_dict4['extras'][ 'last_updated'] assert count_pkgs() == num_pkgs + 3, (count_pkgs() - num_pkgs) assert len(pkg.resources) == 2, pkg.resources assert pkg.resources[0].url == pkg_dict['resources'][0][ 'url'], pkg.resources[0].url assert pkg.resources[0].description == pkg_dict['resources'][0][ 'description'], pkg.resources[0]['description'] assert pkg.resources[1].url == pkg_dict4['resources'][0][ 'url'], pkg.resources[1].url assert pkg.resources[1].description == pkg_dict4['resources'][0][ 'description'], pkg.resources[1]['description']