class RobustnessTests(TestCase): layer = COLLECTIVE_SOLR_MOCK_REGISTRY_FIXTURE def setUp(self): self.mngr = SolrConnectionManager() self.mngr.setHost(active=True) self.conn = self.mngr.getConnection() self.proc = SolrIndexProcessor(self.mngr) self.log = [] # catch log messages... def logger(*args): self.log.extend(args) logger_indexer.warning = logger config = getConfig() config.atomic_updates = True def tearDown(self): self.mngr.closeConnection() self.mngr.setHost(active=False) def testIndexingWithUniqueKeyMissing(self): # fake schema response fakehttp(self.conn, getData('simple_schema.xml')) # read and cache the schema self.mngr.getSchema() response = getData('add_response.txt') output = fakehttp(self.conn, response) # fake add response foo = Foo(id='500', name='foo') # indexing sends data self.proc.index(foo) # nothing happened... self.assertEqual(len(output), 0) self.assertEqual( self.log, ['schema is missing unique key, skipping indexing of %r', foo]) def testUnindexingWithUniqueKeyMissing(self): # fake schema response fakehttp(self.conn, getData('simple_schema.xml')) # read and cache the schema self.mngr.getSchema() response = getData('delete_response.txt') # fake delete response output = fakehttp(self.conn, response) foo = Foo(id='500', name='foo') # unindexing sends data self.proc.unindex(foo) # nothing happened... self.assertEqual(len(output), 0) self.assertEqual( self.log, ['schema is missing unique key, skipping unindexing of %r', foo])
def testThreeRequests(self): mngr = SolrConnectionManager(active=True) proc = SolrIndexProcessor(mngr) output = fakehttp(mngr.getConnection(), getData('schema.xml'), getData('add_response.txt'), getData('delete_response.txt')) proc.index(self.foo) proc.unindex(self.foo) mngr.closeConnection() self.assertEqual(len(output), 3) self.failUnless(output.get().startswith(self.schema_request)) self.assertEqual(sortFields(output.get()), getData('add_request.txt')) self.assertEqual(output.get(), getData('delete_request.txt'))
class RobustnessTests(TestCase): layer = COLLECTIVE_SOLR_MOCK_REGISTRY_FIXTURE def setUp(self): self.mngr = SolrConnectionManager() self.mngr.setHost(active=True) self.conn = self.mngr.getConnection() self.proc = SolrIndexProcessor(self.mngr) self.log = [] # catch log messages... def logger(*args): self.log.extend(args) logger_indexer.warning = logger config = getConfig() config.atomic_updates = True def tearDown(self): self.mngr.closeConnection() self.mngr.setHost(active=False) def testIndexingWithUniqueKeyMissing(self): # fake schema response fakehttp(self.conn, getData('simple_schema.xml')) # read and cache the schema self.mngr.getSchema() response = getData('add_response.txt') output = fakehttp(self.conn, response) # fake add response foo = Foo(id='500', name='foo') # indexing sends data self.proc.index(foo) # nothing happened... self.assertEqual(len(output), 0) self.assertEqual(self.log, [ 'schema is missing unique key, skipping indexing of %r', foo]) def testUnindexingWithUniqueKeyMissing(self): # fake schema response fakehttp(self.conn, getData('simple_schema.xml')) # read and cache the schema self.mngr.getSchema() response = getData('delete_response.txt') # fake delete response output = fakehttp(self.conn, response) foo = Foo(id='500', name='foo') # unindexing sends data self.proc.unindex(foo) # nothing happened... self.assertEqual(len(output), 0) self.assertEqual(self.log, [ 'schema is missing unique key, skipping unindexing of %r', foo])
def testExtraRequest(self): # basically the same as `testThreeRequests`, except it # tests adding fake responses consecutively mngr = SolrConnectionManager(active=True) proc = SolrIndexProcessor(mngr) conn = mngr.getConnection() output = fakehttp(conn, getData('schema.xml')) fakemore(conn, getData('add_response.txt')) proc.index(self.foo) fakemore(conn, getData('delete_response.txt')) proc.unindex(self.foo) mngr.closeConnection() self.assertEqual(len(output), 3) self.failUnless(output.get().startswith(self.schema_request)) self.assertEqual(sortFields(output.get()), getData('add_request.txt')) self.assertEqual(output.get(), getData('delete_request.txt'))
def testFourRequests(self): mngr = SolrConnectionManager(active=True) proc = SolrIndexProcessor(mngr) output = fakehttp( mngr.getConnection(), getData("schema.xml"), getData("add_response.txt"), getData("delete_response.txt"), getData("commit_response.txt"), ) proc.index(self.foo) proc.unindex(self.foo) proc.commit() mngr.closeConnection() self.assertEqual(len(output), 4) self.failUnless(output.get().decode("utf-8").startswith(self.schema_request)) self.assertEqual( sortFields(output.get()), getData("add_request.txt").rstrip(b"\n") ) self.assertEqual(output.get(), getData("delete_request.txt").rstrip(b"\n")) self.assertEqual(output.get(), getData("commit_request.txt").rstrip(b"\n"))
class QueueIndexerTests(TestCase): def setUp(self): provideUtility(SolrConnectionConfig(), ISolrConnectionConfig) self.mngr = SolrConnectionManager() self.mngr.setHost(active=True) conn = self.mngr.getConnection() fakehttp(conn, getData('schema.xml')) # fake schema response self.mngr.getSchema() # read and cache the schema self.proc = SolrIndexProcessor(self.mngr) def tearDown(self): self.mngr.closeConnection() self.mngr.setHost(active=False) def testPrepareData(self): data = {'allowedRolesAndUsers': [ 'user:test_user_1_', 'user:portal_owner']} prepareData(data) self.assertEqual( data, { 'allowedRolesAndUsers': [ 'user$test_user_1_', 'user$portal_owner' ] } ) def testLanguageParameterHandling(self): # empty strings are replaced... data = {'Language': ['en', '']} prepareData(data) self.assertEqual(data, {'Language': ['en', 'any']}) data = {'Language': ''} prepareData(data) self.assertEqual(data, {'Language': 'any'}) # for other indices this shouldn't happen... data = {'Foo': ['en', '']} prepareData(data) self.assertEqual(data, {'Foo': ['en', '']}) def testIndexObject(self): response = getData('add_response.txt') # fake add response output = fakehttp(self.mngr.getConnection(), response) # indexing sends data self.proc.index(Foo(id='500', name='python test doc')) self.assertEqual(sortFields(str(output)), getData('add_request.txt')) def testIndexAccessorRaises(self): response = getData('add_response.txt') # fake add response output = fakehttp(self.mngr.getConnection(), response) def brokenfunc(): raise ValueError self.proc.index(Foo(id='500', name='python test doc', text=brokenfunc)) # indexing sends data self.assertEqual(sortFields(str(output)), getData('add_request.txt')) def testPartialIndexObject(self): foo = Foo(id='500', name='foo', price=42.0) # first index all attributes... response = getData('add_response.txt') output = fakehttp(self.mngr.getConnection(), response) self.proc.index(foo) self.assert_(str(output).find( '<field name="price">42.0</field>') > 0, '"price" data not found') # then only a subset... response = getData('add_response.txt') output = fakehttp(self.mngr.getConnection(), response) self.proc.index(foo, attributes=['id', 'name']) output = str(output) self.assert_( output.find('<field name="name">foo</field>') > 0, '"name" data not found' ) # at this point we'd normally check for a partial update: # self.assertEqual(output.find('price'), -1, '"price" data found?') # self.assertEqual(output.find('42'), -1, '"price" data found?') # however, until SOLR-139 has been implemented (re)index operations # always need to provide data for all attributes in the schema... self.assert_( output.find('<field name="price">42.0</field>') > 0, '"price" data not found' ) def testDateIndexing(self): foo = Foo(id='zeidler', name='andi', cat='nerd', timestamp=DateTime('May 11 1972 03:45 GMT')) response = getData('add_response.txt') # fake add response output = fakehttp(self.mngr.getConnection(), response) self.proc.index(foo) required = '<field name="timestamp">1972-05-11T03:45:00.000Z</field>' self.assert_(str(output).find(required) > 0, '"date" data not found') def testDateIndexingWithPythonDateTime(self): foo = Foo(id='gerken', name='patrick', cat='nerd', timestamp=datetime(1980, 9, 29, 14, 02)) response = getData('add_response.txt') # fake add response output = fakehttp(self.mngr.getConnection(), response) self.proc.index(foo) required = '<field name="timestamp">1980-09-29T14:02:00.000Z</field>' self.assert_(str(output).find(required) > 0, '"date" data not found') def testDateIndexingWithPythonDate(self): foo = Foo(id='brand', name='jan-carel', cat='nerd', timestamp=date(1982, 8, 05)) response = getData('add_response.txt') # fake add response output = fakehttp(self.mngr.getConnection(), response) self.proc.index(foo) required = '<field name="timestamp">1982-08-05T00:00:00.000Z</field>' self.assert_(str(output).find(required) > 0, '"date" data not found') def testReindexObject(self): response = getData('add_response.txt') # fake add response output = fakehttp(self.mngr.getConnection(), response) # reindexing sends data self.proc.reindex(Foo(id='500', name='python test doc')) self.assertEqual(sortFields(str(output)), getData('add_request.txt')) def testUnindexObject(self): response = getData('delete_response.txt') # fake response output = fakehttp(self.mngr.getConnection(), response) # unindexing sends data self.proc.unindex(Foo(id='500', name='python test doc')) self.assertEqual(str(output), getData('delete_request.txt')) def testCommit(self): response = getData('commit_response.txt') # fake response output = fakehttp(self.mngr.getConnection(), response) # committing sends data self.proc.commit() self.assertEqual(str(output), getData('commit_request.txt')) def testNoIndexingWithoutAllRequiredFields(self): response = getData('dummy_response.txt') # fake add response output = fakehttp(self.mngr.getConnection(), response) # indexing sends data self.proc.index(Foo(id='500')) self.assertEqual(str(output), '') def testIndexerMethods(self): class Bar(Foo): def cat(self): return 'nerd' def price(self): raise AttributeError('price') foo = Bar(id='500', name='foo') # raising the exception should keep the attribute from being indexed response = getData('add_response.txt') output = fakehttp(self.mngr.getConnection(), response) self.proc.index(foo) output = str(output) self.assertTrue( output.find('<field name="cat">nerd</field>') > 0, '"cat" data not found' ) self.assertEqual(output.find('price'), -1, '"price" data found?')
class QueueIndexerTests(TestCase): def setUp(self): provideUtility(SolrConnectionConfig(), ISolrConnectionConfig) self.mngr = SolrConnectionManager() self.mngr.setHost(active=True) conn = self.mngr.getConnection() fakehttp(conn, getData('schema.xml')) # fake schema response self.mngr.getSchema() # read and cache the schema self.proc = SolrIndexProcessor(self.mngr) def tearDown(self): self.mngr.closeConnection() self.mngr.setHost(active=False) def testPrepareData(self): data = {'allowedRolesAndUsers': ['user:test_user_1_', 'user:portal_owner']} prepareData(data) self.assertEqual(data, {'allowedRolesAndUsers': ['user$test_user_1_', 'user$portal_owner']}) def testLanguageParameterHandling(self): # empty strings are replaced... data = {'Language': ['en', '']} prepareData(data) self.assertEqual(data, {'Language': ['en', 'any']}) data = {'Language': ''} prepareData(data) self.assertEqual(data, {'Language': 'any'}) # for other indices this shouldn't happen... data = {'Foo': ['en', '']} prepareData(data) self.assertEqual(data, {'Foo': ['en', '']}) def testIndexObject(self): response = getData('add_response.txt') output = fakehttp(self.mngr.getConnection(), response) # fake add response self.proc.index(Foo(id='500', name='python test doc')) # indexing sends data self.assertEqual(sortFields(str(output)), getData('add_request.txt')) def testIndexAccessorRaises(self): response = getData('add_response.txt') output = fakehttp(self.mngr.getConnection(), response) # fake add response def brokenfunc(): raise ValueError self.proc.index(Foo(id='500', name='python test doc', text=brokenfunc)) # indexing sends data self.assertEqual(sortFields(str(output)), getData('add_request.txt')) def testPartialIndexObject(self): foo = Foo(id='500', name='foo', price=42.0) # first index all attributes... response = getData('add_response.txt') output = fakehttp(self.mngr.getConnection(), response) self.proc.index(foo) self.assert_(str(output).find('<field name="price">42.0</field>') > 0, '"price" data not found') # then only a subset... response = getData('add_response.txt') output = fakehttp(self.mngr.getConnection(), response) self.proc.index(foo, attributes=['id', 'name']) output = str(output) self.assert_(output.find('<field name="name">foo</field>') > 0, '"name" data not found') # at this point we'd normally check for a partial update: # self.assertEqual(output.find('price'), -1, '"price" data found?') # self.assertEqual(output.find('42'), -1, '"price" data found?') # however, until SOLR-139 has been implemented (re)index operations # always need to provide data for all attributes in the schema... self.assert_(output.find('<field name="price">42.0</field>') > 0, '"price" data not found') def testDateIndexing(self): foo = Foo(id='zeidler', name='andi', cat='nerd', timestamp=DateTime('May 11 1972 03:45 GMT')) response = getData('add_response.txt') output = fakehttp(self.mngr.getConnection(), response) # fake add response self.proc.index(foo) required = '<field name="timestamp">1972-05-11T03:45:00.000Z</field>' self.assert_(str(output).find(required) > 0, '"date" data not found') def testDateIndexingWithPythonDateTime(self): foo = Foo(id='gerken', name='patrick', cat='nerd', timestamp=datetime(1980, 9, 29, 14, 02)) response = getData('add_response.txt') output = fakehttp(self.mngr.getConnection(), response) # fake add response self.proc.index(foo) required = '<field name="timestamp">1980-09-29T14:02:00.000Z</field>' self.assert_(str(output).find(required) > 0, '"date" data not found') def testReindexObject(self): response = getData('add_response.txt') output = fakehttp(self.mngr.getConnection(), response) # fake add response self.proc.reindex(Foo(id='500', name='python test doc')) # reindexing sends data self.assertEqual(sortFields(str(output)), getData('add_request.txt')) def testUnindexObject(self): response = getData('delete_response.txt') output = fakehttp(self.mngr.getConnection(), response) # fake response self.proc.unindex(Foo(id='500', name='python test doc')) # unindexing sends data self.assertEqual(str(output), getData('delete_request.txt')) def testCommit(self): response = getData('commit_response.txt') output = fakehttp(self.mngr.getConnection(), response) # fake response self.proc.commit() # committing sends data self.assertEqual(str(output), getData('commit_request.txt')) def testNoIndexingWithoutAllRequiredFields(self): response = getData('dummy_response.txt') output = fakehttp(self.mngr.getConnection(), response) # fake add response self.proc.index(Foo(id='500')) # indexing sends data self.assertEqual(str(output), '') def testIndexerMethods(self): class Bar(Foo): def cat(self): return 'nerd' def price(self): raise AttributeError('price') foo = Bar(id='500', name='foo') # raising the exception should keep the attribute from being indexed response = getData('add_response.txt') output = fakehttp(self.mngr.getConnection(), response) self.proc.index(foo) output = str(output) self.assertTrue(output.find('<field name="cat">nerd</field>') > 0, '"cat" data not found') self.assertEqual(output.find('price'), -1, '"price" data found?')
class QueueIndexerTests(TestCase): layer = COLLECTIVE_SOLR_MOCK_REGISTRY_FIXTURE def setUp(self): self.mngr = SolrConnectionManager() self.mngr.setHost(active=True) conn = self.mngr.getConnection() fakehttp(conn, getData("schema.xml")) # fake schema response self.mngr.getSchema() # read and cache the schema self.proc = SolrIndexProcessor(self.mngr) config = getConfig() config.atomic_updates = True def tearDown(self): self.mngr.closeConnection() self.mngr.setHost(active=False) def testPrepareData(self): data = {"allowedRolesAndUsers": ["user:test_user_1_", "user:portal_owner"]} prepareData(data) self.assertEqual( data, {"allowedRolesAndUsers": ["user$test_user_1_", "user$portal_owner"]} ) def testLanguageParameterHandling(self): # empty strings are replaced... data = {"Language": ["en", ""]} prepareData(data) self.assertEqual(data, {"Language": ["en", "any"]}) data = {"Language": ""} prepareData(data) self.assertEqual(data, {"Language": "any"}) # for other indices this shouldn't happen... data = {"Foo": ["en", ""]} prepareData(data) self.assertEqual(data, {"Foo": ["en", ""]}) def testIndexObject(self): response = getData("add_response.txt") # fake add response output = fakehttp(self.mngr.getConnection(), response) # indexing sends data self.proc.index(Foo(id="500", name="python test doc")) self.assertEqual( sortFields(str(output).encode("utf-8")), getData("add_request.txt").rstrip(b"\n"), ) def testIndexAccessorRaises(self): response = getData("add_response.txt") # fake add response output = fakehttp(self.mngr.getConnection(), response) def brokenfunc(): raise ValueError self.proc.index( Foo(id="500", name="python test doc", text=brokenfunc) ) # indexing sends data self.assertEqual( sortFields(str(output).encode("utf-8")), getData("add_request.txt").rstrip(b"\n"), ) def testPartialIndexObject(self): foo = Foo(id="500", name="foo", price=42.0) # first index all attributes... response = getData("add_response.txt") output = fakehttp(self.mngr.getConnection(), response) self.proc.index(foo) self.assert_( str(output).find('<field name="price" update="set">42.0</field>') > 0, '"price" data not found', ) # then only a subset... response = getData("add_response.txt") output = fakehttp(self.mngr.getConnection(), response) self.proc.index(foo, attributes=["id", "name"]) output = str(output) self.assert_( output.find('<field name="name" update="set">foo</field>') > 0, '"name" data not found', ) # at this point we'd normally check for a partial update: self.assertEqual(output.find("price"), -1, '"price" data found?') self.assertEqual(output.find("42"), -1, '"price" data found?') def testDateIndexing(self): foo = Foo( id="zeidler", name="andi", cat="nerd", timestamp=DateTime("May 11 1972 03:45:59.999730 GMT"), ) response = getData("add_response.txt") # fake add response output = fakehttp(self.mngr.getConnection(), response) self.proc.index(foo) required = ( '<field name="timestamp" update="set">' "1972-05-11T03:45:59.999Z</field>" ) self.assert_(str(output).find(required) > 0, '"date" data not found') def testDateIndexingWithPythonDateTime(self): foo = Foo( id="gerken", name="patrick", cat="nerd", timestamp=datetime(1980, 9, 29, 14, 0o2, 59, 999730), ) response = getData("add_response.txt") # fake add response output = fakehttp(self.mngr.getConnection(), response) self.proc.index(foo) required = ( '<field name="timestamp" update="set">' "1980-09-29T14:02:59.999Z</field>" ) self.assert_(str(output).find(required) > 0, '"date" data not found') def testDateIndexingWithPythonDate(self): foo = Foo( id="brand", name="jan-carel", cat="nerd", timestamp=date(1982, 8, 0o5) ) response = getData("add_response.txt") # fake add response output = fakehttp(self.mngr.getConnection(), response) self.proc.index(foo) required = ( '<field name="timestamp" update="set">' "1982-08-05T00:00:00.000Z</field>" ) self.assert_(str(output).find(required) > 0, '"date" data not found') def testReindexObject(self): response = getData("add_response.txt") # fake add response output = fakehttp(self.mngr.getConnection(), response) # reindexing sends data self.proc.reindex(Foo(id="500", name="python test doc")) self.assertEqual( sortFields(str(output).encode("utf-8")), getData("add_request.txt").rstrip(b"\n"), ) def testUnindexObject(self): response = getData("delete_response.txt") # fake response output = fakehttp(self.mngr.getConnection(), response) # unindexing sends data self.proc.unindex(Foo(id="500", name="python test doc")) self.assertEqual( str(output), getData("delete_request.txt").decode("utf-8").rstrip("\n") ) def testCommit(self): response = getData("commit_response.txt") # fake response output = fakehttp(self.mngr.getConnection(), response) # committing sends data self.proc.commit() self.assertEqual( str(output), getData("commit_request.txt").decode("utf-8").rstrip("\n") ) def testNoIndexingWithoutAllRequiredFields(self): response = getData("dummy_response.txt") # fake add response output = fakehttp(self.mngr.getConnection(), response) # indexing sends data self.proc.index(Foo(id="500")) self.assertEqual(str(output), "") def testIndexerMethods(self): class Bar(Foo): def cat(self): return "nerd" def price(self): raise AttributeError("price") foo = Bar(id="500", name="foo") # raising the exception should keep the attribute from being indexed response = getData("add_response.txt") output = fakehttp(self.mngr.getConnection(), response) self.proc.index(foo) output = str(output) self.assertTrue( output.find('<field name="cat" update="set">nerd</field>') > 0, '"cat" data not found', ) self.assertEqual(output.find("price"), -1, '"price" data found?')