def test_toc(self): # tests the main TOC method, not the helper methods (they are # tested separately) self.repo.facets = MagicMock() self.repo.facet_select = MagicMock() self.repo.facet_query = MagicMock() self.repo.faceted_data = MagicMock() self.repo.log = Mock() self.repo.toc_pagesets = Mock() self.repo.toc_select_for_pages = Mock() self.repo.toc_generate_pages = Mock() self.repo.toc_generate_first_page = Mock() with patch('json.dump'): self.repo.toc() # assert facet_query was properly called, error and info msg # was printed self.assertEqual("http://localhost:8000/dataset/base", self.repo.facet_query.call_args[0][0]) self.assertTrue(self.repo.log.error.called) self.assertTrue(self.repo.log.info.called) # and that the rest of the methods were NOT called self.assertFalse(self.repo.toc_pagesets.called) self.assertFalse(self.repo.toc_select_for_pages.called) self.assertFalse(self.repo.toc_generate_pages.called) # test2: facet_select returns something self.repo.faceted_data.return_value = ["fake", "data"] with patch('json.load'): self.repo.toc() # Now all other methods should be called self.assertTrue(self.repo.toc_pagesets.called) self.assertTrue(self.repo.toc_select_for_pages.called) self.assertTrue(self.repo.toc_generate_pages.called)
def test_status(self): want = """ Status for document repository 'base' (ferenda.documentrepository.DocumentRepository) download: None. parse: None. generated: None. """.strip() repo = DocumentRepository(datadir=self.tempdir) with patch("builtins.print") as printmock: manager.status(repo) got = "\n".join([x[1][0] for x in printmock.mock_calls]) self.assertEqual(want,got) # test both status and get_status in one swoop. for basefile in range(1,13): util.writefile(repo.store.downloaded_path(str(basefile)), "downloaded %s" % basefile) for basefile in range(1,9): util.writefile(repo.store.parsed_path(str(basefile)), "parsed %s" % basefile) for basefile in range(1,5): util.writefile(repo.store.generated_path(str(basefile)), "generated %s" % basefile) want = """ Status for document repository 'base' (ferenda.documentrepository.DocumentRepository) download: 12, 11, 10... (9 more) parse: 8, 7, 6... (5 more) Todo: 12, 11, 10... (1 more) generated: 4, 3, 2... (1 more) Todo: 8, 7, 6... (1 more) """.strip() with patch("builtins.print") as printmock: manager.status(repo) got = "\n".join([x[1][0] for x in printmock.mock_calls]) self.assertEqual(want,got)
def test_queryindex(self): res = [{ 'label': 'Doc #1', 'uri': 'http://example.org/doc1', 'text': 'matching doc 1' }, { 'label': 'Doc #2', 'uri': 'http://example.org/doc2', 'text': 'matching doc 2' }] pager = None config = { 'connect.return_value': Mock(**{'query.return_value': (res, pager)}) } printmock = MagicMock() with patch('ferenda.devel.FulltextIndex', **config): with patch('builtins.print', printmock): d = Devel() d.config = LayeredConfig( Defaults({ 'indextype': 'a', 'indexlocation': 'b' })) d.queryindex("doc") want = """ Doc #1 (http://example.org/doc1): matching doc 1 Doc #2 (http://example.org/doc2): matching doc 2 """.strip() got = "\n".join([x[1][0] for x in printmock.mock_calls]) self.maxDiff = None self.assertEqual(want, got)
def test_status(self): want = """ Status for document repository 'base' (ferenda.documentrepository.DocumentRepository) download: None. parse: None. generated: None. """.strip() repo = DocumentRepository(datadir=self.tempdir) with patch("builtins.print") as printmock: manager.status(repo) got = "\n".join([x[1][0] for x in printmock.mock_calls]) self.assertEqual(want, got) # test both status and get_status in one swoop. for basefile in range(1, 13): util.writefile(repo.store.downloaded_path(str(basefile)), "downloaded %s" % basefile) for basefile in range(1, 9): util.writefile(repo.store.parsed_path(str(basefile)), "parsed %s" % basefile) for basefile in range(1, 5): util.writefile(repo.store.generated_path(str(basefile)), "generated %s" % basefile) want = """ Status for document repository 'base' (ferenda.documentrepository.DocumentRepository) download: 12, 11, 10... (9 more) parse: 8, 7, 6... (5 more) Todo: 12, 11, 10... (1 more) generated: 4, 3, 2... (1 more) Todo: 8, 7, 6... (1 more) """.strip() with patch("builtins.print") as printmock: manager.status(repo) got = "\n".join([x[1][0] for x in printmock.mock_calls]) self.assertEqual(want, got)
def test_ifneeded_parse(self): @ifneeded("parse") def testfunc(repo, basefile): repo.called = True # mockdoc = Mock() # mockdoc.basefile="1234" mockbasefile = "1234" mockrepo = Mock() mockrepo.store.needed = DocumentStore(datadir='fake').needed mockrepo.called = False mockrepo.config.force = False # test 1: Outfile is newer - the ifneeded decorator should # make sure the actual testfunc code is never reached with patch('ferenda.util.outfile_is_newer', return_value=True): testfunc(mockrepo, mockbasefile) self.assertFalse(mockrepo.called) mockrepo.called = False # test 2: Outfile is older with patch('ferenda.util.outfile_is_newer', return_value=False): testfunc(mockrepo, mockbasefile) self.assertTrue(mockrepo.called) mockrepo.called = False # test 3: Outfile is newer, but the global force option was set mockrepo.config.force = True with patch('ferenda.util.outfile_is_newer', return_value=True): testfunc(mockrepo, mockbasefile) self.assertTrue(mockrepo.called) mockrepo.config.force = None mockrepo.called = False
def test_preflight(self): log = Mock() # test 1: python too old with patch('ferenda.manager.sys') as sysmock: sysmock.version_info = (2, 5, 6, 'final', 0) sysmock.version = sys.version self.assertFalse(manager._preflight_check(log, verbose=True)) self.assertTrue(log.error.called) log.error.reset_mock() # test 2: modules are old / or missing with patch('importlib.import_module') as importmock: setattr(importmock.return_value, '__version__', '0.0.1') self.assertFalse(manager._preflight_check(log, verbose=True)) self.assertTrue(log.error.called) log.error.reset_mock() importmock.side_effect = ImportError self.assertFalse(manager._preflight_check(log, verbose=True)) self.assertTrue(log.error.called) log.error.reset_mock() # test 3: binaries are nonexistent or errors with patch('ferenda.manager.subprocess.call') as callmock: callmock.return_value = 127 self.assertFalse(manager._preflight_check(log, verbose=True)) self.assertTrue(log.error.called) log.error.reset_mock() callmock.side_effect = OSError self.assertFalse(manager._preflight_check(log, verbose=True)) self.assertTrue(log.error.called) log.error.reset_mock()
def test_preflight(self): log = Mock() # test 1: python too old with patch('ferenda.manager.sys') as sysmock: sysmock.version_info = (2,5,6,'final',0) sysmock.version = sys.version self.assertFalse(manager._preflight_check(log, verbose=True)) self.assertTrue(log.error.called) log.error.reset_mock() # test 2: modules are old / or missing with patch('importlib.import_module') as importmock: setattr(importmock.return_value, '__version__', '0.0.1') self.assertFalse(manager._preflight_check(log, verbose=True)) self.assertTrue(log.error.called) log.error.reset_mock() importmock.side_effect = ImportError self.assertFalse(manager._preflight_check(log, verbose=True)) self.assertTrue(log.error.called) log.error.reset_mock() # test 3: binaries are nonexistent or errors with patch('ferenda.manager.subprocess.call') as callmock: callmock.return_value = 127 self.assertFalse(manager._preflight_check(log, verbose=True)) self.assertTrue(log.error.called) log.error.reset_mock() callmock.side_effect = OSError self.assertFalse(manager._preflight_check(log, verbose=True)) self.assertTrue(log.error.called) log.error.reset_mock()
def test_select(self): uri = "http://example.org/doc" with open("testselecttemplate.rq", "wb") as fp: fp.write("""PREFIX dcterms: <http://purl.org/dc/terms/> SELECT ?p ?o WHERE { <%(uri)s> ?p ?o . } """.encode()) result = """ [ { "p": "http://purl.org/dc/terms/title", "o": "Document title" }, { "p": "http://purl.org/dc/terms/identifier", "o": "Document ID" } ]""".lstrip().encode("utf-8") config = { 'connect.return_value': Mock(**{'select.return_value': result}) } printmock = MagicMock() with patch('ferenda.devel.TripleStore', **config): with patch('builtins.print', printmock): d = Devel() d.config = LayeredConfig( Defaults({ 'storetype': 'a', 'storelocation': 'b', 'storerepository': 'c' })) d.select("testselecttemplate.rq", uri) want = """ # Constructing the following from b, repository c, type a # PREFIX dcterms: <http://purl.org/dc/terms/> # # SELECT ?p ?o # WHERE { <http://example.org/doc> ?p ?o . } # [ { "p": "http://purl.org/dc/terms/title", "o": "Document title" }, { "p": "http://purl.org/dc/terms/identifier", "o": "Document ID" } ] # Selected in 0.001s """.strip() got = "\n".join([x[1][0] for x in printmock.mock_calls]) self.maxDiff = None self.assertEqual(self.mask_time(want), self.mask_time(got)) os.unlink("testselecttemplate.rq")
def test_runsetup(self): with patch('ferenda.manager.sys.exit') as mockexit: with patch('ferenda.manager.setup', return_value=True): manager.runsetup() self.assertFalse(mockexit.called) mockexit.reset_mock() with patch('ferenda.manager.setup', return_value=False): manager.runsetup() self.assertTrue(mockexit.called)
def test_render(self,mock_graph): @render def testfunc(repo,doc): pass mockdoc = Mock() mockrepo = Mock() mockrepo.store.parsed_path.return_value = "parsed_path.xhtml" with open("parsed_path.xhtml", "w") as fp: fp.write("""<?xml version="1.0" encoding="utf-8"?> <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML+RDFa 1.0//EN" "http://www.w3.org/MarkUp/DTD/xhtml-rdfa-1.dtd"> <html xmlns="http://www.w3.org/1999/xhtml" xmlns:dct="http://purl.org/dc/terms/"> <head about="http://example.org/doc"> <title property="dct:title">Document title</title> </head> <body> <h1>Hello!</h1> </body> </html>""") mockrepo.store.distilled_path.return_value = "distilled_path.xhtml" mockrepo.get_globals.return_value = {'symbol table':'fake'} mockdoc.meta = MagicMock() # need Magicmock which supports magic funcs like __iter__ bodypart = MagicMock() bodypart.meta = MagicMock() mockdoc.body = [bodypart] mockdoc.meta.__iter__.return_value = [] mockdoc.uri = "http://example.org/doc" with patch('ferenda.util.ensure_dir', return_value=True): testfunc(mockrepo, mockdoc) # 1 ensure that DocumentRepository.render_xhtml is called with # four arguments mockrepo.render_xhtml.assert_called_with(mockdoc, "parsed_path.xhtml") # 2 ensure that DocumentRepository.create_external_resources # is called with 1 argument mockrepo.create_external_resources.assert_called_with(mockdoc) # 3 ensure that a Graph object is created, its parse and # serialize methods called # FIXME: Why doesn't the patching work?! # self.assertTrue(mock_graph().parse.called) # self.assertTrue(mock_graph().serialize.called) # (4. ensure that a warning gets printed if doc.meta and # distilled_graph do not agree) mock_graph().__iter__.return_value = ['a','b'] mockdoc.meta.__iter__.return_value = ['a','b','c'] mockdoc.meta.serialize.return_value = b"<c>" with patch('ferenda.util.ensure_dir', return_value=True): testfunc(mockrepo, mockdoc) self.assertTrue(mockrepo.log.warning.called) os.remove("parsed_path.xhtml") os.remove("distilled_path.xhtml")
def test_run_single_errors(self): self._enable_repos() argv = ["test", "errmethod", "--all"] with patch('ferenda.manager.setup_logger'): with patch('builtins.print') as printmock: res = manager.run(argv) self.assertEqual(res[0][0], Exception) self.assertEqual(res[1][0], errors.DocumentRemovedError) self.assertEqual(res[2], None) self.assertTrue(os.path.exists("dummyfile.txt"))
def test_select(self): uri = "http://example.org/doc" with open("testselecttemplate.rq", "wb") as fp: fp.write( """PREFIX dct: <http://purl.org/dc/terms/> SELECT ?p ?o WHERE { <%(uri)s> ?p ?o . } """.encode() ) result = """ [ { "p": "http://purl.org/dc/terms/title", "o": "Document title" }, { "p": "http://purl.org/dc/terms/identifier", "o": "Document ID" } ]""".lstrip().encode( "utf-8" ) config = {"connect.return_value": Mock(**{"select.return_value": result})} printmock = MagicMock() with patch("ferenda.devel.TripleStore", **config): with patch(builtins + ".print", printmock): d = Devel() d.config = LayeredConfig({"storetype": "a", "storelocation": "b", "storerepository": "c"}) d.select("testselecttemplate.rq", uri) want = """ # Constructing the following from b, repository c, type a # PREFIX dct: <http://purl.org/dc/terms/> # # SELECT ?p ?o # WHERE { <http://example.org/doc> ?p ?o . } # [ { "p": "http://purl.org/dc/terms/title", "o": "Document title" }, { "p": "http://purl.org/dc/terms/identifier", "o": "Document ID" } ] # Selected in 0.001s """.strip() got = "\n".join([x[1][0] for x in printmock.mock_calls]) self.maxDiff = None self.assertEqual(self.mask_time(want), self.mask_time(got)) os.unlink("testselecttemplate.rq")
def test_dumpstore(self): d = Devel() d.config = Mock() # only test that Triplestore is called correctly, mock any # calls to any real database config = {"connect.return_value": Mock(**{"get_serialized.return_value": b"[fake store content]"})} printmock = MagicMock() with patch("ferenda.devel.TripleStore", **config): with patch(builtins + ".print", printmock): d.dumpstore(format="trix") want = "[fake store content]" printmock.assert_has_calls([call(want)])
def test_construct(self): uri = "http://example.org/doc" with open("testconstructtemplate.rq", "wb") as fp: fp.write("""PREFIX dcterms: <http://purl.org/dc/terms/> CONSTRUCT { ?s ?p ?o . } WHERE { ?s ?p ?o . <%(uri)s> ?p ?o . } """.encode()) g = Graph() g.bind("dcterms", str(DCTERMS)) g.add((URIRef(uri), DCTERMS.title, Literal("Document title"))) config = { 'connect.return_value': Mock(**{'construct.return_value': g}) } printmock = MagicMock() with patch('ferenda.devel.TripleStore', **config): with patch('builtins.print', printmock): d = Devel() d.config = LayeredConfig( Defaults({ 'storetype': 'a', 'storelocation': 'b', 'storerepository': 'c' })) d.construct("testconstructtemplate.rq", uri) want = """ # Constructing the following from b, repository c, type a # PREFIX dcterms: <http://purl.org/dc/terms/> # # CONSTRUCT { ?s ?p ?o . } # WHERE { ?s ?p ?o . # <http://example.org/doc> ?p ?o . } # @prefix dcterms: <http://purl.org/dc/terms/> . @prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> . @prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> . @prefix xml: <http://www.w3.org/XML/1998/namespace> . @prefix xsd: <http://www.w3.org/2001/XMLSchema#> . <http://example.org/doc> dcterms:title "Document title" . # 1 triples constructed in 0.001s """.strip() got = "\n".join([x[1][0] for x in printmock.mock_calls]) self.maxDiff = None self.assertEqual(self.mask_time(want), self.mask_time(got)) os.unlink("testconstructtemplate.rq")
def test_construct(self): uri = "http://example.org/doc" with open("testconstructtemplate.rq", "wb") as fp: fp.write("""PREFIX dcterms: <http://purl.org/dc/terms/> CONSTRUCT { ?s ?p ?o . } WHERE { ?s ?p ?o . <%(uri)s> ?p ?o . } """.encode()) g = Graph() g.bind("dcterms", str(DCTERMS)) g.add((URIRef(uri), DCTERMS.title, Literal("Document title"))) config = {'connect.return_value': Mock(**{'construct.return_value': g})} printmock = MagicMock() with patch('ferenda.devel.TripleStore', **config): with patch('builtins.print', printmock): d = Devel() d.config = LayeredConfig(Defaults({'storetype': 'a', 'storelocation': 'b', 'storerepository': 'c'})) d.construct("testconstructtemplate.rq", uri) want = """ # Constructing the following from b, repository c, type a # PREFIX dcterms: <http://purl.org/dc/terms/> # # CONSTRUCT { ?s ?p ?o . } # WHERE { ?s ?p ?o . # <http://example.org/doc> ?p ?o . } # @prefix dcterms: <http://purl.org/dc/terms/> . @prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> . @prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> . @prefix xml: <http://www.w3.org/XML/1998/namespace> . @prefix xsd: <http://www.w3.org/2001/XMLSchema#> . <http://example.org/doc> dcterms:title "Document title" . # 1 triples constructed in 0.001s """.strip() got = "\n".join([x[1][0] for x in printmock.mock_calls]) self.maxDiff = None self.assertEqual(self.mask_time(want), self.mask_time(got)) os.unlink("testconstructtemplate.rq")
def test_dumpstore(self): d = Devel() d.config = Mock() # only test that Triplestore is called correctly, mock any # calls to any real database config = { 'connect.return_value': Mock(**{'get_serialized.return_value': b'[fake store content]'}) } printmock = MagicMock() with patch('ferenda.devel.TripleStore', **config): with patch('builtins.print', printmock): d.dumpstore(format="trix") want = "[fake store content]" printmock.assert_has_calls([call(want)])
def test_runserver(self): self._enable_repos() m = Mock() with patch('ferenda.manager.make_server', return_value=m) as m2: manager.run(["all", "runserver"]) self.assertTrue(m2.called) self.assertTrue(m.serve_forever.called)
def test_parameters_legacy(self): # legacy api res = ([], {'firstresult': 1, 'totalresults': 0}) # FIXME: we leave out free=true (should map to schema_free=True) self.env['QUERY_STRING'] = "type=Standard&title=Hello+World&issued=2014-06-30&schema_free=true" self.app.config.legacyapi = True config = {'connect.return_value': Mock(**{'query.return_value': res, 'schema.return_value': {'dcterms_issued': fulltextindex.Datetime(), 'schema_free': fulltextindex.Boolean(), 'dcterms_title': None, 'rdf_type': None}})} want = {'q': None, 'dcterms_title': "Hello World", 'dcterms_issued': datetime.datetime(2014,6,30,0,0,0), 'schema_free': True, 'rdf_type': '*Standard', # should be bibo:Standard or even http://purl.org/ontology/bibo/Standard, but requires proper context handling to work 'pagenum': 1, 'pagelen': 10, 'ac_query': False, 'boost_types': None, 'exclude_types': None} with patch('ferenda.wsgiapp.FulltextIndex', **config): status, headers, content = self.call_wsgi(self.env) config['connect.return_value'].query.assert_called_once_with(**want)
def test_parameters_legacy(self): # legacy api res = ([], {'firstresult': 1, 'totalresults': 0}) # FIXME: we leave out free=true (should map to schema_free=True) self.builder.query_string = "type=Standard&title=Hello+World&issued=2014-06-30&schema_free=true" self.app.config.legacyapi = True config = {'connect.return_value': Mock(**{'query.return_value': res, 'schema.return_value': {'dcterms_issued': fulltextindex.Datetime(), 'schema_free': fulltextindex.Boolean(), 'dcterms_title': None, 'rdf_type': None}})} want = {'q': None, 'dcterms_title': "Hello World", 'dcterms_issued': datetime.datetime(2014,6,30,0,0,0), 'schema_free': True, 'rdf_type': '*Standard', # should be bibo:Standard or even http://purl.org/ontology/bibo/Standard, but requires proper context handling to work 'pagenum': 1, 'pagelen': 10, 'ac_query': False, 'boost_repos': None, 'exclude_repos': None, 'include_fragments': None} with patch('ferenda.wsgiapp.FulltextIndex', **config): status, headers, content = self.call_wsgi() config['connect.return_value'].query.assert_called_once_with(**want)
def test_highlighted_snippet(self): res = ([{'title':'Example', 'uri':'http://example.org/base/123/b1', 'text':html.P(['sollicitudin justo ', html.Strong(['needle'], **{'class':'match'}), ' tempor ut eu enim ... himenaeos. ', html.Strong(['Needle'], **{'class':'match'}), ' id tincidunt orci'])}], {'pagenum': 1, 'pagecount': 1, 'firstresult': 1, 'lastresult': 1, 'totalresults': 1}) self.builder.query_string = "q=needle" config = {'connect.return_value': Mock(**{'query.return_value': res})} with patch('ferenda.wsgiapp.FulltextIndex', **config): status, headers, content = self.call_wsgi() self.assertResponse("200 OK", {'Content-Type': 'text/html; charset=utf-8'}, None, status, headers, None) t = etree.fromstring(content) docs = t.findall(".//section[@class='hit']") self.assertEqualXML(res[0][0]['text'].as_xhtml(), docs[0][1], namespace_aware=False)
def test_parameters(self): # normal api res = ([], {'firstresult': 1, 'totalresults': 0}) self.env[ 'QUERY_STRING'] = "rdf_type=bibo:Standard&dcterms_title=Hello+World&dcterms_issued=2014-06-30&schema_free=true" config = { 'connect.return_value': Mock( **{ 'query.return_value': res, 'schema.return_value': { 'dcterms_issued': fulltextindex.Datetime(), 'schema_free': fulltextindex.Boolean() } }) } want = { 'q': None, 'dcterms_title': "Hello World", 'dcterms_issued': datetime.datetime(2014, 6, 30, 0, 0, 0), 'schema_free': True, 'rdf_type': 'bibo:Standard', # FIXME: should be http://purl.org/ontology/bibo/Standard -- but requires that self.repos in wsgiapp is set up 'pagenum': 1, 'pagelen': 10, 'ac_query': False, 'boost_types': None, 'exclude_types': None } with patch('ferenda.wsgiapp.FulltextIndex', **config): status, headers, content = self.call_wsgi(self.env) config['connect.return_value'].query.assert_called_once_with( **want)
def test_run_single(self): # test1: run standard (custom) method self._enable_repos() argv = ["test", "mymethod", "myarg"] self.assertEqual(manager.run(argv), "ok!") # test2: specify invalid alias argv[0] = "invalid" with patch('ferenda.manager.setup_logger'): self.assertEqual(manager.run(argv), None) with patch('builtins.print') as printmock: with patch('ferenda.manager.setup_logger'): # test3: specify invalid method argv = ["test", "invalid"] self.assertEqual(manager.run(argv), None)
def silence(): """The same functionality as quiet(), but as a context manager so that one can use "with silence:" constructs.""" state = _setup() with patch('builtins.print') as printmock: yield _restore(state)
def test_highlighted_snippet(self): res = ([{'title':'Example', 'uri':'http://example.org/base/123/b1', 'text':html.P(['sollicitudin justo ', html.Strong(['needle'], **{'class':'match'}), ' tempor ut eu enim ... himenaeos. ', html.Strong(['Needle'], **{'class':'match'}), ' id tincidunt orci'])}], {'pagenum': 1, 'pagecount': 1, 'firstresult': 1, 'lastresult': 1, 'totalresults': 1}) self.env['QUERY_STRING'] = "q=needle" config = {'connect.return_value': Mock(**{'query.return_value': res})} with patch('ferenda.manager.FulltextIndex', **config): status, headers, content = self.call_wsgi(self.env) self.assertResponse("200 OK", {'Content-Type': 'text/html; charset=utf-8'}, None, status, headers, None) t = etree.fromstring(content) docs = t.findall(".//section[@class='hit']") self.assertEqualXML(res[0][0]['text'].as_xhtml(), docs[0][1], namespace_aware=False)
def test_run_single(self): # test1: run standard (custom) method self._enable_repos() argv = ["test","mymethod","myarg"] self.assertEqual(manager.run(argv), "ok!") # test2: specify invalid alias argv[0] = "invalid" with patch('ferenda.manager.setup_logger'): self.assertEqual(manager.run(argv), None) with patch('builtins.print') as printmock: with patch('ferenda.manager.setup_logger'): # test3: specify invalid method argv = ["test", "invalid"] self.assertEqual(manager.run(argv), None)
def test_select_triplestore(self): log = Mock() # first manipulate requests.get to give the impression that # fuseki or sesame either is or isn't available with patch('ferenda.manager.requests.get') as mock_get: r = manager._select_triplestore("sitename", log, verbose=True) self.assertEqual("FUSEKI", r[0]) mock_get.side_effect = requests.exceptions.HTTPError r = manager._select_triplestore("sitename", log, verbose=True) self.assertNotEqual("FUSEKI", r[0]) def get_sesame(url): if not 'openrdf-sesame' in url: raise requests.exceptions.HTTPError resp = Mock() resp.text = "ok" return resp mock_get.side_effect = get_sesame r = manager._select_triplestore("sitename", log, verbose=True) self.assertEqual("SESAME", r[0]) mock_get.side_effect = requests.exceptions.HTTPError r = manager._select_triplestore("sitename", log, verbose=True) self.assertNotEqual("SESAME", r[0]) # all request.get calls still raises HTTP error with patch('ferenda.manager.TripleStore.connect') as mock_connect: r = manager._select_triplestore("sitename", log, verbose=True) self.assertEqual("SQLITE", r[0]) def connectfail(storetype, location, repository): if storetype == "SQLITE": raise ImportError("BOOM") mock_connect.side_effect = connectfail r = manager._select_triplestore("sitename", log, verbose=True) self.assertNotEqual("SQLITE", r[0]) r = manager._select_triplestore("sitename", log, verbose=True) self.assertEqual("SLEEPYCAT", r[0]) mock_connect.side_effect = ImportError r = manager._select_triplestore("sitename", log, verbose=True) self.assertEqual(None, r[0])
def test_ensure_dir(self): self.assertFalse(os.path.exists(self.dname)) util.ensure_dir(self.fname) self.assertTrue(os.path.exists(self.dname)) self.assertTrue(os.path.isdir(self.dname)) util.ensure_dir(self.fname) os.rmdir(self.dname) with patch('ferenda.util.mkdir', side_effect=OSError): util.ensure_dir(self.fname)
def silence(): """The same functionality as quiet(), but as a context manager so that one can use "with silence:" constructs.""" state = _setup() try: with patch('builtins.print') as printmock: yield finally: _restore(state)
def test_run_single_all(self): self._enable_repos() argv = ["test","mymethod","--all"] # Test 1: make sure that if setup signals that no work should # be done, this is respected with patch("example.Testrepo.setup", return_value=False): self.assertEqual(manager.run(list(argv)), []) # pass # Test 2: but if not, do the work self.assertEqual(manager.run(list(argv)), [None, "ok!", None])
def test_run_single_all(self): self._enable_repos() argv = ["test", "mymethod", "--all"] # Test 1: make sure that if setup signals that no work should # be done, this is respected with patch("example.Testrepo.setup", return_value=False): self.assertEqual(manager.run(list(argv)), []) # pass # Test 2: but if not, do the work self.assertEqual(manager.run(list(argv)), [None, "ok!", None])
def test_queryindex(self): res = [ {"identifier": "Doc #1", "about": "http://example.org/doc1", "text": "matching doc 1"}, {"identifier": "Doc #2", "about": "http://example.org/doc2", "text": "matching doc 2"}, ] config = {"connect.return_value": Mock(**{"query.return_value": res})} printmock = MagicMock() with patch("ferenda.devel.FulltextIndex", **config): with patch(builtins + ".print", printmock): d = Devel() d.config = LayeredConfig({"indextype": "a", "indexlocation": "b"}) d.queryindex("doc") want = """ Doc #1 (http://example.org/doc1): matching doc 1 Doc #2 (http://example.org/doc2): matching doc 2 """.strip() got = "\n".join([x[1][0] for x in printmock.mock_calls]) self.maxDiff = None self.assertEqual(want, got)
def test_select_fulltextindex(self): log = Mock() # first manipulate requests.get to give the impression that # elasticsearch either is or isn't available with patch('ferenda.manager.requests.get') as mock_get: r = manager._select_fulltextindex(log, "mysite", verbose=True) self.assertEqual("ELASTICSEARCH", r[0]) self.assertEqual("http://localhost:9200/mysite/", r[1]) mock_get.side_effect = requests.exceptions.HTTPError r = manager._select_fulltextindex(log, "mysite", verbose=True) self.assertEqual("WHOOSH", r[0])
def test_recordlastdownload(self): @recordlastdownload def testfunc(repo): pass mockrepo = Mock() with patch('ferenda.decorators.LayeredConfig.write') as mockconf: testfunc(mockrepo) # check that config.lastdownload has been set to a datetime self.assertIsInstance(mockrepo.config.lastdownload, datetime.datetime) # and that LayeredConfig.write has been called self.assertTrue(mockconf.called)
def test_distill_setfile(self): os.mkdir(self.datadir+"/downloaded") util.writefile(self.datadir+"/downloaded/a.html", "<p>This is doc A</p>") util.writefile(self.datadir+"/distilled/a.ttl", "") os.environ["FERENDA_SET_TESTFILE"] = "1" with patch("builtins.print") as printmock: self._runtest() del os.environ["FERENDA_SET_TESTFILE"] self.assertEqual(self.expected_ttl, util.readfile(self.datadir+"/distilled/a.ttl")) pass
def test_distill_setfile(self): os.mkdir(self.datadir + "/downloaded") util.writefile(self.datadir + "/downloaded/a.html", "<p>This is doc A</p>") util.writefile(self.datadir + "/distilled/a.ttl", "") os.environ["FERENDA_SET_TESTFILE"] = "1" with patch("builtins.print") as printmock: self._runtest() del os.environ["FERENDA_SET_TESTFILE"] self.assertEqual(self.expected_ttl, util.readfile(self.datadir + "/distilled/a.ttl")) pass
def download_test(self, specfile): def my_get(url, **kwargs): urlspec = spec[url] if isinstance(urlspec, str): urlspec = {'file': urlspec} if 'charset' not in urlspec: urlspec['charset'] = 'utf-8' url_location = os.path.join(os.path.dirname(specfile), urlspec['file']) res = Mock() # load the .content property with open(url_location, "rb") as fp: res.content = fp.read() # but only load .text if a charset is present (note # default value of 'utf-8' above -- set 'charset': null in # the json file for binary files if urlspec['charset']: with codecs.open(url_location, "r", encoding=urlspec['charset']) as fp: res.text = fp.read() # FIXME: Using a defaultdict ensures that we'll never trip # over the non-existance of certain headers. WE should # specify only the most basic headers to make sure calling # code doesn't rely on eg. the etag header always being # there, because it won't res.headers = collections.defaultdict(lambda: None) res.headers['X-These-Headers-Are'] = 'Faked' res.status_code = 200 return res with codecs.open(specfile, encoding="utf-8") as fp: spec = json.load(fp) with patch('requests.get', side_effect=my_get): self.repo.download() # organize a temporary copy of files that we can compare our results to wantdir = "%s/%s-want" % (self.datadir, self.repoclass.alias) expected = False for url in spec: if "expect" in spec[url]: expected = True sourcefile = os.path.join(os.path.dirname(specfile), spec[url]['file']) wantfile = "%s/%s" % (wantdir, spec[url]['expect']) util.copy_if_different(sourcefile, wantfile) if expected: self.assertEqualDirs(wantdir, "%s/%s" % (self.datadir, self.repoclass.alias)) else: self.fail('No files were marked as "expect" in specfile %s' % specfile)
def test_handleerror(self): @handleerror def testfunc(repo, doc): if doc.exception: raise doc.exception else: return True mockrepo = Mock() mockrepo.config = MagicMock() # must support __in__() mockdoc = Mock() # 1. should not raise an exception (but should call log.info # and util.robust_remove, and return false) with patch('ferenda.util.robust_remove') as robust_remove: mockdoc.exception = DocumentRemovedError self.assertFalse(testfunc(mockrepo, mockdoc)) self.assertTrue(mockrepo.log.info.called) self.assertTrue(robust_remove.called) # 2. should raise the same exception mockdoc.exception = KeyboardInterrupt with self.assertRaises(KeyboardInterrupt): testfunc(mockrepo, mockdoc) # 3.1 Should raise the same exeption mockdoc.exception = ParseError mockrepo.config.fatalexceptions = True with self.assertRaises(ParseError): testfunc(mockrepo, mockdoc) mockrepo.config.fatalexceptions = None # 3.2 Should not raise an exception (but should call log.error and return false) mockdoc.exception = ParseError self.assertFalse(testfunc(mockrepo, mockdoc)) self.assertTrue(mockrepo.log.error.called) # 4.1 Should raise the same exception mockdoc.exception = Exception mockrepo.config.fatalexceptions = True with self.assertRaises(Exception): testfunc(mockrepo, mockdoc) mockrepo.config.fatalexceptions = None # 4.2 Should not raise an exception mockdoc.exception = Exception self.assertFalse(testfunc(mockrepo, mockdoc)) self.assertTrue(mockrepo.log.error.called) # 5. No exceptions - everything should go fine mockdoc.exception = None self.assertTrue(testfunc(mockrepo, mockdoc))
def test_queryindex(self): res = [{'label': 'Doc #1', 'uri': 'http://example.org/doc1', 'text': 'matching doc 1'}, {'label': 'Doc #2', 'uri': 'http://example.org/doc2', 'text': 'matching doc 2'}] pager = None config = {'connect.return_value': Mock(**{'query.return_value': (res, pager)})} printmock = MagicMock() with patch('ferenda.devel.FulltextIndex', **config): with patch('builtins.print', printmock): d = Devel() d.config = LayeredConfig(Defaults({'indextype': 'a', 'indexlocation': 'b'})) d.queryindex("doc") want = """ Doc #1 (http://example.org/doc1): matching doc 1 Doc #2 (http://example.org/doc2): matching doc 2 """.strip() got = "\n".join([x[1][0] for x in printmock.mock_calls]) self.maxDiff = None self.assertEqual(want, got)
def test_ifneeded_generate(self): @ifneeded("generate") def testfunc(repo, basefile): repo.called = True mockbasefile = "1234" mockrepo = Mock() mockrepo.store.needed = DocumentStore(datadir='fake').needed mockrepo.called = False mockrepo.config.force = False # test 1: Outfile is newer - the ifneeded decorator should # make sure the actual testfunc code is never reached with patch('ferenda.util.outfile_is_newer', return_value=True): testfunc(mockrepo, mockbasefile) self.assertFalse(mockrepo.called) mockrepo.called = False # test 2: Outfile is older than source file with patch('ferenda.util.outfile_is_newer', return_value=False): testfunc(mockrepo, mockbasefile) self.assertTrue(mockrepo.called) mockrepo.called = False # FIXME: we could add more tests, eg create a dependency file # and make sure an arbitrary file named in that depfile is # newer then outfile. but the tests in testDocStore.Needed # should cover that pretty well. # test 3: Outfile is newer, but the global force option was set mockrepo.config.force = True with patch('ferenda.util.outfile_is_newer', return_value=True): testfunc(mockrepo, mockbasefile) self.assertTrue(mockrepo.called) mockrepo.config.force = None mockrepo.called = False
def test_parseifneeded(self): @parseifneeded def testfunc(repo,doc): repo.called = True mockdoc = Mock() mockrepo = Mock() mockrepo.called = False mockrepo.config.force = False # test 1: Outfile is newer - the parseifneeded decorator # should make sure the actual testfunc code is never reached with patch('ferenda.util.outfile_is_newer', return_value=True): testfunc(mockrepo,mockdoc) self.assertFalse(mockrepo.called) mockrepo.called = False # test 2: Outfile is older with patch('ferenda.util.outfile_is_newer', return_value=False): testfunc(mockrepo,mockdoc) self.assertTrue(mockrepo.called) mockrepo.called = False # test 3: Outfile is newer, but the global force option was set mockrepo.config.force = True with patch('ferenda.util.outfile_is_newer', return_value=True): testfunc(mockrepo,mockdoc) self.assertTrue(mockrepo.called) mockrepo.config.force = None mockrepo.called = False # test 4: Outfile is newer, but the module parseforce option was set mockrepo.config.parseforce = True with patch('ferenda.util.outfile_is_newer', return_value=True): testfunc(mockrepo,mockdoc) self.assertTrue(mockrepo.called) mockrepo.called = False
def test_parse_setfile(self): os.mkdir(self.datadir + "/downloaded") util.writefile(self.datadir + "/downloaded/a.html", "<p>This is doc A</p>") util.writefile(self.datadir + "/parsed/a.xhtml", "") os.environ["FERENDA_SET_TESTFILE"] = "1" with patch("builtins.print") as printmock: self._runtest() output = printmock.mock_calls[0][1][0] output = re.sub("'[^']*'", "''", output, 1) self.assertEqual("Overwriting '' with result of parse ('a')", output) del os.environ["FERENDA_SET_TESTFILE"] self.assertEqualXML(self.expected_xhtml, util.readfile(self.datadir + "/parsed/a.xhtml")) pass
def test_parse_setfile(self): os.mkdir(self.datadir+"/downloaded") util.writefile(self.datadir+"/downloaded/a.html", "<p>This is doc A</p>") util.writefile(self.datadir+"/parsed/a.xhtml", "") os.environ["FERENDA_SET_TESTFILE"] = "1" with patch("builtins.print") as printmock: self._runtest() output = printmock.mock_calls[0][1][0] output = re.sub("'[^']*'", "''", output, 1) self.assertEqual("Overwriting '' with result of parse ('a')", output) del os.environ["FERENDA_SET_TESTFILE"] self.assertEqualXML(self.expected_xhtml, util.readfile(self.datadir+"/parsed/a.xhtml")) pass
def test_faceted_data(self): canned = [ { "uri": "http://example.org/books/A_Tale_of_Two_Cities", "dcterms_title": "A Tale of Two Cities" }, { "uri": "http://example.org/books/The_Lord_of_the_Rings", "dcterms_title": "The Lord of the Rings" }, ] with patch('ferenda.DocumentRepository.facet_select', return_value=canned) as mock: faceted_data = self.repo.faceted_data() self.assertEqual(faceted_data, canned) self.assertTrue( os.path.exists(self.datadir + "/base/toc/faceted_data.json")) # on second run, faceted_data should be read from the cache # (if outfile_is_newer is called, we're far enough down in # that branch to know that the cache file is used if # outfile_is_newer returns True) with patch('ferenda.util.outfile_is_newer', return_value=True): faceted_data = self.repo.faceted_data() self.assertEqual(faceted_data, canned)
def test_print_usage(self): self._enable_repos() with patch('builtins.print') as printmock: manager.run([]) executable = sys.argv[0] got = "\n".join([x[1][0] for x in printmock.mock_calls]) got = got.replace(executable, "[EXEC]") want = """Usage: [EXEC] [class-or-alias] [action] <arguments> <options> e.g. '[EXEC] ferenda.sources.EurlexCaselaw enable' '[EXEC] ecj parse 62008J0042' '[EXEC] all generate' Available modules: * test: [Undocumented] * test2: [Undocumented]""" self.assertEqual(got, want)
def test_search_single(self): self.env['QUERY_STRING'] = "q=subsection" res = ([{'title': 'Result #1', 'uri': 'http://example.org', 'text': ['Text that contains the subsection term']}], {'pagenum': 1, 'pagecount': 1, 'firstresult': 1, 'lastresult': 1, 'totalresults': 1}) config = {'connect.return_value': Mock(**{'query.return_value': res})} with patch('ferenda.manager.FulltextIndex', **config): status, headers, content = self.call_wsgi(self.env) t = etree.fromstring(content) resulthead = t.find(".//article/h1").text self.assertEqual(resulthead, "1 match for 'subsection'")
def test_search_single(self): self.builder.query_string = "q=subsection" res = ([{'dcterms_title': 'Result #1', 'uri': 'http://example.org', 'text': 'Text that contains the subsection term'}], {'pagenum': 1, 'pagecount': 1, 'firstresult': 1, 'lastresult': 1, 'totalresults': 1}) config = {'connect.return_value': Mock(**{'query.return_value': res})} with patch('ferenda.wsgiapp.FulltextIndex', **config): status, headers, content = self.call_wsgi() t = etree.fromstring(content) resulthead = t.find(".//article/h1").text self.assertEqual("1 match for 'subsection'", resulthead)
def test_dumprdf(self): fileno, tmpfile = mkstemp() fp = os.fdopen(fileno, "w") fp.write("""<html xmlns="http://www.w3.org/1999/xhtml"> <head about="http://example.org/doc"> <title property="http://purl.org/dc/terms/">Doc title</title> </head> <body>...</body> </html>""") fp.close() d = Devel() mock = MagicMock() with patch('builtins.print', mock): d.dumprdf(tmpfile, format="nt") os.unlink(tmpfile) self.assertTrue(mock.called) want = '<http://example.org/doc> <http://purl.org/dc/terms/> "Doc title" .\n\n' mock.assert_has_calls([call(want)])
def test_setup(self, mockprint): # restart the log system since setup() will do that otherwise manager.shutdown_logger() manager.setup_logger('CRITICAL') projdir = self.datadir + os.sep + 'myproject' argv = ['ferenda-build.py', projdir] # test1: normal, setup succeeds res = manager.setup(force=True, verbose=False, unattended=True, argv=argv) self.assertTrue(res) self.assertTrue(os.path.exists(projdir)) # test2: directory exists, setup fails res = manager.setup(verbose=False, unattended=True, argv=argv) self.assertFalse(res) shutil.rmtree(projdir) # test2: no argv, rely on sys.argv, assert False with patch('ferenda.manager.sys.argv'): self.assertFalse(manager.setup()) self.assertFalse(os.path.exists(projdir)) # test3: preflight fails with patch('ferenda.manager._preflight_check', return_value=False): self.assertFalse(manager.setup(unattended=True, argv=argv)) self.assertFalse(os.path.exists(projdir)) with patch('ferenda.manager.input', return_value="n") as input_mock: self.assertFalse(manager.setup(unattended=False, argv=argv)) self.assertFalse(os.path.exists(projdir)) self.assertTrue(input_mock.called) # test4: select_triplestore fails with patch('ferenda.manager._preflight_check', return_value=True): with patch('ferenda.manager._select_triplestore', return_value=(False, None, None)): self.assertFalse(manager.setup(unattended=True, argv=argv)) self.assertFalse(os.path.exists(projdir)) with patch('ferenda.manager.input', return_value="n") as input_mock: self.assertFalse(manager.setup(unattended=False, argv=argv)) self.assertFalse(os.path.exists(projdir)) self.assertTrue(input_mock.called)