Esempio n. 1
0
    def test_toc(self):
        # tests the main TOC method, not the helper methods (they are
        # tested separately)
        self.repo.facets = MagicMock()
        self.repo.facet_select = MagicMock()
        self.repo.facet_query = MagicMock()
        self.repo.faceted_data = MagicMock()
        self.repo.log = Mock()
        self.repo.toc_pagesets = Mock()
        self.repo.toc_select_for_pages = Mock()
        self.repo.toc_generate_pages = Mock()
        self.repo.toc_generate_first_page = Mock()
        with patch('json.dump'):
            self.repo.toc()

        # assert facet_query was properly called, error and info msg
        # was printed
        self.assertEqual("http://localhost:8000/dataset/base",
                         self.repo.facet_query.call_args[0][0])
        self.assertTrue(self.repo.log.error.called)
        self.assertTrue(self.repo.log.info.called)
        # and that the rest of the methods were NOT called
        self.assertFalse(self.repo.toc_pagesets.called)
        self.assertFalse(self.repo.toc_select_for_pages.called)
        self.assertFalse(self.repo.toc_generate_pages.called)

        # test2: facet_select returns something
        self.repo.faceted_data.return_value = ["fake", "data"]
        with patch('json.load'):
            self.repo.toc()
        # Now all other methods should be called
        self.assertTrue(self.repo.toc_pagesets.called)
        self.assertTrue(self.repo.toc_select_for_pages.called)
        self.assertTrue(self.repo.toc_generate_pages.called)
Esempio n. 2
0
    def test_status(self):
        want  = """
Status for document repository 'base' (ferenda.documentrepository.DocumentRepository)
 download: None.
 parse: None.
 generated: None.
""".strip()
        repo = DocumentRepository(datadir=self.tempdir)
        with patch("builtins.print") as printmock:
            manager.status(repo)
        got = "\n".join([x[1][0] for x in printmock.mock_calls])
        self.assertEqual(want,got)

        # test both status and get_status in one swoop.
        for basefile in range(1,13):
            util.writefile(repo.store.downloaded_path(str(basefile)),
                           "downloaded %s" % basefile)
        for basefile in range(1,9):
            util.writefile(repo.store.parsed_path(str(basefile)),
                           "parsed %s" % basefile)
        for basefile in range(1,5):
            util.writefile(repo.store.generated_path(str(basefile)),
                           "generated %s" % basefile)

        want  = """
Status for document repository 'base' (ferenda.documentrepository.DocumentRepository)
 download: 12, 11, 10... (9 more)
 parse: 8, 7, 6... (5 more) Todo: 12, 11, 10... (1 more)
 generated: 4, 3, 2... (1 more) Todo: 8, 7, 6... (1 more)
""".strip()
        with patch("builtins.print") as printmock:
            manager.status(repo)
        got = "\n".join([x[1][0] for x in printmock.mock_calls])
        self.assertEqual(want,got)
Esempio n. 3
0
    def test_queryindex(self):
        res = [{
            'label': 'Doc #1',
            'uri': 'http://example.org/doc1',
            'text': 'matching doc 1'
        }, {
            'label': 'Doc #2',
            'uri': 'http://example.org/doc2',
            'text': 'matching doc 2'
        }]
        pager = None
        config = {
            'connect.return_value':
            Mock(**{'query.return_value': (res, pager)})
        }
        printmock = MagicMock()
        with patch('ferenda.devel.FulltextIndex', **config):
            with patch('builtins.print', printmock):
                d = Devel()
                d.config = LayeredConfig(
                    Defaults({
                        'indextype': 'a',
                        'indexlocation': 'b'
                    }))
                d.queryindex("doc")
        want = """
Doc #1 (http://example.org/doc1): matching doc 1
Doc #2 (http://example.org/doc2): matching doc 2
""".strip()
        got = "\n".join([x[1][0] for x in printmock.mock_calls])
        self.maxDiff = None
        self.assertEqual(want, got)
Esempio n. 4
0
    def test_status(self):
        want = """
Status for document repository 'base' (ferenda.documentrepository.DocumentRepository)
 download: None.
 parse: None.
 generated: None.
""".strip()
        repo = DocumentRepository(datadir=self.tempdir)
        with patch("builtins.print") as printmock:
            manager.status(repo)
        got = "\n".join([x[1][0] for x in printmock.mock_calls])
        self.assertEqual(want, got)

        # test both status and get_status in one swoop.
        for basefile in range(1, 13):
            util.writefile(repo.store.downloaded_path(str(basefile)),
                           "downloaded %s" % basefile)
        for basefile in range(1, 9):
            util.writefile(repo.store.parsed_path(str(basefile)),
                           "parsed %s" % basefile)
        for basefile in range(1, 5):
            util.writefile(repo.store.generated_path(str(basefile)),
                           "generated %s" % basefile)

        want = """
Status for document repository 'base' (ferenda.documentrepository.DocumentRepository)
 download: 12, 11, 10... (9 more)
 parse: 8, 7, 6... (5 more) Todo: 12, 11, 10... (1 more)
 generated: 4, 3, 2... (1 more) Todo: 8, 7, 6... (1 more)
""".strip()
        with patch("builtins.print") as printmock:
            manager.status(repo)
        got = "\n".join([x[1][0] for x in printmock.mock_calls])
        self.assertEqual(want, got)
Esempio n. 5
0
    def test_toc(self):
        # tests the main TOC method, not the helper methods (they are
        # tested separately)
        self.repo.facets = MagicMock()
        self.repo.facet_select = MagicMock()
        self.repo.facet_query = MagicMock()
        self.repo.faceted_data = MagicMock()
        self.repo.log = Mock()
        self.repo.toc_pagesets = Mock()
        self.repo.toc_select_for_pages = Mock()
        self.repo.toc_generate_pages = Mock()
        self.repo.toc_generate_first_page = Mock()
        with patch('json.dump'):
            self.repo.toc()

        # assert facet_query was properly called, error and info msg
        # was printed
        self.assertEqual("http://localhost:8000/dataset/base",
                         self.repo.facet_query.call_args[0][0])
        self.assertTrue(self.repo.log.error.called)
        self.assertTrue(self.repo.log.info.called)
        # and that the rest of the methods were NOT called
        self.assertFalse(self.repo.toc_pagesets.called)
        self.assertFalse(self.repo.toc_select_for_pages.called)
        self.assertFalse(self.repo.toc_generate_pages.called)

        # test2: facet_select returns something
        self.repo.faceted_data.return_value = ["fake", "data"]
        with patch('json.load'):
            self.repo.toc()
        # Now all other methods should be called
        self.assertTrue(self.repo.toc_pagesets.called)
        self.assertTrue(self.repo.toc_select_for_pages.called)
        self.assertTrue(self.repo.toc_generate_pages.called)
Esempio n. 6
0
    def test_ifneeded_parse(self):
        @ifneeded("parse")
        def testfunc(repo, basefile):
            repo.called = True

        # mockdoc = Mock()
        # mockdoc.basefile="1234"
        mockbasefile = "1234"
        mockrepo = Mock()
        mockrepo.store.needed = DocumentStore(datadir='fake').needed
        mockrepo.called = False
        mockrepo.config.force = False

        # test 1: Outfile is newer - the ifneeded decorator should
        # make sure the actual testfunc code is never reached
        with patch('ferenda.util.outfile_is_newer', return_value=True):
            testfunc(mockrepo, mockbasefile)

        self.assertFalse(mockrepo.called)
        mockrepo.called = False

        # test 2: Outfile is older
        with patch('ferenda.util.outfile_is_newer', return_value=False):
            testfunc(mockrepo, mockbasefile)
        self.assertTrue(mockrepo.called)
        mockrepo.called = False

        # test 3: Outfile is newer, but the global force option was set
        mockrepo.config.force = True
        with patch('ferenda.util.outfile_is_newer', return_value=True):
            testfunc(mockrepo, mockbasefile)
        self.assertTrue(mockrepo.called)
        mockrepo.config.force = None
        mockrepo.called = False
Esempio n. 7
0
    def test_ifneeded_parse(self):
        @ifneeded("parse")
        def testfunc(repo, basefile):
            repo.called = True

        # mockdoc = Mock()
        # mockdoc.basefile="1234"
        mockbasefile = "1234"
        mockrepo = Mock()
        mockrepo.store.needed = DocumentStore(datadir='fake').needed
        mockrepo.called = False
        mockrepo.config.force = False

        # test 1: Outfile is newer - the ifneeded decorator should
        # make sure the actual testfunc code is never reached
        with patch('ferenda.util.outfile_is_newer', return_value=True):
            testfunc(mockrepo, mockbasefile)

        self.assertFalse(mockrepo.called)
        mockrepo.called = False

        # test 2: Outfile is older
        with patch('ferenda.util.outfile_is_newer', return_value=False):
            testfunc(mockrepo, mockbasefile)
        self.assertTrue(mockrepo.called)
        mockrepo.called = False

        # test 3: Outfile is newer, but the global force option was set
        mockrepo.config.force = True
        with patch('ferenda.util.outfile_is_newer', return_value=True):
            testfunc(mockrepo, mockbasefile)
        self.assertTrue(mockrepo.called)
        mockrepo.config.force = None
        mockrepo.called = False
Esempio n. 8
0
    def test_preflight(self):
        log = Mock()

        # test 1: python too old

        with patch('ferenda.manager.sys') as sysmock:
            sysmock.version_info = (2, 5, 6, 'final', 0)
            sysmock.version = sys.version
            self.assertFalse(manager._preflight_check(log, verbose=True))
            self.assertTrue(log.error.called)
            log.error.reset_mock()

        # test 2: modules are old / or missing
        with patch('importlib.import_module') as importmock:
            setattr(importmock.return_value, '__version__', '0.0.1')
            self.assertFalse(manager._preflight_check(log, verbose=True))
            self.assertTrue(log.error.called)
            log.error.reset_mock()

            importmock.side_effect = ImportError
            self.assertFalse(manager._preflight_check(log, verbose=True))
            self.assertTrue(log.error.called)
            log.error.reset_mock()

        # test 3: binaries are nonexistent or errors
        with patch('ferenda.manager.subprocess.call') as callmock:
            callmock.return_value = 127
            self.assertFalse(manager._preflight_check(log, verbose=True))
            self.assertTrue(log.error.called)
            log.error.reset_mock()

            callmock.side_effect = OSError
            self.assertFalse(manager._preflight_check(log, verbose=True))
            self.assertTrue(log.error.called)
            log.error.reset_mock()
Esempio n. 9
0
    def test_preflight(self):
        log = Mock()
        
        # test 1: python too old

        with patch('ferenda.manager.sys') as sysmock:
            sysmock.version_info = (2,5,6,'final',0)
            sysmock.version = sys.version
            self.assertFalse(manager._preflight_check(log, verbose=True))
            self.assertTrue(log.error.called)
            log.error.reset_mock()

        # test 2: modules are old / or missing
        with patch('importlib.import_module') as importmock:
            setattr(importmock.return_value, '__version__', '0.0.1')
            self.assertFalse(manager._preflight_check(log, verbose=True))
            self.assertTrue(log.error.called)
            log.error.reset_mock()

            importmock.side_effect = ImportError
            self.assertFalse(manager._preflight_check(log, verbose=True))
            self.assertTrue(log.error.called)
            log.error.reset_mock()

        # test 3: binaries are nonexistent or errors
        with patch('ferenda.manager.subprocess.call') as callmock:
            callmock.return_value = 127
            self.assertFalse(manager._preflight_check(log, verbose=True))
            self.assertTrue(log.error.called)
            log.error.reset_mock()

            callmock.side_effect = OSError
            self.assertFalse(manager._preflight_check(log, verbose=True))
            self.assertTrue(log.error.called)
            log.error.reset_mock()
Esempio n. 10
0
    def test_select(self):
        uri = "http://example.org/doc"
        with open("testselecttemplate.rq", "wb") as fp:
            fp.write("""PREFIX dcterms: <http://purl.org/dc/terms/>

SELECT ?p ?o
WHERE { <%(uri)s> ?p ?o . }
""".encode())

        result = """
[
    {
        "p": "http://purl.org/dc/terms/title", 
        "o": "Document title"
    }, 
    {
        "p": "http://purl.org/dc/terms/identifier", 
        "o": "Document ID"
    }
]""".lstrip().encode("utf-8")
        config = {
            'connect.return_value': Mock(**{'select.return_value': result})
        }
        printmock = MagicMock()
        with patch('ferenda.devel.TripleStore', **config):
            with patch('builtins.print', printmock):
                d = Devel()
                d.config = LayeredConfig(
                    Defaults({
                        'storetype': 'a',
                        'storelocation': 'b',
                        'storerepository': 'c'
                    }))
                d.select("testselecttemplate.rq", uri)
        want = """
# Constructing the following from b, repository c, type a
# PREFIX dcterms: <http://purl.org/dc/terms/>
# 
# SELECT ?p ?o
# WHERE { <http://example.org/doc> ?p ?o . }
# 

[
    {
        "p": "http://purl.org/dc/terms/title", 
        "o": "Document title"
    }, 
    {
        "p": "http://purl.org/dc/terms/identifier", 
        "o": "Document ID"
    }
]
# Selected in 0.001s
""".strip()
        got = "\n".join([x[1][0] for x in printmock.mock_calls])
        self.maxDiff = None
        self.assertEqual(self.mask_time(want), self.mask_time(got))
        os.unlink("testselecttemplate.rq")
Esempio n. 11
0
 def test_runsetup(self):
     with patch('ferenda.manager.sys.exit') as mockexit:
         with patch('ferenda.manager.setup', return_value=True):
             manager.runsetup()
             self.assertFalse(mockexit.called)
             mockexit.reset_mock()
         with patch('ferenda.manager.setup', return_value=False):
             manager.runsetup()
             self.assertTrue(mockexit.called)
Esempio n. 12
0
    def test_render(self,mock_graph):
        @render
        def testfunc(repo,doc):
            pass

        mockdoc = Mock()
        mockrepo = Mock()
        mockrepo.store.parsed_path.return_value = "parsed_path.xhtml"
        with open("parsed_path.xhtml", "w") as fp:
            fp.write("""<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML+RDFa 1.0//EN" "http://www.w3.org/MarkUp/DTD/xhtml-rdfa-1.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xmlns:dct="http://purl.org/dc/terms/">
  <head about="http://example.org/doc">
     <title property="dct:title">Document title</title>
  </head>
  <body>
     <h1>Hello!</h1>
  </body>
</html>""")

        mockrepo.store.distilled_path.return_value = "distilled_path.xhtml"
        mockrepo.get_globals.return_value = {'symbol table':'fake'}
        mockdoc.meta = MagicMock() # need Magicmock which supports magic funcs like __iter__
        bodypart = MagicMock()
        bodypart.meta  = MagicMock()
        mockdoc.body = [bodypart]
        mockdoc.meta.__iter__.return_value = []
        mockdoc.uri = "http://example.org/doc"
        with patch('ferenda.util.ensure_dir', return_value=True):
            testfunc(mockrepo, mockdoc)
        
        # 1 ensure that DocumentRepository.render_xhtml is called with
        # four arguments
        mockrepo.render_xhtml.assert_called_with(mockdoc, "parsed_path.xhtml")

        # 2 ensure that DocumentRepository.create_external_resources
        # is called with 1 argument
        mockrepo.create_external_resources.assert_called_with(mockdoc)
        
        # 3 ensure that a Graph object is created, its parse and
        # serialize methods called

        # FIXME: Why doesn't the patching work?!
        # self.assertTrue(mock_graph().parse.called)
        # self.assertTrue(mock_graph().serialize.called)
        
        # (4. ensure that a warning gets printed if doc.meta and
        # distilled_graph do not agree)
        mock_graph().__iter__.return_value = ['a','b']
        mockdoc.meta.__iter__.return_value = ['a','b','c']
        mockdoc.meta.serialize.return_value = b"<c>"

        with patch('ferenda.util.ensure_dir', return_value=True):
            testfunc(mockrepo, mockdoc)
        self.assertTrue(mockrepo.log.warning.called)
        os.remove("parsed_path.xhtml")
        os.remove("distilled_path.xhtml")
Esempio n. 13
0
 def test_runsetup(self):
     with patch('ferenda.manager.sys.exit') as mockexit:
         with patch('ferenda.manager.setup', return_value=True):
             manager.runsetup()
             self.assertFalse(mockexit.called)
             mockexit.reset_mock()
         with patch('ferenda.manager.setup', return_value=False):
             manager.runsetup()
             self.assertTrue(mockexit.called)
Esempio n. 14
0
 def test_run_single_errors(self):
     self._enable_repos()
     argv = ["test", "errmethod", "--all"]
     with patch('ferenda.manager.setup_logger'):
         with patch('builtins.print') as printmock:
             res = manager.run(argv)
     self.assertEqual(res[0][0], Exception)
     self.assertEqual(res[1][0], errors.DocumentRemovedError)
     self.assertEqual(res[2], None)
     self.assertTrue(os.path.exists("dummyfile.txt"))
Esempio n. 15
0
 def test_run_single_errors(self):
     self._enable_repos()
     argv = ["test", "errmethod", "--all"]
     with patch('ferenda.manager.setup_logger'):
         with patch('builtins.print') as printmock:
             res = manager.run(argv)
     self.assertEqual(res[0][0], Exception)
     self.assertEqual(res[1][0], errors.DocumentRemovedError)
     self.assertEqual(res[2], None)
     self.assertTrue(os.path.exists("dummyfile.txt"))
Esempio n. 16
0
    def test_select(self):
        uri = "http://example.org/doc"
        with open("testselecttemplate.rq", "wb") as fp:
            fp.write(
                """PREFIX dct: <http://purl.org/dc/terms/>

SELECT ?p ?o
WHERE { <%(uri)s> ?p ?o . }
""".encode()
            )

        result = """
[
    {
        "p": "http://purl.org/dc/terms/title", 
        "o": "Document title"
    }, 
    {
        "p": "http://purl.org/dc/terms/identifier", 
        "o": "Document ID"
    }
]""".lstrip().encode(
            "utf-8"
        )
        config = {"connect.return_value": Mock(**{"select.return_value": result})}
        printmock = MagicMock()
        with patch("ferenda.devel.TripleStore", **config):
            with patch(builtins + ".print", printmock):
                d = Devel()
                d.config = LayeredConfig({"storetype": "a", "storelocation": "b", "storerepository": "c"})
                d.select("testselecttemplate.rq", uri)
        want = """
# Constructing the following from b, repository c, type a
# PREFIX dct: <http://purl.org/dc/terms/>
# 
# SELECT ?p ?o
# WHERE { <http://example.org/doc> ?p ?o . }
# 

[
    {
        "p": "http://purl.org/dc/terms/title", 
        "o": "Document title"
    }, 
    {
        "p": "http://purl.org/dc/terms/identifier", 
        "o": "Document ID"
    }
]
# Selected in 0.001s
""".strip()
        got = "\n".join([x[1][0] for x in printmock.mock_calls])
        self.maxDiff = None
        self.assertEqual(self.mask_time(want), self.mask_time(got))
        os.unlink("testselecttemplate.rq")
Esempio n. 17
0
 def test_dumpstore(self):
     d = Devel()
     d.config = Mock()
     # only test that Triplestore is called correctly, mock any
     # calls to any real database
     config = {"connect.return_value": Mock(**{"get_serialized.return_value": b"[fake store content]"})}
     printmock = MagicMock()
     with patch("ferenda.devel.TripleStore", **config):
         with patch(builtins + ".print", printmock):
             d.dumpstore(format="trix")
     want = "[fake store content]"
     printmock.assert_has_calls([call(want)])
Esempio n. 18
0
    def test_construct(self):
        uri = "http://example.org/doc"
        with open("testconstructtemplate.rq", "wb") as fp:
            fp.write("""PREFIX dcterms: <http://purl.org/dc/terms/>

CONSTRUCT { ?s ?p ?o . }
WHERE { ?s ?p ?o .
        <%(uri)s> ?p ?o . }
""".encode())
        g = Graph()
        g.bind("dcterms", str(DCTERMS))
        g.add((URIRef(uri), DCTERMS.title, Literal("Document title")))
        config = {
            'connect.return_value': Mock(**{'construct.return_value': g})
        }
        printmock = MagicMock()
        with patch('ferenda.devel.TripleStore', **config):
            with patch('builtins.print', printmock):
                d = Devel()
                d.config = LayeredConfig(
                    Defaults({
                        'storetype': 'a',
                        'storelocation': 'b',
                        'storerepository': 'c'
                    }))
                d.construct("testconstructtemplate.rq", uri)
        want = """
# Constructing the following from b, repository c, type a
# PREFIX dcterms: <http://purl.org/dc/terms/>
# 
# CONSTRUCT { ?s ?p ?o . }
# WHERE { ?s ?p ?o .
#         <http://example.org/doc> ?p ?o . }
# 

@prefix dcterms: <http://purl.org/dc/terms/> .
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@prefix xml: <http://www.w3.org/XML/1998/namespace> .
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .

<http://example.org/doc> dcterms:title "Document title" .


# 1 triples constructed in 0.001s
""".strip()
        got = "\n".join([x[1][0] for x in printmock.mock_calls])
        self.maxDiff = None
        self.assertEqual(self.mask_time(want), self.mask_time(got))
        os.unlink("testconstructtemplate.rq")
Esempio n. 19
0
    def test_construct(self):
        uri = "http://example.org/doc"
        with open("testconstructtemplate.rq", "wb") as fp:
            fp.write("""PREFIX dcterms: <http://purl.org/dc/terms/>

CONSTRUCT { ?s ?p ?o . }
WHERE { ?s ?p ?o .
        <%(uri)s> ?p ?o . }
""".encode())            
        g = Graph()
        g.bind("dcterms", str(DCTERMS))
        g.add((URIRef(uri),
               DCTERMS.title,
               Literal("Document title")))
        config = {'connect.return_value':
                  Mock(**{'construct.return_value': g})}
        printmock = MagicMock()
        with patch('ferenda.devel.TripleStore', **config):
            with patch('builtins.print', printmock):
                d = Devel()
                d.config = LayeredConfig(Defaults({'storetype': 'a',
                                                   'storelocation': 'b',
                                                   'storerepository': 'c'}))
                d.construct("testconstructtemplate.rq", uri)
        want = """
# Constructing the following from b, repository c, type a
# PREFIX dcterms: <http://purl.org/dc/terms/>
# 
# CONSTRUCT { ?s ?p ?o . }
# WHERE { ?s ?p ?o .
#         <http://example.org/doc> ?p ?o . }
# 

@prefix dcterms: <http://purl.org/dc/terms/> .
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@prefix xml: <http://www.w3.org/XML/1998/namespace> .
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .

<http://example.org/doc> dcterms:title "Document title" .


# 1 triples constructed in 0.001s
""".strip()
        got = "\n".join([x[1][0] for x in printmock.mock_calls])
        self.maxDiff = None
        self.assertEqual(self.mask_time(want),
                         self.mask_time(got))
        os.unlink("testconstructtemplate.rq")
Esempio n. 20
0
 def test_dumpstore(self):
     d = Devel()
     d.config = Mock()
     # only test that Triplestore is called correctly, mock any
     # calls to any real database
     config = {
         'connect.return_value':
         Mock(**{'get_serialized.return_value': b'[fake store content]'})
     }
     printmock = MagicMock()
     with patch('ferenda.devel.TripleStore', **config):
         with patch('builtins.print', printmock):
             d.dumpstore(format="trix")
     want = "[fake store content]"
     printmock.assert_has_calls([call(want)])
Esempio n. 21
0
 def test_runserver(self):
     self._enable_repos()
     m = Mock()
     with patch('ferenda.manager.make_server', return_value=m) as m2:
         manager.run(["all", "runserver"])
         self.assertTrue(m2.called)
         self.assertTrue(m.serve_forever.called)
Esempio n. 22
0
 def test_runserver(self):
     self._enable_repos()
     m = Mock()
     with patch('ferenda.manager.make_server', return_value=m) as m2:
         manager.run(["all", "runserver"])
         self.assertTrue(m2.called)
         self.assertTrue(m.serve_forever.called)
Esempio n. 23
0
    def test_parameters_legacy(self):
        # legacy api
        res = ([], {'firstresult': 1,
                    'totalresults': 0})
        # FIXME: we leave out free=true (should map to schema_free=True)
        self.env['QUERY_STRING'] = "type=Standard&title=Hello+World&issued=2014-06-30&schema_free=true"
        self.app.config.legacyapi = True
        config = {'connect.return_value': 
                  Mock(**{'query.return_value': res,
                          'schema.return_value': {'dcterms_issued': fulltextindex.Datetime(),
                                                  'schema_free': fulltextindex.Boolean(),
                                                  'dcterms_title': None,
                                                  'rdf_type': None}})}

        want = {'q': None,
                'dcterms_title': "Hello World",
                'dcterms_issued': datetime.datetime(2014,6,30,0,0,0),
                'schema_free': True,
                'rdf_type': '*Standard', # should be bibo:Standard or even http://purl.org/ontology/bibo/Standard, but requires proper context handling to work
                'pagenum': 1,
                'pagelen': 10,
                'ac_query': False,
                'boost_types': None,
                'exclude_types': None}

        with patch('ferenda.wsgiapp.FulltextIndex', **config):
            status, headers, content = self.call_wsgi(self.env)
            config['connect.return_value'].query.assert_called_once_with(**want)
Esempio n. 24
0
    def test_parameters_legacy(self):
        # legacy api
        res = ([], {'firstresult': 1,
                    'totalresults': 0})
        # FIXME: we leave out free=true (should map to schema_free=True)
        self.builder.query_string = "type=Standard&title=Hello+World&issued=2014-06-30&schema_free=true"
        self.app.config.legacyapi = True
        config = {'connect.return_value': 
                  Mock(**{'query.return_value': res,
                          'schema.return_value': {'dcterms_issued': fulltextindex.Datetime(),
                                                  'schema_free': fulltextindex.Boolean(),
                                                  'dcterms_title': None,
                                                  'rdf_type': None}})}

        want = {'q': None,
                'dcterms_title': "Hello World",
                'dcterms_issued': datetime.datetime(2014,6,30,0,0,0),
                'schema_free': True,
                'rdf_type': '*Standard', # should be bibo:Standard or even http://purl.org/ontology/bibo/Standard, but requires proper context handling to work
                'pagenum': 1,
                'pagelen': 10,
                'ac_query': False,
                'boost_repos': None,
                'exclude_repos': None,
                'include_fragments': None}

        with patch('ferenda.wsgiapp.FulltextIndex', **config):
            status, headers, content = self.call_wsgi()
            config['connect.return_value'].query.assert_called_once_with(**want)
Esempio n. 25
0
    def test_highlighted_snippet(self):
        res = ([{'title':'Example',
                 'uri':'http://example.org/base/123/b1',
                 'text':html.P(['sollicitudin justo ',
                                html.Strong(['needle'], **{'class':'match'}),
                                ' tempor ut eu enim ... himenaeos. ',
                                html.Strong(['Needle'], **{'class':'match'}),
                                ' id tincidunt orci'])}],
               {'pagenum': 1,
                'pagecount': 1,
                'firstresult': 1,
                'lastresult': 1,
                'totalresults': 1})

        self.builder.query_string = "q=needle"
        config = {'connect.return_value': Mock(**{'query.return_value': res})}
        with patch('ferenda.wsgiapp.FulltextIndex', **config):
            status, headers, content = self.call_wsgi()
        
        self.assertResponse("200 OK",
                            {'Content-Type': 'text/html; charset=utf-8'},
                            None,
                            status, headers, None)
        
        t = etree.fromstring(content)
        docs = t.findall(".//section[@class='hit']")
        self.assertEqualXML(res[0][0]['text'].as_xhtml(),
                            docs[0][1],
                            namespace_aware=False)
Esempio n. 26
0
 def test_parameters(self):
     # normal api
     res = ([], {'firstresult': 1, 'totalresults': 0})
     self.env[
         'QUERY_STRING'] = "rdf_type=bibo:Standard&dcterms_title=Hello+World&dcterms_issued=2014-06-30&schema_free=true"
     config = {
         'connect.return_value':
         Mock(
             **{
                 'query.return_value': res,
                 'schema.return_value': {
                     'dcterms_issued': fulltextindex.Datetime(),
                     'schema_free': fulltextindex.Boolean()
                 }
             })
     }
     want = {
         'q': None,
         'dcterms_title': "Hello World",
         'dcterms_issued': datetime.datetime(2014, 6, 30, 0, 0, 0),
         'schema_free': True,
         'rdf_type':
         'bibo:Standard',  # FIXME: should be http://purl.org/ontology/bibo/Standard -- but requires that self.repos in wsgiapp is set up
         'pagenum': 1,
         'pagelen': 10,
         'ac_query': False,
         'boost_types': None,
         'exclude_types': None
     }
     with patch('ferenda.wsgiapp.FulltextIndex', **config):
         status, headers, content = self.call_wsgi(self.env)
         config['connect.return_value'].query.assert_called_once_with(
             **want)
Esempio n. 27
0
    def test_run_single(self):
        # test1: run standard (custom) method
        self._enable_repos()
        argv = ["test", "mymethod", "myarg"]
        self.assertEqual(manager.run(argv), "ok!")
        # test2: specify invalid alias
        argv[0] = "invalid"

        with patch('ferenda.manager.setup_logger'):
            self.assertEqual(manager.run(argv), None)

        with patch('builtins.print') as printmock:
            with patch('ferenda.manager.setup_logger'):
                # test3: specify invalid method
                argv = ["test", "invalid"]
                self.assertEqual(manager.run(argv), None)
Esempio n. 28
0
def silence():
    """The same functionality as quiet(), but as a context manager so that
    one can use "with silence:" constructs."""
    state = _setup()
    with patch('builtins.print') as printmock:
        yield
    _restore(state)
Esempio n. 29
0
    def test_highlighted_snippet(self):
        res = ([{'title':'Example',
                 'uri':'http://example.org/base/123/b1',
                 'text':html.P(['sollicitudin justo ',
                                html.Strong(['needle'], **{'class':'match'}),
                                ' tempor ut eu enim ... himenaeos. ',
                                html.Strong(['Needle'], **{'class':'match'}),
                                ' id tincidunt orci'])}],
               {'pagenum': 1,
                'pagecount': 1,
                'firstresult': 1,
                'lastresult': 1,
                'totalresults': 1})

        self.env['QUERY_STRING'] = "q=needle"
        config = {'connect.return_value': Mock(**{'query.return_value': res})}
        with patch('ferenda.manager.FulltextIndex', **config):
            status, headers, content = self.call_wsgi(self.env)
        
        self.assertResponse("200 OK",
                            {'Content-Type': 'text/html; charset=utf-8'},
                            None,
                            status, headers, None)
        
        t = etree.fromstring(content)
        docs = t.findall(".//section[@class='hit']")
        self.assertEqualXML(res[0][0]['text'].as_xhtml(),
                            docs[0][1],
                            namespace_aware=False)
Esempio n. 30
0
    def test_run_single(self):
        # test1: run standard (custom) method
        self._enable_repos()
        argv = ["test","mymethod","myarg"]
        self.assertEqual(manager.run(argv),
                         "ok!")
        # test2: specify invalid alias
        argv[0] = "invalid"

        with patch('ferenda.manager.setup_logger'):
            self.assertEqual(manager.run(argv), None)

        with patch('builtins.print') as printmock:
            with patch('ferenda.manager.setup_logger'):
                # test3: specify invalid method
                argv = ["test", "invalid"]
                self.assertEqual(manager.run(argv), None)
Esempio n. 31
0
    def test_select_triplestore(self):
        log = Mock()
        # first manipulate requests.get to give the impression that
        # fuseki or sesame either is or isn't available
        with patch('ferenda.manager.requests.get') as mock_get:
            r = manager._select_triplestore("sitename", log, verbose=True)
            self.assertEqual("FUSEKI", r[0])

            mock_get.side_effect = requests.exceptions.HTTPError
            r = manager._select_triplestore("sitename", log, verbose=True)
            self.assertNotEqual("FUSEKI", r[0])

            def get_sesame(url):
                if not 'openrdf-sesame' in url:
                    raise requests.exceptions.HTTPError
                resp = Mock()
                resp.text = "ok"
                return resp

            mock_get.side_effect = get_sesame
            r = manager._select_triplestore("sitename", log, verbose=True)
            self.assertEqual("SESAME", r[0])

            mock_get.side_effect = requests.exceptions.HTTPError
            r = manager._select_triplestore("sitename", log, verbose=True)
            self.assertNotEqual("SESAME", r[0])

            # all request.get calls still raises HTTP error
            with patch('ferenda.manager.TripleStore.connect') as mock_connect:
                r = manager._select_triplestore("sitename", log, verbose=True)
                self.assertEqual("SQLITE", r[0])

                def connectfail(storetype, location, repository):
                    if storetype == "SQLITE":
                        raise ImportError("BOOM")

                mock_connect.side_effect = connectfail
                r = manager._select_triplestore("sitename", log, verbose=True)
                self.assertNotEqual("SQLITE", r[0])

                r = manager._select_triplestore("sitename", log, verbose=True)
                self.assertEqual("SLEEPYCAT", r[0])
                mock_connect.side_effect = ImportError
                r = manager._select_triplestore("sitename", log, verbose=True)
                self.assertEqual(None, r[0])
Esempio n. 32
0
 def test_ensure_dir(self):
     self.assertFalse(os.path.exists(self.dname))
     util.ensure_dir(self.fname)
     self.assertTrue(os.path.exists(self.dname))
     self.assertTrue(os.path.isdir(self.dname))
     util.ensure_dir(self.fname)
     os.rmdir(self.dname)
     with patch('ferenda.util.mkdir', side_effect=OSError):
         util.ensure_dir(self.fname)
Esempio n. 33
0
def silence():
    """The same functionality as quiet(), but as a context manager so that
    one can use "with silence:" constructs."""
    state = _setup()
    try:
        with patch('builtins.print') as printmock:
            yield
    finally:
        _restore(state)
Esempio n. 34
0
    def test_select_triplestore(self):
        log = Mock()
        # first manipulate requests.get to give the impression that
        # fuseki or sesame either is or isn't available
        with patch('ferenda.manager.requests.get') as mock_get:
            r = manager._select_triplestore("sitename", log, verbose=True)
            self.assertEqual("FUSEKI", r[0])
            
            mock_get.side_effect = requests.exceptions.HTTPError
            r = manager._select_triplestore("sitename", log, verbose=True)
            self.assertNotEqual("FUSEKI", r[0])

            def get_sesame(url):
                if not 'openrdf-sesame' in url:
                    raise requests.exceptions.HTTPError
                resp = Mock()
                resp.text = "ok"
                return resp

            mock_get.side_effect = get_sesame
            r = manager._select_triplestore("sitename", log, verbose=True)
            self.assertEqual("SESAME", r[0])

            mock_get.side_effect = requests.exceptions.HTTPError
            r = manager._select_triplestore("sitename", log, verbose=True)
            self.assertNotEqual("SESAME", r[0])

            # all request.get calls still raises HTTP error
            with patch('ferenda.manager.TripleStore.connect') as mock_connect:
                r = manager._select_triplestore("sitename", log, verbose=True)
                self.assertEqual("SQLITE", r[0])
                def connectfail(storetype, location, repository):
                    if storetype == "SQLITE":
                        raise ImportError("BOOM")
                mock_connect.side_effect = connectfail
                r = manager._select_triplestore("sitename", log, verbose=True)
                self.assertNotEqual("SQLITE", r[0])

                r = manager._select_triplestore("sitename", log, verbose=True)
                self.assertEqual("SLEEPYCAT", r[0])
                mock_connect.side_effect = ImportError
                r = manager._select_triplestore("sitename", log, verbose=True)
                self.assertEqual(None, r[0])
Esempio n. 35
0
    def test_run_single_all(self):
        self._enable_repos()
        argv = ["test","mymethod","--all"]
        # Test 1: make sure that if setup signals that no work should
        # be done, this is respected
        with patch("example.Testrepo.setup", return_value=False):
            self.assertEqual(manager.run(list(argv)), [])
            # pass

        # Test 2: but if not, do the work
        self.assertEqual(manager.run(list(argv)), [None, "ok!", None])
Esempio n. 36
0
    def test_run_single_all(self):
        self._enable_repos()
        argv = ["test", "mymethod", "--all"]
        # Test 1: make sure that if setup signals that no work should
        # be done, this is respected
        with patch("example.Testrepo.setup", return_value=False):
            self.assertEqual(manager.run(list(argv)), [])
            # pass

        # Test 2: but if not, do the work
        self.assertEqual(manager.run(list(argv)), [None, "ok!", None])
Esempio n. 37
0
    def test_queryindex(self):
        res = [
            {"identifier": "Doc #1", "about": "http://example.org/doc1", "text": "matching doc 1"},
            {"identifier": "Doc #2", "about": "http://example.org/doc2", "text": "matching doc 2"},
        ]

        config = {"connect.return_value": Mock(**{"query.return_value": res})}
        printmock = MagicMock()
        with patch("ferenda.devel.FulltextIndex", **config):
            with patch(builtins + ".print", printmock):
                d = Devel()
                d.config = LayeredConfig({"indextype": "a", "indexlocation": "b"})
                d.queryindex("doc")
        want = """
Doc #1 (http://example.org/doc1): matching doc 1
Doc #2 (http://example.org/doc2): matching doc 2
""".strip()
        got = "\n".join([x[1][0] for x in printmock.mock_calls])
        self.maxDiff = None
        self.assertEqual(want, got)
Esempio n. 38
0
    def test_select_fulltextindex(self):
        log = Mock()
        # first manipulate requests.get to give the impression that
        # elasticsearch either is or isn't available
        with patch('ferenda.manager.requests.get') as mock_get:
            r = manager._select_fulltextindex(log, "mysite", verbose=True)
            self.assertEqual("ELASTICSEARCH", r[0])
            self.assertEqual("http://localhost:9200/mysite/", r[1])
            mock_get.side_effect = requests.exceptions.HTTPError

            r = manager._select_fulltextindex(log, "mysite", verbose=True)
            self.assertEqual("WHOOSH", r[0])
Esempio n. 39
0
 def test_recordlastdownload(self):
     @recordlastdownload
     def testfunc(repo):
         pass
     mockrepo = Mock()
     with patch('ferenda.decorators.LayeredConfig.write') as mockconf:
         testfunc(mockrepo)
         # check that config.lastdownload has been set to a datetime
         self.assertIsInstance(mockrepo.config.lastdownload,
                               datetime.datetime)
         # and that LayeredConfig.write has been called
         self.assertTrue(mockconf.called)
Esempio n. 40
0
    def test_select_fulltextindex(self):
        log = Mock()
        # first manipulate requests.get to give the impression that
        # elasticsearch either is or isn't available
        with patch('ferenda.manager.requests.get') as mock_get:
            r = manager._select_fulltextindex(log, "mysite", verbose=True)
            self.assertEqual("ELASTICSEARCH", r[0])
            self.assertEqual("http://localhost:9200/mysite/", r[1])
            mock_get.side_effect = requests.exceptions.HTTPError

            r = manager._select_fulltextindex(log, "mysite", verbose=True)
            self.assertEqual("WHOOSH", r[0])
Esempio n. 41
0
 def test_recordlastdownload(self):
     @recordlastdownload
     def testfunc(repo):
         pass
     mockrepo = Mock()
     with patch('ferenda.decorators.LayeredConfig.write') as mockconf:
         testfunc(mockrepo)
         # check that config.lastdownload has been set to a datetime
         self.assertIsInstance(mockrepo.config.lastdownload,
                               datetime.datetime)
         # and that LayeredConfig.write has been called
         self.assertTrue(mockconf.called)
Esempio n. 42
0
 def test_distill_setfile(self):
     os.mkdir(self.datadir+"/downloaded")
     util.writefile(self.datadir+"/downloaded/a.html",
                    "<p>This is doc A</p>")
     util.writefile(self.datadir+"/distilled/a.ttl",  "")
     os.environ["FERENDA_SET_TESTFILE"] = "1"
     with patch("builtins.print") as printmock:
         self._runtest()
     del os.environ["FERENDA_SET_TESTFILE"]
     self.assertEqual(self.expected_ttl,
                      util.readfile(self.datadir+"/distilled/a.ttl"))
     pass
Esempio n. 43
0
 def test_distill_setfile(self):
     os.mkdir(self.datadir + "/downloaded")
     util.writefile(self.datadir + "/downloaded/a.html",
                    "<p>This is doc A</p>")
     util.writefile(self.datadir + "/distilled/a.ttl", "")
     os.environ["FERENDA_SET_TESTFILE"] = "1"
     with patch("builtins.print") as printmock:
         self._runtest()
     del os.environ["FERENDA_SET_TESTFILE"]
     self.assertEqual(self.expected_ttl,
                      util.readfile(self.datadir + "/distilled/a.ttl"))
     pass
Esempio n. 44
0
    def download_test(self, specfile):
        def my_get(url, **kwargs):
            urlspec = spec[url]
            if isinstance(urlspec, str):
                urlspec = {'file': urlspec}
            if 'charset' not in urlspec:
                urlspec['charset'] = 'utf-8'
            url_location = os.path.join(os.path.dirname(specfile),
                                        urlspec['file'])
            res = Mock()
            # load the .content property
            with open(url_location, "rb") as fp:
                res.content = fp.read()
            # but only load .text if a charset is present (note
            # default value of 'utf-8' above -- set 'charset': null in
            # the json file for binary files
            if urlspec['charset']:
                with codecs.open(url_location, "r", encoding=urlspec['charset']) as fp:
                    res.text = fp.read()
            # FIXME: Using a defaultdict ensures that we'll never trip
            # over the non-existance of certain headers. WE should
            # specify only the most basic headers to make sure calling
            # code doesn't rely on eg. the etag header always being
            # there, because it won't
            res.headers = collections.defaultdict(lambda: None)
            res.headers['X-These-Headers-Are'] = 'Faked'
            res.status_code = 200
            return res
        with codecs.open(specfile, encoding="utf-8") as fp:
            spec = json.load(fp)
        with patch('requests.get', side_effect=my_get):
            self.repo.download()

        # organize a temporary copy of files that we can compare our results to
        wantdir = "%s/%s-want" % (self.datadir, self.repoclass.alias)
        expected = False
        for url in spec:
            if "expect" in spec[url]:
                expected = True
                sourcefile = os.path.join(os.path.dirname(specfile),
                                          spec[url]['file'])
                wantfile = "%s/%s" % (wantdir, spec[url]['expect'])

                util.copy_if_different(sourcefile, wantfile)
        if expected:
            self.assertEqualDirs(wantdir,
                                 "%s/%s" % (self.datadir,
                                            self.repoclass.alias))
        else:
            self.fail('No files were marked as "expect" in specfile %s' %
                      specfile)
Esempio n. 45
0
    def test_handleerror(self):
        @handleerror
        def testfunc(repo, doc):
            if doc.exception:
                raise doc.exception
            else:
                return True

        mockrepo = Mock()
        mockrepo.config = MagicMock()  # must support __in__()
        mockdoc = Mock()
        # 1. should not raise an exception (but should call log.info
        #    and util.robust_remove, and return false)
        with patch('ferenda.util.robust_remove') as robust_remove:
            mockdoc.exception = DocumentRemovedError
            self.assertFalse(testfunc(mockrepo, mockdoc))
            self.assertTrue(mockrepo.log.info.called)
            self.assertTrue(robust_remove.called)

        # 2. should raise the same exception
        mockdoc.exception = KeyboardInterrupt
        with self.assertRaises(KeyboardInterrupt):
            testfunc(mockrepo, mockdoc)

        # 3.1 Should raise the same exeption
        mockdoc.exception = ParseError
        mockrepo.config.fatalexceptions = True
        with self.assertRaises(ParseError):
            testfunc(mockrepo, mockdoc)
        mockrepo.config.fatalexceptions = None

        # 3.2 Should not raise an exception (but should call log.error and return false)
        mockdoc.exception = ParseError
        self.assertFalse(testfunc(mockrepo, mockdoc))
        self.assertTrue(mockrepo.log.error.called)

        # 4.1 Should raise the same exception
        mockdoc.exception = Exception
        mockrepo.config.fatalexceptions = True
        with self.assertRaises(Exception):
            testfunc(mockrepo, mockdoc)
        mockrepo.config.fatalexceptions = None

        # 4.2 Should not raise an exception
        mockdoc.exception = Exception
        self.assertFalse(testfunc(mockrepo, mockdoc))
        self.assertTrue(mockrepo.log.error.called)

        # 5. No exceptions - everything should go fine
        mockdoc.exception = None
        self.assertTrue(testfunc(mockrepo, mockdoc))
Esempio n. 46
0
    def test_handleerror(self):
        @handleerror
        def testfunc(repo, doc):
            if doc.exception:
                raise doc.exception
            else:
                return True

        mockrepo = Mock()
        mockrepo.config = MagicMock()  # must support __in__()
        mockdoc = Mock()
        # 1. should not raise an exception (but should call log.info
        #    and util.robust_remove, and return false)
        with patch('ferenda.util.robust_remove') as robust_remove:
            mockdoc.exception = DocumentRemovedError
            self.assertFalse(testfunc(mockrepo, mockdoc))
            self.assertTrue(mockrepo.log.info.called)
            self.assertTrue(robust_remove.called)

        # 2. should raise the same exception
        mockdoc.exception = KeyboardInterrupt
        with self.assertRaises(KeyboardInterrupt):
            testfunc(mockrepo, mockdoc)

        # 3.1 Should raise the same exeption
        mockdoc.exception = ParseError
        mockrepo.config.fatalexceptions = True
        with self.assertRaises(ParseError):
            testfunc(mockrepo, mockdoc)
        mockrepo.config.fatalexceptions = None

        # 3.2 Should not raise an exception (but should call log.error and return false)
        mockdoc.exception = ParseError
        self.assertFalse(testfunc(mockrepo, mockdoc))
        self.assertTrue(mockrepo.log.error.called)

        # 4.1 Should raise the same exception
        mockdoc.exception = Exception
        mockrepo.config.fatalexceptions = True
        with self.assertRaises(Exception):
            testfunc(mockrepo, mockdoc)
        mockrepo.config.fatalexceptions = None

        # 4.2 Should not raise an exception
        mockdoc.exception = Exception
        self.assertFalse(testfunc(mockrepo, mockdoc))
        self.assertTrue(mockrepo.log.error.called)

        # 5. No exceptions - everything should go fine
        mockdoc.exception = None
        self.assertTrue(testfunc(mockrepo, mockdoc))
Esempio n. 47
0
    def test_queryindex(self):
        res = [{'label': 'Doc #1',
                'uri': 'http://example.org/doc1',
                'text': 'matching doc 1'},
               {'label': 'Doc #2',
                'uri': 'http://example.org/doc2',
                'text': 'matching doc 2'}]
        pager = None
        config = {'connect.return_value': Mock(**{'query.return_value': (res, pager)})}
        printmock = MagicMock()
        with patch('ferenda.devel.FulltextIndex', **config):
            with patch('builtins.print', printmock):
                d = Devel()
                d.config = LayeredConfig(Defaults({'indextype': 'a',
                                                   'indexlocation': 'b'}))
                d.queryindex("doc")
        want = """
Doc #1 (http://example.org/doc1): matching doc 1
Doc #2 (http://example.org/doc2): matching doc 2
""".strip()
        got = "\n".join([x[1][0] for x in printmock.mock_calls])
        self.maxDiff = None
        self.assertEqual(want, got)
Esempio n. 48
0
    def test_ifneeded_generate(self):
        @ifneeded("generate")
        def testfunc(repo, basefile):
            repo.called = True

        mockbasefile = "1234"
        mockrepo = Mock()
        mockrepo.store.needed = DocumentStore(datadir='fake').needed
        mockrepo.called = False
        mockrepo.config.force = False

        # test 1: Outfile is newer - the ifneeded decorator should
        # make sure the actual testfunc code is never reached
        with patch('ferenda.util.outfile_is_newer', return_value=True):
            testfunc(mockrepo, mockbasefile)

        self.assertFalse(mockrepo.called)
        mockrepo.called = False

        # test 2: Outfile is older than source file
        with patch('ferenda.util.outfile_is_newer', return_value=False):
            testfunc(mockrepo, mockbasefile)
        self.assertTrue(mockrepo.called)
        mockrepo.called = False

        # FIXME: we could add more tests, eg create a dependency file
        # and make sure an arbitrary file named in that depfile is
        # newer then outfile. but the tests in testDocStore.Needed
        # should cover that pretty well.

        # test 3: Outfile is newer, but the global force option was set
        mockrepo.config.force = True
        with patch('ferenda.util.outfile_is_newer', return_value=True):
            testfunc(mockrepo, mockbasefile)
        self.assertTrue(mockrepo.called)
        mockrepo.config.force = None
        mockrepo.called = False
Esempio n. 49
0
    def test_ifneeded_generate(self):
        @ifneeded("generate")
        def testfunc(repo, basefile):
            repo.called = True

        mockbasefile = "1234"
        mockrepo = Mock()
        mockrepo.store.needed = DocumentStore(datadir='fake').needed
        mockrepo.called = False
        mockrepo.config.force = False

        # test 1: Outfile is newer - the ifneeded decorator should
        # make sure the actual testfunc code is never reached
        with patch('ferenda.util.outfile_is_newer', return_value=True):
            testfunc(mockrepo, mockbasefile)

        self.assertFalse(mockrepo.called)
        mockrepo.called = False

        # test 2: Outfile is older than source file
        with patch('ferenda.util.outfile_is_newer', return_value=False):
            testfunc(mockrepo, mockbasefile)
        self.assertTrue(mockrepo.called)
        mockrepo.called = False

        # FIXME: we could add more tests, eg create a dependency file
        # and make sure an arbitrary file named in that depfile is
        # newer then outfile. but the tests in testDocStore.Needed
        # should cover that pretty well.

        # test 3: Outfile is newer, but the global force option was set
        mockrepo.config.force = True
        with patch('ferenda.util.outfile_is_newer', return_value=True):
            testfunc(mockrepo, mockbasefile)
        self.assertTrue(mockrepo.called)
        mockrepo.config.force = None
        mockrepo.called = False
Esempio n. 50
0
    def test_parseifneeded(self):
        @parseifneeded
        def testfunc(repo,doc):
            repo.called = True

        mockdoc = Mock()
        mockrepo = Mock()
        mockrepo.called = False
        mockrepo.config.force = False
        # test 1: Outfile is newer - the parseifneeded decorator
        # should make sure the actual testfunc code is never reached
        with patch('ferenda.util.outfile_is_newer', return_value=True):
            testfunc(mockrepo,mockdoc)

        self.assertFalse(mockrepo.called)
        mockrepo.called = False

        # test 2: Outfile is older
        with patch('ferenda.util.outfile_is_newer', return_value=False):
            testfunc(mockrepo,mockdoc)
        self.assertTrue(mockrepo.called)
        mockrepo.called = False

        # test 3: Outfile is newer, but the global force option was set
        mockrepo.config.force = True
        with patch('ferenda.util.outfile_is_newer', return_value=True):
            testfunc(mockrepo,mockdoc)
        self.assertTrue(mockrepo.called)
        mockrepo.config.force = None
        mockrepo.called = False

        # test 4: Outfile is newer, but the module parseforce option was set
        mockrepo.config.parseforce = True
        with patch('ferenda.util.outfile_is_newer', return_value=True):
            testfunc(mockrepo,mockdoc)
        self.assertTrue(mockrepo.called)
        mockrepo.called = False
Esempio n. 51
0
 def test_parse_setfile(self):
     os.mkdir(self.datadir + "/downloaded")
     util.writefile(self.datadir + "/downloaded/a.html",
                    "<p>This is doc A</p>")
     util.writefile(self.datadir + "/parsed/a.xhtml", "")
     os.environ["FERENDA_SET_TESTFILE"] = "1"
     with patch("builtins.print") as printmock:
         self._runtest()
     output = printmock.mock_calls[0][1][0]
     output = re.sub("'[^']*'", "''", output, 1)
     self.assertEqual("Overwriting '' with result of parse ('a')", output)
     del os.environ["FERENDA_SET_TESTFILE"]
     self.assertEqualXML(self.expected_xhtml,
                         util.readfile(self.datadir + "/parsed/a.xhtml"))
     pass
Esempio n. 52
0
 def test_parse_setfile(self):
     os.mkdir(self.datadir+"/downloaded")
     util.writefile(self.datadir+"/downloaded/a.html",
                    "<p>This is doc A</p>")
     util.writefile(self.datadir+"/parsed/a.xhtml",  "")
     os.environ["FERENDA_SET_TESTFILE"] = "1"
     with patch("builtins.print") as printmock:
         self._runtest()
     output = printmock.mock_calls[0][1][0]
     output = re.sub("'[^']*'", "''", output, 1)
     self.assertEqual("Overwriting '' with result of parse ('a')", output)
     del os.environ["FERENDA_SET_TESTFILE"]
     self.assertEqualXML(self.expected_xhtml,
                         util.readfile(self.datadir+"/parsed/a.xhtml"))
     pass
Esempio n. 53
0
 def test_faceted_data(self):
     canned = [
         {
             "uri": "http://example.org/books/A_Tale_of_Two_Cities",
             "dcterms_title": "A Tale of Two Cities"
         },
         {
             "uri": "http://example.org/books/The_Lord_of_the_Rings",
             "dcterms_title": "The Lord of the Rings"
         },
     ]
     with patch('ferenda.DocumentRepository.facet_select',
                return_value=canned) as mock:
         faceted_data = self.repo.faceted_data()
     self.assertEqual(faceted_data, canned)
     self.assertTrue(
         os.path.exists(self.datadir + "/base/toc/faceted_data.json"))
     # on second run, faceted_data should be read from the cache
     # (if outfile_is_newer is called, we're far enough down in
     # that branch to know that the cache file is used if
     # outfile_is_newer returns True)
     with patch('ferenda.util.outfile_is_newer', return_value=True):
         faceted_data = self.repo.faceted_data()
     self.assertEqual(faceted_data, canned)
Esempio n. 54
0
    def test_print_usage(self):
        self._enable_repos()
        with patch('builtins.print') as printmock:
            manager.run([])

        executable = sys.argv[0]
        got = "\n".join([x[1][0] for x in printmock.mock_calls])
        got = got.replace(executable, "[EXEC]")
        want = """Usage: [EXEC] [class-or-alias] [action] <arguments> <options>
   e.g. '[EXEC] ferenda.sources.EurlexCaselaw enable'
        '[EXEC] ecj parse 62008J0042'
        '[EXEC] all generate'
Available modules:
 * test: [Undocumented]
 * test2: [Undocumented]"""
        self.assertEqual(got, want)
Esempio n. 55
0
    def test_print_usage(self):
        self._enable_repos()
        with patch('builtins.print') as printmock:
            manager.run([])

        executable = sys.argv[0]
        got = "\n".join([x[1][0] for x in printmock.mock_calls])
        got = got.replace(executable, "[EXEC]")
        want = """Usage: [EXEC] [class-or-alias] [action] <arguments> <options>
   e.g. '[EXEC] ferenda.sources.EurlexCaselaw enable'
        '[EXEC] ecj parse 62008J0042'
        '[EXEC] all generate'
Available modules:
 * test: [Undocumented]
 * test2: [Undocumented]"""
        self.assertEqual(got, want)
Esempio n. 56
0
 def test_search_single(self):
     self.env['QUERY_STRING'] = "q=subsection"
     res = ([{'title': 'Result #1',
              'uri': 'http://example.org',
              'text': ['Text that contains the subsection term']}],
            {'pagenum': 1,
             'pagecount': 1,
             'firstresult': 1,
             'lastresult': 1,
             'totalresults': 1})
     
     config = {'connect.return_value': Mock(**{'query.return_value': res})}
     with patch('ferenda.manager.FulltextIndex', **config):
         status, headers, content = self.call_wsgi(self.env)
     t = etree.fromstring(content)
     resulthead = t.find(".//article/h1").text
     self.assertEqual(resulthead, "1 match for 'subsection'")
Esempio n. 57
0
 def test_search_single(self):
     self.builder.query_string = "q=subsection"
     res = ([{'dcterms_title': 'Result #1',
              'uri': 'http://example.org',
              'text': 'Text that contains the subsection term'}],
            {'pagenum': 1,
             'pagecount': 1,
             'firstresult': 1,
             'lastresult': 1,
             'totalresults': 1})
     
     config = {'connect.return_value': Mock(**{'query.return_value': res})}
     with patch('ferenda.wsgiapp.FulltextIndex', **config):
         status, headers, content = self.call_wsgi()
     t = etree.fromstring(content)
     resulthead = t.find(".//article/h1").text
     self.assertEqual("1 match for 'subsection'", resulthead)
Esempio n. 58
0
 def test_dumprdf(self):
     fileno, tmpfile = mkstemp()
     fp = os.fdopen(fileno, "w")
     fp.write("""<html xmlns="http://www.w3.org/1999/xhtml">
     <head about="http://example.org/doc">
        <title property="http://purl.org/dc/terms/">Doc title</title>
     </head>
     <body>...</body>
     </html>""")
     fp.close()
     d = Devel()
     mock = MagicMock()
     with patch('builtins.print', mock):
         d.dumprdf(tmpfile, format="nt")
     os.unlink(tmpfile)
     self.assertTrue(mock.called)
     want = '<http://example.org/doc> <http://purl.org/dc/terms/> "Doc title" .\n\n'
     mock.assert_has_calls([call(want)])
Esempio n. 59
0
    def test_setup(self, mockprint):
        # restart the log system since setup() will do that otherwise
        manager.shutdown_logger()
        manager.setup_logger('CRITICAL')
        projdir = self.datadir + os.sep + 'myproject'
        argv = ['ferenda-build.py', projdir]

        # test1: normal, setup succeeds
        res = manager.setup(force=True,
                            verbose=False,
                            unattended=True,
                            argv=argv)
        self.assertTrue(res)
        self.assertTrue(os.path.exists(projdir))

        # test2: directory exists, setup fails
        res = manager.setup(verbose=False, unattended=True, argv=argv)
        self.assertFalse(res)
        shutil.rmtree(projdir)

        # test2: no argv, rely on sys.argv, assert False
        with patch('ferenda.manager.sys.argv'):
            self.assertFalse(manager.setup())
            self.assertFalse(os.path.exists(projdir))

        # test3: preflight fails
        with patch('ferenda.manager._preflight_check', return_value=False):
            self.assertFalse(manager.setup(unattended=True, argv=argv))
            self.assertFalse(os.path.exists(projdir))

            with patch('ferenda.manager.input',
                       return_value="n") as input_mock:
                self.assertFalse(manager.setup(unattended=False, argv=argv))
                self.assertFalse(os.path.exists(projdir))
                self.assertTrue(input_mock.called)

        # test4: select_triplestore fails
        with patch('ferenda.manager._preflight_check', return_value=True):
            with patch('ferenda.manager._select_triplestore',
                       return_value=(False, None, None)):
                self.assertFalse(manager.setup(unattended=True, argv=argv))
                self.assertFalse(os.path.exists(projdir))

                with patch('ferenda.manager.input',
                           return_value="n") as input_mock:
                    self.assertFalse(manager.setup(unattended=False,
                                                   argv=argv))
                    self.assertFalse(os.path.exists(projdir))
                    self.assertTrue(input_mock.called)