def test_plot(self):
matplotmock = MagicMock()
mocks = {'matplotlib': matplotmock,
'matplotlib.pyplot': MagicMock()}
with patch.dict('sys.modules', mocks):
self.analyzer.metrics(plotpath="foo/bar/baz")
self.assertTrue(matplotmock.pyplot.savefig.called)
def test_toc(self):
# tests the main TOC method, not the helper methods (they are
# tested separately)
self.repo.facets = MagicMock()
self.repo.facet_select = MagicMock()
self.repo.facet_query = MagicMock()
self.repo.faceted_data = MagicMock()
self.repo.log = Mock()
self.repo.toc_pagesets = Mock()
self.repo.toc_select_for_pages = Mock()
self.repo.toc_generate_pages = Mock()
self.repo.toc_generate_first_page = Mock()
with patch('json.dump'):
self.repo.toc()
# assert facet_query was properly called, error and info msg
# was printed
self.assertEqual("http://localhost:8000/dataset/base",
self.repo.facet_query.call_args[0][0])
self.assertTrue(self.repo.log.error.called)
self.assertTrue(self.repo.log.info.called)
# and that the rest of the methods were NOT called
self.assertFalse(self.repo.toc_pagesets.called)
self.assertFalse(self.repo.toc_select_for_pages.called)
self.assertFalse(self.repo.toc_generate_pages.called)
# test2: facet_select returns something
self.repo.faceted_data.return_value = ["fake", "data"]
with patch('json.load'):
self.repo.toc()
# Now all other methods should be called
self.assertTrue(self.repo.toc_pagesets.called)
self.assertTrue(self.repo.toc_select_for_pages.called)
self.assertTrue(self.repo.toc_generate_pages.called)
def test_render(self,mock_graph):
@render
def testfunc(repo,doc):
pass
mockdoc = Mock()
mockrepo = Mock()
mockrepo.store.parsed_path.return_value = "parsed_path.xhtml"
with open("parsed_path.xhtml", "w") as fp:
fp.write("""<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML+RDFa 1.0//EN" "http://www.w3.org/MarkUp/DTD/xhtml-rdfa-1.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xmlns:dct="http://purl.org/dc/terms/">
<head about="http://example.org/doc">
<title property="dct:title">Document title</title>
</head>
<body>
<h1>Hello!</h1>
</body>
</html>""")
mockrepo.store.distilled_path.return_value = "distilled_path.xhtml"
mockrepo.get_globals.return_value = {'symbol table':'fake'}
mockdoc.meta = MagicMock() # need Magicmock which supports magic funcs like __iter__
bodypart = MagicMock()
bodypart.meta = MagicMock()
mockdoc.body = [bodypart]
mockdoc.meta.__iter__.return_value = []
mockdoc.uri = "http://example.org/doc"
with patch('ferenda.util.ensure_dir', return_value=True):
testfunc(mockrepo, mockdoc)
# 1 ensure that DocumentRepository.render_xhtml is called with
# four arguments
mockrepo.render_xhtml.assert_called_with(mockdoc, "parsed_path.xhtml")
# 2 ensure that DocumentRepository.create_external_resources
# is called with 1 argument
mockrepo.create_external_resources.assert_called_with(mockdoc)
# 3 ensure that a Graph object is created, its parse and
# serialize methods called
# FIXME: Why doesn't the patching work?!
# self.assertTrue(mock_graph().parse.called)
# self.assertTrue(mock_graph().serialize.called)
# (4. ensure that a warning gets printed if doc.meta and
# distilled_graph do not agree)
mock_graph().__iter__.return_value = ['a','b']
mockdoc.meta.__iter__.return_value = ['a','b','c']
mockdoc.meta.serialize.return_value = b"<c>"
with patch('ferenda.util.ensure_dir', return_value=True):
testfunc(mockrepo, mockdoc)
self.assertTrue(mockrepo.log.warning.called)
os.remove("parsed_path.xhtml")
os.remove("distilled_path.xhtml")
def test_dumpstore(self):
d = Devel()
d.config = Mock()
# only test that Triplestore is called correctly, mock any
# calls to any real database
config = {"connect.return_value": Mock(**{"get_serialized.return_value": b"[fake store content]"})}
printmock = MagicMock()
with patch("ferenda.devel.TripleStore", **config):
with patch(builtins + ".print", printmock):
d.dumpstore(format="trix")
want = "[fake store content]"
printmock.assert_has_calls([call(want)])
def test_drawboxes(self):
pypdfmock = MagicMock()
canvasmock = MagicMock()
mocks = {'PyPDF2': pypdfmock,
'reportlab': MagicMock(),
'reportlab.pdfgen': MagicMock(),
'reportlab.pdfgen.canvas': canvasmock}
with patch.dict('sys.modules', mocks):
metrics = self.analyzer.metrics()
pdfpath = "test/files/pdfanalyze/lipsum.debug.pdf"
self.analyzer.drawboxes(pdfpath, metrics=metrics)
self.assertTrue(canvasmock.Canvas.called)
self.assertTrue(pypdfmock.PdfFileReader.called)
self.assertTrue(pypdfmock.PdfFileWriter.called)
util.robust_remove(pdfpath)
def test_dumpstore(self):
d = Devel()
d.config = Mock()
# only test that Triplestore is called correctly, mock any
# calls to any real database
config = {
'connect.return_value':
Mock(**{'get_serialized.return_value': b'[fake store content]'})
}
printmock = MagicMock()
with patch('ferenda.devel.TripleStore', **config):
with patch('builtins.print', printmock):
d.dumpstore(format="trix")
want = "[fake store content]"
printmock.assert_has_calls([call(want)])
def test_queryindex(self):
res = [{
'label': 'Doc #1',
'uri': 'http://example.org/doc1',
'text': 'matching doc 1'
}, {
'label': 'Doc #2',
'uri': 'http://example.org/doc2',
'text': 'matching doc 2'
}]
pager = None
config = {
'connect.return_value':
Mock(**{'query.return_value': (res, pager)})
}
printmock = MagicMock()
with patch('ferenda.devel.FulltextIndex', **config):
with patch('builtins.print', printmock):
d = Devel()
d.config = LayeredConfig(
Defaults({
'indextype': 'a',
'indexlocation': 'b'
}))
d.queryindex("doc")
want = """
Doc #1 (http://example.org/doc1): matching doc 1
Doc #2 (http://example.org/doc2): matching doc 2
""".strip()
got = "\n".join([x[1][0] for x in printmock.mock_calls])
self.maxDiff = None
self.assertEqual(want, got)
def test_select(self):
uri = "http://example.org/doc"
with open("testselecttemplate.rq", "wb") as fp:
fp.write("""PREFIX dcterms: <http://purl.org/dc/terms/>
SELECT ?p ?o
WHERE { <%(uri)s> ?p ?o . }
""".encode())
result = """
[
{
"p": "http://purl.org/dc/terms/title",
"o": "Document title"
},
{
"p": "http://purl.org/dc/terms/identifier",
"o": "Document ID"
}
]""".lstrip().encode("utf-8")
config = {
'connect.return_value': Mock(**{'select.return_value': result})
}
printmock = MagicMock()
with patch('ferenda.devel.TripleStore', **config):
with patch('builtins.print', printmock):
d = Devel()
d.config = LayeredConfig(
Defaults({
'storetype': 'a',
'storelocation': 'b',
'storerepository': 'c'
}))
d.select("testselecttemplate.rq", uri)
want = """
# Constructing the following from b, repository c, type a
# PREFIX dcterms: <http://purl.org/dc/terms/>
#
# SELECT ?p ?o
# WHERE { <http://example.org/doc> ?p ?o . }
#
[
{
"p": "http://purl.org/dc/terms/title",
"o": "Document title"
},
{
"p": "http://purl.org/dc/terms/identifier",
"o": "Document ID"
}
]
# Selected in 0.001s
""".strip()
got = "\n".join([x[1][0] for x in printmock.mock_calls])
self.maxDiff = None
self.assertEqual(self.mask_time(want), self.mask_time(got))
os.unlink("testselecttemplate.rq")
def test_dumprdf(self):
fileno, tmpfile = mkstemp()
fp = os.fdopen(fileno, "w")
fp.write("""<html xmlns="http://www.w3.org/1999/xhtml">
<head about="http://example.org/doc">
<title property="http://purl.org/dc/terms/">Doc title</title>
</head>
<body>...</body>
</html>""")
fp.close()
d = Devel()
mock = MagicMock()
with patch('builtins.print', mock):
d.dumprdf(tmpfile, format="nt")
os.unlink(tmpfile)
self.assertTrue(mock.called)
want = '<http://example.org/doc> <http://purl.org/dc/terms/> "Doc title" .\n\n'
mock.assert_has_calls([call(want)])
def test_dumprdf(self):
fileno, tmpfile = mkstemp()
fp = os.fdopen(fileno, "w")
fp.write("""<html xmlns="http://www.w3.org/1999/xhtml">
<head about="http://example.org/doc">
<title property="http://purl.org/dc/terms/">Doc title</title>
</head>
<body>...</body>
</html>""")
fp.close()
d = Devel()
mock = MagicMock()
with patch('builtins.print', mock):
d.dumprdf(tmpfile, format="nt")
os.unlink(tmpfile)
self.assertTrue(mock.called)
want = '<http://example.org/doc> <http://purl.org/dc/terms/> "Doc title" .\n\n'
mock.assert_has_calls([call(want)])
def test_news(self):
# should test the main method, not the helpers. That'll
# require mocking most methods.
self.repo.news_facet_entries = MagicMock()
self.repo.facets = MagicMock()
self.repo.news_feedsets = Mock()
self.repo.news_select_for_feeds = Mock()
self.repo.news_generate_feeds = Mock()
# this isn't really a good test -- it only verifies the
# internal implementation of the main method not the
# behaviour. But other tests verifieds behaviour of individual
# methods.
self.repo.news()
assert (self.repo.news_facet_entries.called)
assert (self.repo.facets.called)
assert (self.repo.news_feedsets.called)
assert (self.repo.news_select_for_feeds.called)
assert (self.repo.news_generate_feeds.called)
def test_downloadmax(self):
@downloadmax
def testfunc(repo, source):
for x in range(100):
yield x
mockrepo = MagicMock()
mockrepo.config.downloadmax = None
self.assertEqual(100, len(list(testfunc(mockrepo, None))))
mockrepo.config.downloadmax = 10
mockrepo.config.__contains__.return_value = True
self.assertEqual(10, len(list(testfunc(mockrepo, None))))
def test_handleerror(self):
@handleerror
def testfunc(repo, doc):
if doc.exception:
raise doc.exception
else:
return True
mockrepo = Mock()
mockrepo.config = MagicMock() # must support __in__()
mockdoc = Mock()
# 1. should not raise an exception (but should call log.info
# and util.robust_remove, and return false)
with patch('ferenda.util.robust_remove') as robust_remove:
mockdoc.exception = DocumentRemovedError
self.assertFalse(testfunc(mockrepo, mockdoc))
self.assertTrue(mockrepo.log.info.called)
self.assertTrue(robust_remove.called)
# 2. should raise the same exception
mockdoc.exception = KeyboardInterrupt
with self.assertRaises(KeyboardInterrupt):
testfunc(mockrepo, mockdoc)
# 3.1 Should raise the same exeption
mockdoc.exception = ParseError
mockrepo.config.fatalexceptions = True
with self.assertRaises(ParseError):
testfunc(mockrepo, mockdoc)
mockrepo.config.fatalexceptions = None
# 3.2 Should not raise an exception (but should call log.error and return false)
mockdoc.exception = ParseError
self.assertFalse(testfunc(mockrepo, mockdoc))
self.assertTrue(mockrepo.log.error.called)
# 4.1 Should raise the same exception
mockdoc.exception = Exception
mockrepo.config.fatalexceptions = True
with self.assertRaises(Exception):
testfunc(mockrepo, mockdoc)
mockrepo.config.fatalexceptions = None
# 4.2 Should not raise an exception
mockdoc.exception = Exception
self.assertFalse(testfunc(mockrepo, mockdoc))
self.assertTrue(mockrepo.log.error.called)
# 5. No exceptions - everything should go fine
mockdoc.exception = None
self.assertTrue(testfunc(mockrepo, mockdoc))
def test_construct(self):
uri = "http://example.org/doc"
with open("testconstructtemplate.rq", "wb") as fp:
fp.write("""PREFIX dcterms: <http://purl.org/dc/terms/>
CONSTRUCT { ?s ?p ?o . }
WHERE { ?s ?p ?o .
<%(uri)s> ?p ?o . }
""".encode())
g = Graph()
g.bind("dcterms", str(DCTERMS))
g.add((URIRef(uri), DCTERMS.title, Literal("Document title")))
config = {
'connect.return_value': Mock(**{'construct.return_value': g})
}
printmock = MagicMock()
with patch('ferenda.devel.TripleStore', **config):
with patch('builtins.print', printmock):
d = Devel()
d.config = LayeredConfig(
Defaults({
'storetype': 'a',
'storelocation': 'b',
'storerepository': 'c'
}))
d.construct("testconstructtemplate.rq", uri)
want = """
# Constructing the following from b, repository c, type a
# PREFIX dcterms: <http://purl.org/dc/terms/>
#
# CONSTRUCT { ?s ?p ?o . }
# WHERE { ?s ?p ?o .
# <http://example.org/doc> ?p ?o . }
#
@prefix dcterms: <http://purl.org/dc/terms/> .
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@prefix xml: <http://www.w3.org/XML/1998/namespace> .
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
<http://example.org/doc> dcterms:title "Document title" .
# 1 triples constructed in 0.001s
""".strip()
got = "\n".join([x[1][0] for x in printmock.mock_calls])
self.maxDiff = None
self.assertEqual(self.mask_time(want), self.mask_time(got))
os.unlink("testconstructtemplate.rq")
def test_render(self, mock_graph):
@render
def testfunc(repo, doc):
pass
mockdoc = Mock()
mockrepo = MagicMock()
mockrepo.store.parsed_path.return_value = "parsed_path.xhtml"
with open("parsed_path.xhtml", "w") as fp:
fp.write("""<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML+RDFa 1.0//EN" "http://www.w3.org/MarkUp/DTD/xhtml-rdfa-1.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xmlns:dcterms="http://purl.org/dc/terms/">
<head about="http://example.org/doc">
<title property="dcterms:title">Document title</title>
</head>
<body>
<h1>Hello!</h1>
</body>
</html>""")
with open("entry_path.json", "w") as fp:
fp.write("""{
"id": "https://lagen.nu/concept/St\\u00e5ende_anbud",
"title": "St\\u00e5ende anbud",
}""")
mockrepo.store.distilled_path.return_value = "distilled_path.xhtml"
mockrepo.get_globals.return_value = {'symbol table': 'fake'}
mockrepo.required_predicates = []
mockrepo.config = MagicMock()
mockdoc.meta = MagicMock(
) # need Magicmock which supports magic funcs like __iter__
bodypart = MagicMock()
bodypart.meta = MagicMock()
mockdoc.body = [bodypart]
mockdoc.meta.__iter__.return_value = []
mockdoc.uri = "http://example.org/doc"
mockrepo.store.documententry_path.return_value = None
with patch('ferenda.util.ensure_dir', return_value=True):
testfunc(mockrepo, mockdoc)
# 1 ensure that DocumentRepository.render_xhtml is called with
# four arguments
mockrepo.render_xhtml.assert_called_with(mockdoc, "parsed_path.xhtml")
# 2 ensure that DocumentRepository.create_external_resources
# is called with 1 argument
mockrepo.create_external_resources.assert_called_with(mockdoc)
# 3 ensure that a Graph object is created, its parse and
# serialize methods called
# FIXME: Why doesn't the patching work?!
# self.assertTrue(mock_graph().parse.called)
# self.assertTrue(mock_graph().serialize.called)
# (4. ensure that a warning gets printed if doc.meta and
# distilled_graph do not agree)
mock_graph().__iter__.return_value = ['a', 'b']
mockdoc.meta.__iter__.return_value = ['a', 'b', 'c']
mockdoc.meta.serialize.return_value = b"<c>"
with patch('ferenda.util.ensure_dir', return_value=True):
testfunc(mockrepo, mockdoc)
self.assertTrue(mockrepo.log.warning.called)
os.remove("parsed_path.xhtml")
os.remove("distilled_path.xhtml")
os.remove("entry_path.json")
def test_fsmparse(self):
try:
# 1. write a new python module containing a class with a staticmethod
with open("testparser.py", "w") as fp:
fp.write("""
from six import text_type as str
from ferenda.elements import Body, Paragraph
class Testobject(object):
@staticmethod
def get_parser():
return Parser()
class Parser(object):
def parse(self, source):
res = Body()
for chunk in source:
res.append(Paragraph([str(len(chunk.strip()))]))
return res
""")
import imp
fp, pathname, desc = imp.find_module("testparser")
imp.load_module("testparser", fp, pathname, desc)
# 2. write a textfile with two paragraphs
with open("testparseinput.txt", "w") as fp:
fp.write("""This is one paragraph.
And another.
""")
# 3. patch print and call fsmparse
d = Devel()
printmock = MagicMock()
with patch('builtins.print', printmock):
# 3.1 fsmparse dynamically imports the module and call the method
# with every chunk from the text file
# 3.2 fsmparse asserts that the method returned a callable
# 3.3 fsmparse calls it with a iterable of text chunks from the
# textfile
# 3.4 fsmparse recieves a Element structure and prints a
# serialized version
d.fsmparse("testparser.Testobject.get_parser",
"testparseinput.txt")
self.assertTrue(printmock.called)
# 4. check that the expected thing was printed
want = """
<Body>
<Paragraph>
<str>22</str>
</Paragraph>
<Paragraph>
<str>12</str>
</Paragraph>
</Body>
""".strip() + "\n"
printmock.assert_has_calls([call(want)])
finally:
util.robust_remove("testparser.py")
util.robust_remove("testparser.pyc")
util.robust_remove("testparseinput.txt")
if os.path.exists("__pycache__") and os.path.isdir("__pycache__"):
shutil.rmtree("__pycache__")
def test_fsmparse(self):
try:
# 1. write a new python module containing a class with a staticmethod
with open("testparser.py", "w") as fp:
fp.write("""
from six import text_type as str
from ferenda.elements import Body, Paragraph
class Testobject(object):
@staticmethod
def get_parser():
return Parser()
class Parser(object):
def parse(self, source):
res = Body()
for chunk in source:
res.append(Paragraph([str(len(chunk.strip()))]))
return res
""")
import imp
fp, pathname, desc = imp.find_module("testparser")
imp.load_module("testparser", fp, pathname, desc)
# 2. write a textfile with two paragraphs
with open("testparseinput.txt", "w") as fp:
fp.write("""This is one paragraph.
And another.
""")
# 3. patch print and call fsmparse
d = Devel()
printmock = MagicMock()
with patch('builtins.print', printmock):
# 3.1 fsmparse dynamically imports the module and call the method
# with every chunk from the text file
# 3.2 fsmparse asserts that the method returned a callable
# 3.3 fsmparse calls it with a iterable of text chunks from the
# textfile
# 3.4 fsmparse recieves a Element structure and prints a
# serialized version
d.fsmparse("testparser.Testobject.get_parser", "testparseinput.txt")
self.assertTrue(printmock.called)
# 4. check that the expected thing was printed
want = """
<Body>
<Paragraph>
<str>22</str>
</Paragraph>
<Paragraph>
<str>12</str>
</Paragraph>
</Body>
""".strip()+"\n"
printmock.assert_has_calls([call(want)])
finally:
util.robust_remove("testparser.py")
util.robust_remove("testparser.pyc")
util.robust_remove("testparseinput.txt")
if os.path.exists("__pycache__") and os.path.isdir("__pycache__"):
shutil.rmtree("__pycache__")
