def test_runserver(self):
self._enable_repos()
m = Mock()
with patch('ferenda.manager.make_server', return_value=m) as m2:
manager.run(["all", "runserver"])
self.assertTrue(m2.called)
self.assertTrue(m.serve_forever.called)
def test_named_logfile(self):
self._enable_repos()
self.assertFalse(os.path.exists("out.log"))
argv = ["test","mymethod","myarg","--logfile=out.log"]
manager.run(argv)
self.assertTrue(os.path.exists("out.log"))
os.unlink("out.log")
def _enable_repos(self):
# 3. run('example.Testrepo', 'enable')
with patch.object(logging.Logger, 'info') as mocklog:
self.assertEqual(
"test", manager.run([self.modulename + ".Testrepo", "enable"]))
# 4. verify that "alias foo enabled" is logged
log_arg = mocklog.call_args[0][0]
self.assertEqual(
"Enabled class %s.Testrepo (alias 'test')" % self.modulename,
log_arg)
# 5. verify that ferenda.ini has changed
cfg = configparser.ConfigParser()
cfg.read(["ferenda.ini"])
self.assertEqual(cfg.get("test", "class"),
self.modulename + ".Testrepo")
# (same, with 'example.Testrepo2')
self.assertEqual(
"test2",
manager.run([self.modulename + ".Testrepo2", "enable"]))
cfg = configparser.ConfigParser()
cfg.read(["ferenda.ini"])
self.assertEqual(cfg.get("test2", "class"),
self.modulename + ".Testrepo2")
with patch.object(logging.Logger, 'error') as mocklog:
# 6. run('example.Nonexistent', 'enable') -- the ImportError must
# be caught and an error printed.
manager.run([self.modulename + ".Nonexistent", "enable"])
# 7. verify that a suitable error messsage is logged
self.assertEqual(
"No class named '%s.Nonexistent'" % self.modulename,
mocklog.call_args[0][0])
def test_run_makeresources(self):
# 1. setup test_run_enable
# 2. run('all', 'makeresources')
# 3. verify that all css/jss files specified by default and in Testrepo gets copied
# (remove rsrc)
# 4. run('all', 'makeresources', '--combine')
# 5. verify that single css and js file is created
self._enable_repos()
s = os.sep
want = {'css':[s.join(['rsrc', 'css','test.css']),
s.join(['rsrc', 'css','other.css'])],
'js':[s.join(['rsrc', 'js','test.js'])],
'xml':[s.join(['rsrc', 'resources.xml'])]
}
got = manager.run(['all', 'makeresources'])
self.assertEqual(want,got)
# 6. alter the ferenda.ini so that it doesn't specify any css/js files
util.writefile("ferenda.ini", """[__root__]
loglevel=WARNING
datadir = %s
url = http://localhost:8000
searchendpoint = /search/
apiendpoint = /api/
""" % self.tempdir)
want = {'css':[],
'js':[],
'xml':[s.join(['rsrc', 'resources.xml'])]
}
got = manager.run(['all', 'makeresources'])
self.assertEqual(want,got)
def test_named_logfile(self):
self._enable_repos()
self.assertFalse(os.path.exists("out.log"))
argv = ["test", "mymethod", "myarg", "--logfile=out.log"]
manager.run(argv)
self.assertTrue(os.path.exists("out.log"))
os.unlink("out.log")
def _enable_repos(self):
# 3. run('example.Testrepo', 'enable')
with patch.object(logging.Logger, 'info') as mocklog:
self.assertEqual("test",
manager.run([self.modulename+".Testrepo", "enable"]))
# 4. verify that "alias foo enabled" is logged
log_arg = mocklog.call_args[0][0]
self.assertEqual("Enabled class %s.Testrepo (alias 'test')" % self.modulename,
log_arg)
# 5. verify that ferenda.ini has changed
cfg = configparser.ConfigParser()
cfg.read(["ferenda.ini"])
self.assertEqual(cfg.get("test","class"), self.modulename+".Testrepo")
# (same, with 'example.Testrepo2')
self.assertEqual("test2",
manager.run([self.modulename+".Testrepo2", "enable"]))
cfg = configparser.ConfigParser()
cfg.read(["ferenda.ini"])
self.assertEqual(cfg.get("test2","class"), self.modulename+".Testrepo2")
with patch.object(logging.Logger, 'error') as mocklog:
# 6. run('example.Nonexistent', 'enable') -- the ImportError must
# be caught and an error printed.
manager.run([self.modulename+".Nonexistent", "enable"])
# 7. verify that a suitable error messsage is logged
self.assertEqual("No class named '%s.Nonexistent'" % self.modulename,
mocklog.call_args[0][0])
def test_runserver(self):
self._enable_repos()
m = Mock()
with patch('ferenda.manager.make_server', return_value=m) as m2:
manager.run(["all", "runserver"])
self.assertTrue(m2.called)
self.assertTrue(m.serve_forever.called)
def test_run_single_all(self):
self._enable_repos()
argv = ["test", "mymethod", "--all"]
# Test 1: make sure that if setup signals that no work should
# be done, this is respected
with patch("example.Testrepo.setup", return_value=False):
self.assertEqual(manager.run(list(argv)), [])
# pass
# Test 2: but if not, do the work
self.assertEqual(manager.run(list(argv)), [None, "ok!", None])
def test_run_single_all(self):
self._enable_repos()
argv = ["test","mymethod","--all"]
# Test 1: make sure that if setup signals that no work should
# be done, this is respected
with patch("example.Testrepo.setup", return_value=False):
self.assertEqual(manager.run(list(argv)), [])
# pass
# Test 2: but if not, do the work
self.assertEqual(manager.run(list(argv)), [None, "ok!", None])
def test_run_all_allmethods(self):
self._enable_repos()
argv = ["all", "all", "--magic=more"]
s = os.sep
want = OrderedDict(
[('download', OrderedDict([('test','test download ok (magic=more)'),
('test2', 'test2 download ok (magic=more)')])),
('parse', OrderedDict([('test', ['test parse arg1',
'test parse myarg',
'test parse arg2']),
('test2', ['test2 parse arg1',
'test2 parse myarg',
'test2 parse arg2'])])),
('relate', OrderedDict([('test', ['test relate arg1',
'test relate myarg',
'test relate arg2']),
('test2', ['test2 relate arg1',
'test2 relate myarg',
'test2 relate arg2'])])),
('makeresources', {'css':[s.join(['rsrc', 'css','test.css']),
s.join(['rsrc', 'css','other.css'])],
'js':[s.join(['rsrc', 'js','test.js'])],
'img':[s.join(['rsrc', 'img','test.png'])],
'json': [s.join(['rsrc','api','context.json']),
s.join(['rsrc','api','common.json']),
s.join(['rsrc','api','terms.json'])],
'xml':[s.join(['rsrc', 'resources.xml'])]}),
('toc', OrderedDict([('test','test toc ok'),
('test2', 'test2 toc ok')])),
('generate', OrderedDict([('test', ['test generate arg1',
'test generate myarg',
'test generate arg2']),
('test2', ['test2 generate arg1',
'test2 generate myarg',
'test2 generate arg2'])])),
('transformlinks', OrderedDict([('test', ['test transformlinks arg1',
'test transformlinks myarg',
'test transformlinks arg2']),
('test2', ['test2 transformlinks arg1',
'test2 transformlinks myarg',
'test2 transformlinks arg2'])])),
('news', OrderedDict([('test','test news ok'),
('test2', 'test2 news ok')])),
('frontpage', True)])
got = manager.run(argv)
self.maxDiff = None
self.assertEqual(want,got)
# 7. add an unrelated command line flag and verify that this
# does not interfere with the processing
argv.append('--downloadmax=50')
got = manager.run(argv)
self.assertEqual(want,got)
def test_noclobber(self):
manager.run([self.modulename + ".Testrepo", "enable"])
manager.run([self.modulename + ".Testrepo", "save"])
cfg = configparser.ConfigParser()
cfg.read(["ferenda.ini"])
# make sure cfg has one section for testrepo and only two
# attributes ('class' was created when enabling the module,
# saved was created in the 'save' call)
self.assertEqual(
set((('class', 'example.Testrepo'), ('saved', 'True'))),
set(cfg.items('test')))
def test_noclobber(self):
manager.run([self.modulename+".Testrepo", "enable"])
manager.run([self.modulename+".Testrepo", "save"])
cfg = configparser.ConfigParser()
cfg.read(["ferenda.ini"])
# make sure cfg has one section for testrepo and only two
# attributes ('class' was created when enabling the module,
# saved was created in the 'save' call)
self.assertEqual(set((('class', 'example.Testrepo'),
('saved', 'True'))),
set(cfg.items('test')))
def test_run_makeresources(self):
# 1. setup test_run_enable
# 2. run('all', 'makeresources')
# 3. verify that all css/jss files specified by default and in
# Testrepo gets copied (remove rsrc)
# 4. run('all', 'makeresources', '--combine')
# 5. verify that single css and js file is created
self._enable_repos()
s = os.sep
want = {
'css': [
s.join(['rsrc', 'css', 'test.css']),
s.join(['rsrc', 'css', 'other.css'])
],
'js': [s.join(['rsrc', 'js', 'test.js'])],
'img': [s.join(['rsrc', 'img', 'test.png'])],
'json': [
s.join(['rsrc', 'api', 'context.json']),
s.join(['rsrc', 'api', 'common.json']),
s.join(['rsrc', 'api', 'terms.json'])
],
'xml': [s.join(['rsrc', 'resources.xml'])]
}
got = manager.run(['all', 'makeresources'])
self.assertEqual(want, got)
# 6. alter the ferenda.ini so that it doesn't specify any css/js files
util.writefile(
"ferenda.ini", """[__root__]
loglevel=WARNING
datadir = %s
url = http://localhost:8000/
searchendpoint = /search/
apiendpoint = /api/
cssfiles = []
jsfiles = []
imgfiles = []
""" % self.tempdir)
want = {
'css': [],
'js': [],
'img': [],
'json': [
s.join(['rsrc', 'api', 'context.json']),
s.join(['rsrc', 'api', 'common.json']),
s.join(['rsrc', 'api', 'terms.json'])
],
'xml': [s.join(['rsrc', 'resources.xml'])]
}
got = manager.run(['all', 'makeresources'])
self.assertEqual(want, got)
def test_run_all_all(self):
self._enable_repos()
argv = ["all", "mymethod", "--all"]
# FIXME: Check to see if proper setup and teardown on Testrepo/Testrepo2 was properly called (how?)
self.assertEqual(manager.run(argv),
[[None,"ok!",None],
[None,"yeah!",None]])
def test_print_usage(self):
self._enable_repos()
with patch('builtins.print') as printmock:
manager.run([])
executable = sys.argv[0]
got = "\n".join([x[1][0] for x in printmock.mock_calls])
got = got.replace(executable, "[EXEC]")
want = """Usage: [EXEC] [class-or-alias] [action] <arguments> <options>
e.g. '[EXEC] ferenda.sources.EurlexCaselaw enable'
'[EXEC] ecj parse 62008J0042'
'[EXEC] all generate'
Available modules:
* test: [Undocumented]
* test2: [Undocumented]"""
self.assertEqual(got, want)
def test_print_usage(self):
self._enable_repos()
with patch('builtins.print') as printmock:
manager.run([])
executable = sys.argv[0]
got = "\n".join([x[1][0] for x in printmock.mock_calls])
got = got.replace(executable, "[EXEC]")
want = """Usage: [EXEC] [class-or-alias] [action] <arguments> <options>
e.g. '[EXEC] ferenda.sources.EurlexCaselaw enable'
'[EXEC] ecj parse 62008J0042'
'[EXEC] all generate'
Available modules:
* test: [Undocumented]
* test2: [Undocumented]"""
self.assertEqual(got, want)
def test_server(self):
self._enable_repos()
# create two out-of-process clients
foo = Popen(['python', 'ferenda-build.py', 'all',
'buildclient', '--clientname=foo', '--processes=2'],
stderr=PIPE)
bar = Popen(['python', 'ferenda-build.py', 'all',
'buildclient', '--clientname=bar', '--processes=2'],
stderr=PIPE)
try:
# run an in-process server
argv = ["test", "pid", "--all", "--buildserver", "--loglevel=CRITICAL"]
res = manager.run(argv)
# same tests as for RunMultiproc.test_run_single_all
args = [x[0] for x in res]
pids = [x[1] for x in res]
# we're not guaranteed any particular order, therefore we
# compare sets instead of lists
self.assertEqual(set(args), set(["arg1", "myarg", "arg2"]))
self.assertEqual(3, len(set(pids)))
finally:
for p in foo, bar:
p.terminate()
p.wait()
p.stderr.close()
def test_run_single_allmethods(self):
self._enable_repos()
argv = ["test","all"]
s = os.sep
self.maxDiff = None
want = OrderedDict(
[('download', OrderedDict([('test','test download ok (magic=less)'),
])),
('parse', OrderedDict([('test', ['test parse arg1',
'test parse myarg',
'test parse arg2']),
])),
('relate', OrderedDict([('test', ['test relate arg1',
'test relate myarg',
'test relate arg2']),
])),
('makeresources', {'css':[s.join(['rsrc', 'css','test.css']),
s.join(['rsrc', 'css','other.css'])],
'js':[s.join(['rsrc', 'js','test.js'])],
'xml':[s.join(['rsrc', 'resources.xml'])]}),
('generate', OrderedDict([('test', ['test generate arg1',
'test generate myarg',
'test generate arg2']),
])),
('toc', OrderedDict([('test','test toc ok'),
])),
('news', OrderedDict([('test','test news ok'),
])),
('frontpage', True)])
self.assertEqual(manager.run(argv),
want)
def test_server_all(self):
# run multiple repos, one after the other, on the same method
# and make sure the different methods are run.
self._enable_repos()
# create two out-of-process clients
foo = Popen(['python', 'ferenda-build.py', 'all',
'buildclient', '--clientname=foo', '--processes=2'],
stderr=PIPE)
bar = Popen(['python', 'ferenda-build.py', 'all',
'buildclient', '--clientname=bar', '--processes=2'],
stderr=PIPE)
try:
# run an in-process server
argv = ["all", "pid", "--all", "--buildserver", "--loglevel=CRITICAL"]
res = manager.run(argv)
# we should have two main sets of results, the first from
# Testrepo, the second from Testrepo2
self.assertEqual(2, len(res))
# same tests as for RunMultiproc.test_run_single_all
args1 = [x[0] for x in res[0]]
pids1 = [x[1] for x in res[0]]
self.assertEqual(set(args1), set(["arg1", "myarg", "arg2"]))
self.assertEqual(3, len(set(pids1)))
args2 = [x[0] for x in res[1]]
pids2 = [x[1] for x in res[1]]
self.assertEqual(set(args2), set(["repo2:arg1", "repo2:myarg", "repo2:arg2"]))
self.assertEqual(3, len(set(pids2)))
finally:
for p in foo, bar:
p.terminate()
p.wait()
p.stderr.close()
def test_run_makeresources_defaultconfig(self):
util.resource_extract(
ResourceLoader(), 'scripts/ferenda.template.ini', "ferenda.ini", {
'storetype': 'SQLITE',
'storelocation': 'data/ferenda.sqlite',
'storerepository': 'ferenda',
'indextype': 'WHOOSH',
'indexlocation': 'data/whooshindex',
'sitename': 'Test'
})
self._enable_repos()
s = os.sep
got = manager.run(['all', 'makeresources', '--loglevel=CRITICAL'])
want = {
'xml': [s.join(['rsrc', 'resources.xml'])],
'json': [
s.join(['rsrc', 'api', 'context.json']),
s.join(['rsrc', 'api', 'common.json']),
s.join(['rsrc', 'api', 'terms.json'])
],
'img': [
s.join(['rsrc', 'img', 'atom.png']),
s.join(['rsrc', 'img', 'test.png'])
],
'css': [
s.join(['rsrc', 'css', 'ferenda.css']),
s.join(['rsrc', 'css', 'test.css'])
],
'js': [
s.join(['rsrc', 'js', 'ferenda.js']),
s.join(['rsrc', 'js', 'test.js'])
]
}
self.assertEqual(want, got)
def test_server(self):
self._enable_repos()
# create two out-of-process clients
foo = Popen([
'python', 'ferenda-build.py', 'all', 'buildclient',
'--clientname=foo', '--processes=2'
],
stderr=PIPE)
bar = Popen([
'python', 'ferenda-build.py', 'all', 'buildclient',
'--clientname=bar', '--processes=2'
],
stderr=PIPE)
try:
# run an in-process server
argv = [
"test", "pid", "--all", "--buildserver", "--loglevel=CRITICAL"
]
res = manager.run(argv)
# same tests as for RunMultiproc.test_run_single_all
args = [x[0] for x in res]
pids = [x[1] for x in res]
# we're not guaranteed any particular order, therefore we
# compare sets instead of lists
self.assertEqual(set(args), set(["arg1", "myarg", "arg2"]))
self.assertEqual(3, len(set(pids)))
finally:
for p in foo, bar:
p.terminate()
p.wait()
p.stderr.close()
def test_run_single(self):
# test1: run standard (custom) method
self._enable_repos()
argv = ["test", "mymethod", "myarg"]
self.assertEqual(manager.run(argv), "ok!")
# test2: specify invalid alias
argv[0] = "invalid"
with patch('ferenda.manager.setup_logger'):
self.assertEqual(manager.run(argv), None)
with patch('builtins.print') as printmock:
with patch('ferenda.manager.setup_logger'):
# test3: specify invalid method
argv = ["test", "invalid"]
self.assertEqual(manager.run(argv), None)
def update(self, article):
"""Update all generated pages that are dependent on/include the given wiki article."""
self.config.force = True
self.config.refresh = True
self.download(article)
self.parse(article)
self.relate(article)
if article.startswith("SFS/"):
self.sfsrepo.config.force = True
self.sfsrepo.generate(article.split("/", 1)[1])
elif article == "Lagen.nu:Huvudsida":
# FIXME: This re-downloads and re-parses the wiki page
from ferenda import manager
manager.run(["all", "frontpage"], self.config._parent)
else:
self.keywordrepo.config.force = True
self.keywordrepo.generate(article)
def test_run_single(self):
# test1: run standard (custom) method
self._enable_repos()
argv = ["test","mymethod","myarg"]
self.assertEqual(manager.run(argv),
"ok!")
# test2: specify invalid alias
argv[0] = "invalid"
with patch('ferenda.manager.setup_logger'):
self.assertEqual(manager.run(argv), None)
with patch('builtins.print') as printmock:
with patch('ferenda.manager.setup_logger'):
# test3: specify invalid method
argv = ["test", "invalid"]
self.assertEqual(manager.run(argv), None)
def update(self, article):
"""Update all generated pages that are dependent on/include the given wiki article."""
self.config.force = True
self.config.refresh = True
self.download(article)
self.parse(article)
self.relate(article)
if article.startswith("SFS/"):
self.sfsrepo.config.force = True
self.sfsrepo.generate(article.split("/", 1)[1])
elif article == "Lagen.nu:Huvudsida":
# FIXME: This re-downloads and re-parses the wiki page
from ferenda import manager
manager.run(["all", "frontpage"], self.config._parent)
else:
self.keywordrepo.config.force = True
self.keywordrepo.generate(article)
def test_global_config(self):
# this makes sure that the subprocesses use instances that
# have access to the global/manager-provided DEFAULT_CONFIG
# config variables
self._enable_repos()
argv = ["test", "mpinspect", "--all", "--processes=2"]
res = manager.run(argv)
self.assertEqual(res, [(True, False), (True, False), (True, False)])
def test_queue(self):
# runs a separate queue-handling process (which is the only
# way to do it on windows). also, test with non-default port
self._enable_repos()
# create a out-of-process buildqueue server
queue = Popen([
'python', 'ferenda-build.py', 'all', 'buildqueue',
'--serverport=3456'
])
# create two out-of-process clients
foo = Popen([
'python', 'ferenda-build.py', 'all', 'buildclient',
'--clientname=foo', '--serverport=3456', '--processes=2'
],
stderr=PIPE)
bar = Popen([
'python', 'ferenda-build.py', 'all', 'buildclient',
'--clientname=bar', '--serverport=3456', '--processes=2'
],
stderr=PIPE)
sleep(2) # to allow both clients to spin up so that one won't
# be hogging all the jobs (since they have 2 procs
# each, that will lead to duplicated pids). NOTE: this
# does not guarantee anything
try:
# then, in-process, push jobs to that queue and watch them
# return results
argv = [
"test", "pid", "--all", "--buildqueue", "--serverport=3456",
"--loglevel=CRITICAL"
]
res = manager.run(argv)
# same tests as for RunMultiproc.test_run_single_all
args = [x[0] for x in res]
pids = [x[1] for x in res]
# we're not guaranteed any particular order, therefore we
# compare sets instead of lists
self.assertEqual(set(args), set(["arg1", "myarg", "arg2"]))
self.assertEqual(3, len(set(pids)))
finally:
# NOTE: On Win32/py2 just terminate() will not kill children of foo
# and bar (of which tree out of four are left hanging due
# to us not being able to send DONE signals to them all).
# See http://stackoverflow.com/questions/1230669/subprocess-deleting-child-processes-in-windows
if sys.platform == "win32":
import psutil
for child in psutil.Process(foo.pid).children(recursive=True):
child.kill()
for child in psutil.Process(bar.pid).children(recursive=True):
child.kill()
for p in foo, bar, queue:
p.terminate()
p.wait(
) # to clear up the process table (otherwise p lives on as zombie)
if p.stderr:
p.stderr.close()
def test_run_single_all_multiprocessing(self):
self._enable_repos()
argv = ["test", "pid", "--all", "--processes=3"]
res = manager.run(argv)
args = [x[0] for x in res]
pids = [x[1] for x in res]
self.assertEqual(args, ["arg1", "myarg", "arg2"])
# assert that all pids are unique
self.assertEqual(3, len(set(pids)))
def test_run_single_allmethods(self):
self._enable_repos()
argv = ["test", "all"]
s = os.sep
self.maxDiff = None
want = OrderedDict([
('download',
OrderedDict([
('test', 'test download ok (magic=less)'),
])),
('parse',
OrderedDict([
('test',
['test parse arg1', 'test parse myarg', 'test parse arg2']),
])),
('relate',
OrderedDict([
('test', [
'test relate arg1', 'test relate myarg',
'test relate arg2'
]),
])),
('makeresources', {
'css': [
s.join(['rsrc', 'css', 'test.css']),
s.join(['rsrc', 'css', 'other.css'])
],
'img': [s.join(['rsrc', 'img', 'test.png'])],
'js': [s.join(['rsrc', 'js', 'test.js'])],
'json': [
s.join(['rsrc', 'api', 'context.json']),
s.join(['rsrc', 'api', 'common.json']),
s.join(['rsrc', 'api', 'terms.json'])
],
'xml': [s.join(['rsrc', 'resources.xml'])]
}), ('toc', OrderedDict([
('test', 'test toc ok'),
])),
('generate',
OrderedDict([
('test', [
'test generate arg1', 'test generate myarg',
'test generate arg2'
]),
])),
('transformlinks',
OrderedDict([
('test', [
'test transformlinks arg1', 'test transformlinks myarg',
'test transformlinks arg2'
]),
])), ('news', OrderedDict([
('test', 'test news ok'),
])), ('frontpage', True)
])
self.assertEqual(want, manager.run(argv))
def test_run_single_all_multiprocessing(self):
self._enable_repos()
argv = ["test", "pid", "--all", "--processes=3"]
res = manager.run(argv)
args = [x[0] for x in res]
pids = [x[1] for x in res]
self.assertEqual(args, ["arg1", "myarg", "arg2"])
# assert that all pids are unique
self.assertEqual(3, len(set(pids)))
def test_run_single_errors(self):
self._enable_repos()
argv = ["test", "errmethod", "--all"]
with patch('ferenda.manager.setup_logger'):
with patch('builtins.print') as printmock:
res = manager.run(argv)
self.assertEqual(res[0][0], Exception)
self.assertEqual(res[1][0], errors.DocumentRemovedError)
self.assertEqual(res[2], None)
self.assertTrue(os.path.exists("dummyfile.txt"))
def test_run_single_errors(self):
self._enable_repos()
argv = ["test", "errmethod", "--all"]
with patch('ferenda.manager.setup_logger'):
with patch('builtins.print') as printmock:
res = manager.run(argv)
self.assertEqual(res[0][0], Exception)
self.assertEqual(res[1][0], errors.DocumentRemovedError)
self.assertEqual(res[2], None)
self.assertTrue(os.path.exists("dummyfile.txt"))
def test_run_single_all_multiprocessing_fail(self):
self._enable_repos()
# print("running multiproc for pid %s, datadir %s" % (os.getpid(), self.tempdir))
argv = ["test","errmethod","--all", "--processes=3", "--loglevel=CRITICAL"]
res = manager.run(argv)
# this doesn't test that errors get reported immediately
# (which they do)
self.assertEqual(res[0][0], Exception)
self.assertEqual(res[1][0], errors.DocumentRemovedError)
self.assertEqual(res[2], None)
self.assertTrue(os.path.exists("dummyfile.txt"))
def test_queue(self):
# runs a separate queue-handling process (which is the only
# way to do it on windows). also, test with non-default port
self._enable_repos()
# create a out-of-process buildqueue server
queue = Popen(['python', 'ferenda-build.py', 'all',
'buildqueue', '--serverport=3456'])
# create two out-of-process clients
foo = Popen(['python', 'ferenda-build.py', 'all',
'buildclient', '--clientname=foo', '--serverport=3456',
'--processes=2'],
stderr=PIPE)
bar = Popen(['python', 'ferenda-build.py', 'all',
'buildclient', '--clientname=bar', '--serverport=3456',
'--processes=2'],
stderr=PIPE)
sleep(2) # to allow both clients to spin up so that one won't
# be hogging all the jobs (since they have 2 procs
# each, that will lead to duplicated pids). NOTE: this
# does not guarantee anything
try:
# then, in-process, push jobs to that queue and watch them
# return results
argv = ["test", "pid", "--all", "--buildqueue", "--serverport=3456", "--loglevel=CRITICAL"]
res = manager.run(argv)
# same tests as for RunMultiproc.test_run_single_all
args = [x[0] for x in res]
pids = [x[1] for x in res]
# we're not guaranteed any particular order, therefore we
# compare sets instead of lists
self.assertEqual(set(args), set(["arg1", "myarg", "arg2"]))
self.assertEqual(3, len(set(pids)))
finally:
# NOTE: On Win32/py2 just terminate() will not kill children of foo
# and bar (of which tree out of four are left hanging due
# to us not being able to send DONE signals to them all).
# See http://stackoverflow.com/questions/1230669/subprocess-deleting-child-processes-in-windows
if sys.platform == "win32":
import psutil
for child in psutil.Process(foo.pid).children(recursive=True):
child.kill()
for child in psutil.Process(bar.pid).children(recursive=True):
child.kill()
for p in foo, bar, queue:
p.terminate()
p.wait() # to clear up the process table (otherwise p lives on as zombie)
if p.stderr:
p.stderr.close()
def test_run_single_all_multiprocessing_fail(self):
self._enable_repos()
# print("running multiproc for pid %s, datadir %s" % (os.getpid(), self.tempdir))
argv = [
"test", "errmethod", "--all", "--processes=3",
"--loglevel=CRITICAL"
]
res = manager.run(argv)
# this doesn't test that errors get reported immediately
# (which they do)
self.assertEqual(res[0][0], Exception)
self.assertEqual(res[1][0], errors.DocumentRemovedError)
self.assertEqual(res[2], None)
self.assertTrue(os.path.exists("dummyfile.txt"))
def test_config_init(self):
# make sure that the sub-config object created by run() is
# identical to the config object used by the instance
manager.config_loaded = False
self._enable_repos()
argv = ['test', 'inspect', 'config']
ourcfg = manager._load_config(argv=argv,
defaults={'loglevel': 'CRITICAL',
'logfile': None,
'datadir': 'data',
'profile': False,
'test': {'hello': 'world'}})
with patch('ferenda.manager._load_config', return_value=ourcfg):
instcfg = manager.run(argv)
self.assertIsInstance(instcfg, LayeredConfig)
self.assertEqual(id(ourcfg.test),
id(instcfg))
def test_server_all(self):
# run multiple repos, one after the other, on the same method
# and make sure the different methods are run.
self._enable_repos()
# create two out-of-process clients
foo = Popen([
'python', 'ferenda-build.py', 'all', 'buildclient',
'--clientname=foo', '--processes=2'
],
stderr=PIPE)
bar = Popen([
'python', 'ferenda-build.py', 'all', 'buildclient',
'--clientname=bar', '--processes=2'
],
stderr=PIPE)
try:
# run an in-process server
argv = [
"all", "pid", "--all", "--buildserver", "--loglevel=CRITICAL"
]
res = manager.run(argv)
# we should have two main sets of results, the first from
# Testrepo, the second from Testrepo2
self.assertEqual(2, len(res))
# same tests as for RunMultiproc.test_run_single_all
args1 = [x[0] for x in res[0]]
pids1 = [x[1] for x in res[0]]
self.assertEqual(set(args1), set(["arg1", "myarg", "arg2"]))
self.assertEqual(3, len(set(pids1)))
args2 = [x[0] for x in res[1]]
pids2 = [x[1] for x in res[1]]
self.assertEqual(set(args2),
set(["repo2:arg1", "repo2:myarg", "repo2:arg2"]))
self.assertEqual(3, len(set(pids2)))
finally:
for p in foo, bar:
p.terminate()
p.wait()
p.stderr.close()
def test_config_init(self):
# make sure that the sub-config object created by run() is
# identical to the config object used by the instance
manager.config_loaded = False
self._enable_repos()
argv = ['test', 'inspect', 'config']
ourcfg = manager._load_config(argv=argv,
defaults={
'loglevel': 'CRITICAL',
'logfile': None,
'datadir': 'data',
'profile': False,
'test': {
'hello': 'world'
}
})
with patch('ferenda.manager._load_config', return_value=ourcfg):
instcfg = manager.run(argv)
self.assertIsInstance(instcfg, LayeredConfig)
self.assertEqual(id(ourcfg.test), id(instcfg))
def test_run_makeresources_defaultconfig(self):
util.resource_extract(ResourceLoader(),
'scripts/ferenda.template.ini',
"ferenda.ini",
{'storetype': 'SQLITE',
'storelocation': 'data/ferenda.sqlite',
'storerepository': 'ferenda',
'indextype': 'WHOOSH',
'indexlocation': 'data/whooshindex',
'sitename': 'Test'})
self._enable_repos()
s = os.sep
got = manager.run(['all', 'makeresources', '--loglevel=CRITICAL'])
want = {'xml':[s.join(['rsrc', 'resources.xml'])],
'json': [s.join(['rsrc','api','context.json']),
s.join(['rsrc','api','common.json']),
s.join(['rsrc','api','terms.json'])],
'img': [s.join(['rsrc', 'img', 'test.png'])],
'css': [s.join(['rsrc', 'css', 'ferenda.css']),
s.join(['rsrc', 'css', 'test.css'])],
'js': [s.join(['rsrc', 'js', 'ferenda.js']),
s.join(['rsrc', 'js', 'test.js'])]}
self.assertEqual(want, got)
def test_run_all_allmethods(self):
self._enable_repos()
argv = ["all", "all", "--magic=more"]
s = os.sep
want = OrderedDict(
[('download', OrderedDict([('test','test download ok (magic=more)'),
('test2', 'test2 download ok (magic=more)')])),
('parse', OrderedDict([('test', ['test parse arg1',
'test parse myarg',
'test parse arg2']),
('test2', ['test2 parse arg1',
'test2 parse myarg',
'test2 parse arg2'])])),
('relate', OrderedDict([('test', ['test relate arg1',
'test relate myarg',
'test relate arg2']),
('test2', ['test2 relate arg1',
'test2 relate myarg',
'test2 relate arg2'])])),
('makeresources', {'css':[s.join(['rsrc', 'css','test.css']),
s.join(['rsrc', 'css','other.css'])],
'js':[s.join(['rsrc', 'js','test.js'])],
'xml':[s.join(['rsrc', 'resources.xml'])]}),
('generate', OrderedDict([('test', ['test generate arg1',
'test generate myarg',
'test generate arg2']),
('test2', ['test2 generate arg1',
'test2 generate myarg',
'test2 generate arg2'])])),
('toc', OrderedDict([('test','test toc ok'),
('test2', 'test2 toc ok')])),
('news', OrderedDict([('test','test news ok'),
('test2', 'test2 news ok')])),
('frontpage', True)])
got = manager.run(argv)
self.maxDiff = None
self.assertEqual(want,got)
def test_delayed_config(self):
# Make sure configuration values gets stored properly in the instance created by run()
self._enable_repos()
self.assertEqual(".html",
manager.run(['test', 'inspect', 'downloaded_suffix']))
self.assertEqual("file",
manager.run(['test', 'inspect', 'storage_policy']))
self.assertEqual(self.tempdir,
manager.run(['test', 'inspect', 'config', 'datadir']))
self.assertEqual(
[".html"],
manager.run(['test', 'inspect', 'store', 'downloaded_suffixes']))
self.assertEqual(
"file", manager.run(['test', 'inspect', 'store',
'storage_policy']))
self.assertEqual(self.tempdir + os.sep + "test",
manager.run(['test', 'inspect', 'store', 'datadir']))
self.assertEqual(
".txt", manager.run(['test2', 'inspect', 'downloaded_suffix']))
self.assertEqual("dir",
manager.run(['test2', 'inspect', 'storage_policy']))
self.assertEqual(
self.tempdir,
manager.run(['test2', 'inspect', 'config', 'datadir']))
self.assertEqual(
[".txt"],
manager.run(['test2', 'inspect', 'store', 'downloaded_suffixes']))
self.assertEqual(
"dir", manager.run(['test2', 'inspect', 'store',
'storage_policy']))
self.assertEqual(self.tempdir + os.sep + "test2",
manager.run(['test2', 'inspect', 'store', 'datadir']))
def test_noconfig(self):
os.unlink("ferenda.ini")
with self.assertRaises(errors.ConfigurationError):
manager.run(["test", "mymethod", "myarg"])
def test_run_all(self):
self._enable_repos()
argv = ["all", "mymethod", "myarg"]
self.assertEqual(manager.run(argv),
["ok!", "yeah!"])
def test_run_all_allmethods(self):
self._enable_repos()
argv = ["all", "all", "--magic=more"]
s = os.sep
want = OrderedDict([
('download',
OrderedDict([('test', 'test download ok (magic=more)'),
('test2', 'test2 download ok (magic=more)')])),
('parse',
OrderedDict([
('test',
['test parse arg1', 'test parse myarg', 'test parse arg2']),
('test2', [
'test2 parse arg1', 'test2 parse myarg',
'test2 parse arg2'
])
])),
('relate',
OrderedDict([('test', [
'test relate arg1', 'test relate myarg', 'test relate arg2'
]),
('test2', [
'test2 relate arg1', 'test2 relate myarg',
'test2 relate arg2'
])])),
('makeresources', {
'css': [
s.join(['rsrc', 'css', 'test.css']),
s.join(['rsrc', 'css', 'other.css'])
],
'js': [s.join(['rsrc', 'js', 'test.js'])],
'img': [s.join(['rsrc', 'img', 'test.png'])],
'json': [
s.join(['rsrc', 'api', 'context.json']),
s.join(['rsrc', 'api', 'common.json']),
s.join(['rsrc', 'api', 'terms.json'])
],
'xml': [s.join(['rsrc', 'resources.xml'])]
}),
('toc',
OrderedDict([('test', 'test toc ok'),
('test2', 'test2 toc ok')])),
('generate',
OrderedDict([('test', [
'test generate arg1', 'test generate myarg',
'test generate arg2'
]),
('test2', [
'test2 generate arg1', 'test2 generate myarg',
'test2 generate arg2'
])])),
('transformlinks',
OrderedDict([('test', [
'test transformlinks arg1', 'test transformlinks myarg',
'test transformlinks arg2'
]),
('test2', [
'test2 transformlinks arg1',
'test2 transformlinks myarg',
'test2 transformlinks arg2'
])])),
('news',
OrderedDict([('test', 'test news ok'),
('test2', 'test2 news ok')])), ('frontpage', True)
])
got = manager.run(argv)
self.maxDiff = None
self.assertEqual(want, got)
# 7. add an unrelated command line flag and verify that this
# does not interfere with the processing
argv.append('--downloadmax=50')
got = manager.run(argv)
self.assertEqual(want, got)
def test_run_all_all(self):
self._enable_repos()
argv = ["all", "mymethod", "--all"]
# FIXME: Check to see if proper setup and teardown on Testrepo/Testrepo2 was properly called (how?)
self.assertEqual(manager.run(argv),
[[None, "ok!", None], [None, "yeah!", None]])
def test_run_all(self):
self._enable_repos()
argv = ["all", "mymethod", "myarg"]
self.assertEqual(manager.run(argv), ["ok!", "yeah!"])
def test_delayed_config(self):
# Make sure configuration values gets stored properly in the instance created by run()
self._enable_repos()
self.assertEqual(".html", manager.run(['test', 'inspect', 'downloaded_suffix']))
self.assertEqual("file", manager.run(['test', 'inspect', 'storage_policy']))
self.assertEqual(self.tempdir, manager.run(['test', 'inspect', 'config', 'datadir']))
self.assertEqual([".html"], manager.run(['test', 'inspect', 'store', 'downloaded_suffixes']))
self.assertEqual("file", manager.run(['test', 'inspect', 'store', 'storage_policy']))
self.assertEqual(self.tempdir+os.sep+"test", manager.run(['test','inspect','store', 'datadir']))
self.assertEqual(".txt", manager.run(['test2', 'inspect', 'downloaded_suffix']))
self.assertEqual("dir", manager.run(['test2', 'inspect', 'storage_policy']))
self.assertEqual(self.tempdir, manager.run(['test2', 'inspect', 'config', 'datadir']))
self.assertEqual([".txt"], manager.run(['test2', 'inspect', 'store', 'downloaded_suffixes']))
self.assertEqual("dir", manager.run(['test2', 'inspect', 'store','storage_policy']))
self.assertEqual(self.tempdir+os.sep+"test2", manager.run(['test2', 'inspect', 'store', 'datadir']))
def test_custom_docstore(self):
self._enable_repos()
got = manager.run(['test2', 'callstore'])
self.assertEqual("CustomStore OK", got)
def test_run_ctrlc(self):
self._enable_repos()
argv = ["test", "keyboardinterrupt", "--all"]
with self.assertRaises(KeyboardInterrupt):
manager.run(argv)
def test_noconfig(self):
os.unlink("ferenda.ini")
with self.assertRaises(errors.ConfigurationError):
manager.run(["test", "mymethod", "myarg"])
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import sys
# use when developing the ferenda source code
# import os
# sys.path.append(os.path.normpath(os.getcwd() + os.sep + os.pardir))
from ferenda import manager
if __name__ == '__main__':
manager.run(sys.argv[1:])
def test_custom_docstore(self):
self._enable_repos()
got = manager.run(['test2', 'callstore'])
self.assertEqual("CustomStore OK", got)
def test_run_ctrlc_multiprocessing(self):
self._enable_repos()
argv = ["test", "keyboardinterrupt", "--all", "--processes=2"]
with self.assertRaises(KeyboardInterrupt):
manager.run(argv)
#!/usr/share/ferenda/.virtualenv/bin/python
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import sys
import os
sys.path.append(os.path.normpath(os.getcwd() + os.sep + os.pardir))
from ferenda import manager
if __name__ == '__main__':
manager.run(sys.argv[1:])
def test_run_single_all(self):
self._enable_repos()
argv = ["test","mymethod","--all"]
with patch("example.Testrepo.setup", return_value=False):
self.assertEqual(manager.run(argv), [])
