def test_runserver(self): self._enable_repos() m = Mock() with patch('ferenda.manager.make_server', return_value=m) as m2: manager.run(["all", "runserver"]) self.assertTrue(m2.called) self.assertTrue(m.serve_forever.called)
def test_named_logfile(self): self._enable_repos() self.assertFalse(os.path.exists("out.log")) argv = ["test","mymethod","myarg","--logfile=out.log"] manager.run(argv) self.assertTrue(os.path.exists("out.log")) os.unlink("out.log")
def _enable_repos(self): # 3. run('example.Testrepo', 'enable') with patch.object(logging.Logger, 'info') as mocklog: self.assertEqual( "test", manager.run([self.modulename + ".Testrepo", "enable"])) # 4. verify that "alias foo enabled" is logged log_arg = mocklog.call_args[0][0] self.assertEqual( "Enabled class %s.Testrepo (alias 'test')" % self.modulename, log_arg) # 5. verify that ferenda.ini has changed cfg = configparser.ConfigParser() cfg.read(["ferenda.ini"]) self.assertEqual(cfg.get("test", "class"), self.modulename + ".Testrepo") # (same, with 'example.Testrepo2') self.assertEqual( "test2", manager.run([self.modulename + ".Testrepo2", "enable"])) cfg = configparser.ConfigParser() cfg.read(["ferenda.ini"]) self.assertEqual(cfg.get("test2", "class"), self.modulename + ".Testrepo2") with patch.object(logging.Logger, 'error') as mocklog: # 6. run('example.Nonexistent', 'enable') -- the ImportError must # be caught and an error printed. manager.run([self.modulename + ".Nonexistent", "enable"]) # 7. verify that a suitable error messsage is logged self.assertEqual( "No class named '%s.Nonexistent'" % self.modulename, mocklog.call_args[0][0])
def test_run_makeresources(self): # 1. setup test_run_enable # 2. run('all', 'makeresources') # 3. verify that all css/jss files specified by default and in Testrepo gets copied # (remove rsrc) # 4. run('all', 'makeresources', '--combine') # 5. verify that single css and js file is created self._enable_repos() s = os.sep want = {'css':[s.join(['rsrc', 'css','test.css']), s.join(['rsrc', 'css','other.css'])], 'js':[s.join(['rsrc', 'js','test.js'])], 'xml':[s.join(['rsrc', 'resources.xml'])] } got = manager.run(['all', 'makeresources']) self.assertEqual(want,got) # 6. alter the ferenda.ini so that it doesn't specify any css/js files util.writefile("ferenda.ini", """[__root__] loglevel=WARNING datadir = %s url = http://localhost:8000 searchendpoint = /search/ apiendpoint = /api/ """ % self.tempdir) want = {'css':[], 'js':[], 'xml':[s.join(['rsrc', 'resources.xml'])] } got = manager.run(['all', 'makeresources']) self.assertEqual(want,got)
def test_named_logfile(self): self._enable_repos() self.assertFalse(os.path.exists("out.log")) argv = ["test", "mymethod", "myarg", "--logfile=out.log"] manager.run(argv) self.assertTrue(os.path.exists("out.log")) os.unlink("out.log")
def _enable_repos(self): # 3. run('example.Testrepo', 'enable') with patch.object(logging.Logger, 'info') as mocklog: self.assertEqual("test", manager.run([self.modulename+".Testrepo", "enable"])) # 4. verify that "alias foo enabled" is logged log_arg = mocklog.call_args[0][0] self.assertEqual("Enabled class %s.Testrepo (alias 'test')" % self.modulename, log_arg) # 5. verify that ferenda.ini has changed cfg = configparser.ConfigParser() cfg.read(["ferenda.ini"]) self.assertEqual(cfg.get("test","class"), self.modulename+".Testrepo") # (same, with 'example.Testrepo2') self.assertEqual("test2", manager.run([self.modulename+".Testrepo2", "enable"])) cfg = configparser.ConfigParser() cfg.read(["ferenda.ini"]) self.assertEqual(cfg.get("test2","class"), self.modulename+".Testrepo2") with patch.object(logging.Logger, 'error') as mocklog: # 6. run('example.Nonexistent', 'enable') -- the ImportError must # be caught and an error printed. manager.run([self.modulename+".Nonexistent", "enable"]) # 7. verify that a suitable error messsage is logged self.assertEqual("No class named '%s.Nonexistent'" % self.modulename, mocklog.call_args[0][0])
def test_run_single_all(self): self._enable_repos() argv = ["test", "mymethod", "--all"] # Test 1: make sure that if setup signals that no work should # be done, this is respected with patch("example.Testrepo.setup", return_value=False): self.assertEqual(manager.run(list(argv)), []) # pass # Test 2: but if not, do the work self.assertEqual(manager.run(list(argv)), [None, "ok!", None])
def test_run_single_all(self): self._enable_repos() argv = ["test","mymethod","--all"] # Test 1: make sure that if setup signals that no work should # be done, this is respected with patch("example.Testrepo.setup", return_value=False): self.assertEqual(manager.run(list(argv)), []) # pass # Test 2: but if not, do the work self.assertEqual(manager.run(list(argv)), [None, "ok!", None])
def test_run_all_allmethods(self): self._enable_repos() argv = ["all", "all", "--magic=more"] s = os.sep want = OrderedDict( [('download', OrderedDict([('test','test download ok (magic=more)'), ('test2', 'test2 download ok (magic=more)')])), ('parse', OrderedDict([('test', ['test parse arg1', 'test parse myarg', 'test parse arg2']), ('test2', ['test2 parse arg1', 'test2 parse myarg', 'test2 parse arg2'])])), ('relate', OrderedDict([('test', ['test relate arg1', 'test relate myarg', 'test relate arg2']), ('test2', ['test2 relate arg1', 'test2 relate myarg', 'test2 relate arg2'])])), ('makeresources', {'css':[s.join(['rsrc', 'css','test.css']), s.join(['rsrc', 'css','other.css'])], 'js':[s.join(['rsrc', 'js','test.js'])], 'img':[s.join(['rsrc', 'img','test.png'])], 'json': [s.join(['rsrc','api','context.json']), s.join(['rsrc','api','common.json']), s.join(['rsrc','api','terms.json'])], 'xml':[s.join(['rsrc', 'resources.xml'])]}), ('toc', OrderedDict([('test','test toc ok'), ('test2', 'test2 toc ok')])), ('generate', OrderedDict([('test', ['test generate arg1', 'test generate myarg', 'test generate arg2']), ('test2', ['test2 generate arg1', 'test2 generate myarg', 'test2 generate arg2'])])), ('transformlinks', OrderedDict([('test', ['test transformlinks arg1', 'test transformlinks myarg', 'test transformlinks arg2']), ('test2', ['test2 transformlinks arg1', 'test2 transformlinks myarg', 'test2 transformlinks arg2'])])), ('news', OrderedDict([('test','test news ok'), ('test2', 'test2 news ok')])), ('frontpage', True)]) got = manager.run(argv) self.maxDiff = None self.assertEqual(want,got) # 7. add an unrelated command line flag and verify that this # does not interfere with the processing argv.append('--downloadmax=50') got = manager.run(argv) self.assertEqual(want,got)
def test_noclobber(self): manager.run([self.modulename + ".Testrepo", "enable"]) manager.run([self.modulename + ".Testrepo", "save"]) cfg = configparser.ConfigParser() cfg.read(["ferenda.ini"]) # make sure cfg has one section for testrepo and only two # attributes ('class' was created when enabling the module, # saved was created in the 'save' call) self.assertEqual( set((('class', 'example.Testrepo'), ('saved', 'True'))), set(cfg.items('test')))
def test_noclobber(self): manager.run([self.modulename+".Testrepo", "enable"]) manager.run([self.modulename+".Testrepo", "save"]) cfg = configparser.ConfigParser() cfg.read(["ferenda.ini"]) # make sure cfg has one section for testrepo and only two # attributes ('class' was created when enabling the module, # saved was created in the 'save' call) self.assertEqual(set((('class', 'example.Testrepo'), ('saved', 'True'))), set(cfg.items('test')))
def test_run_makeresources(self): # 1. setup test_run_enable # 2. run('all', 'makeresources') # 3. verify that all css/jss files specified by default and in # Testrepo gets copied (remove rsrc) # 4. run('all', 'makeresources', '--combine') # 5. verify that single css and js file is created self._enable_repos() s = os.sep want = { 'css': [ s.join(['rsrc', 'css', 'test.css']), s.join(['rsrc', 'css', 'other.css']) ], 'js': [s.join(['rsrc', 'js', 'test.js'])], 'img': [s.join(['rsrc', 'img', 'test.png'])], 'json': [ s.join(['rsrc', 'api', 'context.json']), s.join(['rsrc', 'api', 'common.json']), s.join(['rsrc', 'api', 'terms.json']) ], 'xml': [s.join(['rsrc', 'resources.xml'])] } got = manager.run(['all', 'makeresources']) self.assertEqual(want, got) # 6. alter the ferenda.ini so that it doesn't specify any css/js files util.writefile( "ferenda.ini", """[__root__] loglevel=WARNING datadir = %s url = http://localhost:8000/ searchendpoint = /search/ apiendpoint = /api/ cssfiles = [] jsfiles = [] imgfiles = [] """ % self.tempdir) want = { 'css': [], 'js': [], 'img': [], 'json': [ s.join(['rsrc', 'api', 'context.json']), s.join(['rsrc', 'api', 'common.json']), s.join(['rsrc', 'api', 'terms.json']) ], 'xml': [s.join(['rsrc', 'resources.xml'])] } got = manager.run(['all', 'makeresources']) self.assertEqual(want, got)
def test_run_all_all(self): self._enable_repos() argv = ["all", "mymethod", "--all"] # FIXME: Check to see if proper setup and teardown on Testrepo/Testrepo2 was properly called (how?) self.assertEqual(manager.run(argv), [[None,"ok!",None], [None,"yeah!",None]])
def test_print_usage(self): self._enable_repos() with patch('builtins.print') as printmock: manager.run([]) executable = sys.argv[0] got = "\n".join([x[1][0] for x in printmock.mock_calls]) got = got.replace(executable, "[EXEC]") want = """Usage: [EXEC] [class-or-alias] [action] <arguments> <options> e.g. '[EXEC] ferenda.sources.EurlexCaselaw enable' '[EXEC] ecj parse 62008J0042' '[EXEC] all generate' Available modules: * test: [Undocumented] * test2: [Undocumented]""" self.assertEqual(got, want)
def test_server(self): self._enable_repos() # create two out-of-process clients foo = Popen(['python', 'ferenda-build.py', 'all', 'buildclient', '--clientname=foo', '--processes=2'], stderr=PIPE) bar = Popen(['python', 'ferenda-build.py', 'all', 'buildclient', '--clientname=bar', '--processes=2'], stderr=PIPE) try: # run an in-process server argv = ["test", "pid", "--all", "--buildserver", "--loglevel=CRITICAL"] res = manager.run(argv) # same tests as for RunMultiproc.test_run_single_all args = [x[0] for x in res] pids = [x[1] for x in res] # we're not guaranteed any particular order, therefore we # compare sets instead of lists self.assertEqual(set(args), set(["arg1", "myarg", "arg2"])) self.assertEqual(3, len(set(pids))) finally: for p in foo, bar: p.terminate() p.wait() p.stderr.close()
def test_run_single_allmethods(self): self._enable_repos() argv = ["test","all"] s = os.sep self.maxDiff = None want = OrderedDict( [('download', OrderedDict([('test','test download ok (magic=less)'), ])), ('parse', OrderedDict([('test', ['test parse arg1', 'test parse myarg', 'test parse arg2']), ])), ('relate', OrderedDict([('test', ['test relate arg1', 'test relate myarg', 'test relate arg2']), ])), ('makeresources', {'css':[s.join(['rsrc', 'css','test.css']), s.join(['rsrc', 'css','other.css'])], 'js':[s.join(['rsrc', 'js','test.js'])], 'xml':[s.join(['rsrc', 'resources.xml'])]}), ('generate', OrderedDict([('test', ['test generate arg1', 'test generate myarg', 'test generate arg2']), ])), ('toc', OrderedDict([('test','test toc ok'), ])), ('news', OrderedDict([('test','test news ok'), ])), ('frontpage', True)]) self.assertEqual(manager.run(argv), want)
def test_server_all(self): # run multiple repos, one after the other, on the same method # and make sure the different methods are run. self._enable_repos() # create two out-of-process clients foo = Popen(['python', 'ferenda-build.py', 'all', 'buildclient', '--clientname=foo', '--processes=2'], stderr=PIPE) bar = Popen(['python', 'ferenda-build.py', 'all', 'buildclient', '--clientname=bar', '--processes=2'], stderr=PIPE) try: # run an in-process server argv = ["all", "pid", "--all", "--buildserver", "--loglevel=CRITICAL"] res = manager.run(argv) # we should have two main sets of results, the first from # Testrepo, the second from Testrepo2 self.assertEqual(2, len(res)) # same tests as for RunMultiproc.test_run_single_all args1 = [x[0] for x in res[0]] pids1 = [x[1] for x in res[0]] self.assertEqual(set(args1), set(["arg1", "myarg", "arg2"])) self.assertEqual(3, len(set(pids1))) args2 = [x[0] for x in res[1]] pids2 = [x[1] for x in res[1]] self.assertEqual(set(args2), set(["repo2:arg1", "repo2:myarg", "repo2:arg2"])) self.assertEqual(3, len(set(pids2))) finally: for p in foo, bar: p.terminate() p.wait() p.stderr.close()
def test_run_makeresources_defaultconfig(self): util.resource_extract( ResourceLoader(), 'scripts/ferenda.template.ini', "ferenda.ini", { 'storetype': 'SQLITE', 'storelocation': 'data/ferenda.sqlite', 'storerepository': 'ferenda', 'indextype': 'WHOOSH', 'indexlocation': 'data/whooshindex', 'sitename': 'Test' }) self._enable_repos() s = os.sep got = manager.run(['all', 'makeresources', '--loglevel=CRITICAL']) want = { 'xml': [s.join(['rsrc', 'resources.xml'])], 'json': [ s.join(['rsrc', 'api', 'context.json']), s.join(['rsrc', 'api', 'common.json']), s.join(['rsrc', 'api', 'terms.json']) ], 'img': [ s.join(['rsrc', 'img', 'atom.png']), s.join(['rsrc', 'img', 'test.png']) ], 'css': [ s.join(['rsrc', 'css', 'ferenda.css']), s.join(['rsrc', 'css', 'test.css']) ], 'js': [ s.join(['rsrc', 'js', 'ferenda.js']), s.join(['rsrc', 'js', 'test.js']) ] } self.assertEqual(want, got)
def test_server(self): self._enable_repos() # create two out-of-process clients foo = Popen([ 'python', 'ferenda-build.py', 'all', 'buildclient', '--clientname=foo', '--processes=2' ], stderr=PIPE) bar = Popen([ 'python', 'ferenda-build.py', 'all', 'buildclient', '--clientname=bar', '--processes=2' ], stderr=PIPE) try: # run an in-process server argv = [ "test", "pid", "--all", "--buildserver", "--loglevel=CRITICAL" ] res = manager.run(argv) # same tests as for RunMultiproc.test_run_single_all args = [x[0] for x in res] pids = [x[1] for x in res] # we're not guaranteed any particular order, therefore we # compare sets instead of lists self.assertEqual(set(args), set(["arg1", "myarg", "arg2"])) self.assertEqual(3, len(set(pids))) finally: for p in foo, bar: p.terminate() p.wait() p.stderr.close()
def test_run_single(self): # test1: run standard (custom) method self._enable_repos() argv = ["test", "mymethod", "myarg"] self.assertEqual(manager.run(argv), "ok!") # test2: specify invalid alias argv[0] = "invalid" with patch('ferenda.manager.setup_logger'): self.assertEqual(manager.run(argv), None) with patch('builtins.print') as printmock: with patch('ferenda.manager.setup_logger'): # test3: specify invalid method argv = ["test", "invalid"] self.assertEqual(manager.run(argv), None)
def update(self, article): """Update all generated pages that are dependent on/include the given wiki article.""" self.config.force = True self.config.refresh = True self.download(article) self.parse(article) self.relate(article) if article.startswith("SFS/"): self.sfsrepo.config.force = True self.sfsrepo.generate(article.split("/", 1)[1]) elif article == "Lagen.nu:Huvudsida": # FIXME: This re-downloads and re-parses the wiki page from ferenda import manager manager.run(["all", "frontpage"], self.config._parent) else: self.keywordrepo.config.force = True self.keywordrepo.generate(article)
def test_run_single(self): # test1: run standard (custom) method self._enable_repos() argv = ["test","mymethod","myarg"] self.assertEqual(manager.run(argv), "ok!") # test2: specify invalid alias argv[0] = "invalid" with patch('ferenda.manager.setup_logger'): self.assertEqual(manager.run(argv), None) with patch('builtins.print') as printmock: with patch('ferenda.manager.setup_logger'): # test3: specify invalid method argv = ["test", "invalid"] self.assertEqual(manager.run(argv), None)
def test_global_config(self): # this makes sure that the subprocesses use instances that # have access to the global/manager-provided DEFAULT_CONFIG # config variables self._enable_repos() argv = ["test", "mpinspect", "--all", "--processes=2"] res = manager.run(argv) self.assertEqual(res, [(True, False), (True, False), (True, False)])
def test_queue(self): # runs a separate queue-handling process (which is the only # way to do it on windows). also, test with non-default port self._enable_repos() # create a out-of-process buildqueue server queue = Popen([ 'python', 'ferenda-build.py', 'all', 'buildqueue', '--serverport=3456' ]) # create two out-of-process clients foo = Popen([ 'python', 'ferenda-build.py', 'all', 'buildclient', '--clientname=foo', '--serverport=3456', '--processes=2' ], stderr=PIPE) bar = Popen([ 'python', 'ferenda-build.py', 'all', 'buildclient', '--clientname=bar', '--serverport=3456', '--processes=2' ], stderr=PIPE) sleep(2) # to allow both clients to spin up so that one won't # be hogging all the jobs (since they have 2 procs # each, that will lead to duplicated pids). NOTE: this # does not guarantee anything try: # then, in-process, push jobs to that queue and watch them # return results argv = [ "test", "pid", "--all", "--buildqueue", "--serverport=3456", "--loglevel=CRITICAL" ] res = manager.run(argv) # same tests as for RunMultiproc.test_run_single_all args = [x[0] for x in res] pids = [x[1] for x in res] # we're not guaranteed any particular order, therefore we # compare sets instead of lists self.assertEqual(set(args), set(["arg1", "myarg", "arg2"])) self.assertEqual(3, len(set(pids))) finally: # NOTE: On Win32/py2 just terminate() will not kill children of foo # and bar (of which tree out of four are left hanging due # to us not being able to send DONE signals to them all). # See http://stackoverflow.com/questions/1230669/subprocess-deleting-child-processes-in-windows if sys.platform == "win32": import psutil for child in psutil.Process(foo.pid).children(recursive=True): child.kill() for child in psutil.Process(bar.pid).children(recursive=True): child.kill() for p in foo, bar, queue: p.terminate() p.wait( ) # to clear up the process table (otherwise p lives on as zombie) if p.stderr: p.stderr.close()
def test_run_single_all_multiprocessing(self): self._enable_repos() argv = ["test", "pid", "--all", "--processes=3"] res = manager.run(argv) args = [x[0] for x in res] pids = [x[1] for x in res] self.assertEqual(args, ["arg1", "myarg", "arg2"]) # assert that all pids are unique self.assertEqual(3, len(set(pids)))
def test_run_single_allmethods(self): self._enable_repos() argv = ["test", "all"] s = os.sep self.maxDiff = None want = OrderedDict([ ('download', OrderedDict([ ('test', 'test download ok (magic=less)'), ])), ('parse', OrderedDict([ ('test', ['test parse arg1', 'test parse myarg', 'test parse arg2']), ])), ('relate', OrderedDict([ ('test', [ 'test relate arg1', 'test relate myarg', 'test relate arg2' ]), ])), ('makeresources', { 'css': [ s.join(['rsrc', 'css', 'test.css']), s.join(['rsrc', 'css', 'other.css']) ], 'img': [s.join(['rsrc', 'img', 'test.png'])], 'js': [s.join(['rsrc', 'js', 'test.js'])], 'json': [ s.join(['rsrc', 'api', 'context.json']), s.join(['rsrc', 'api', 'common.json']), s.join(['rsrc', 'api', 'terms.json']) ], 'xml': [s.join(['rsrc', 'resources.xml'])] }), ('toc', OrderedDict([ ('test', 'test toc ok'), ])), ('generate', OrderedDict([ ('test', [ 'test generate arg1', 'test generate myarg', 'test generate arg2' ]), ])), ('transformlinks', OrderedDict([ ('test', [ 'test transformlinks arg1', 'test transformlinks myarg', 'test transformlinks arg2' ]), ])), ('news', OrderedDict([ ('test', 'test news ok'), ])), ('frontpage', True) ]) self.assertEqual(want, manager.run(argv))
def test_run_single_errors(self): self._enable_repos() argv = ["test", "errmethod", "--all"] with patch('ferenda.manager.setup_logger'): with patch('builtins.print') as printmock: res = manager.run(argv) self.assertEqual(res[0][0], Exception) self.assertEqual(res[1][0], errors.DocumentRemovedError) self.assertEqual(res[2], None) self.assertTrue(os.path.exists("dummyfile.txt"))
def test_run_single_all_multiprocessing_fail(self): self._enable_repos() # print("running multiproc for pid %s, datadir %s" % (os.getpid(), self.tempdir)) argv = ["test","errmethod","--all", "--processes=3", "--loglevel=CRITICAL"] res = manager.run(argv) # this doesn't test that errors get reported immediately # (which they do) self.assertEqual(res[0][0], Exception) self.assertEqual(res[1][0], errors.DocumentRemovedError) self.assertEqual(res[2], None) self.assertTrue(os.path.exists("dummyfile.txt"))
def test_queue(self): # runs a separate queue-handling process (which is the only # way to do it on windows). also, test with non-default port self._enable_repos() # create a out-of-process buildqueue server queue = Popen(['python', 'ferenda-build.py', 'all', 'buildqueue', '--serverport=3456']) # create two out-of-process clients foo = Popen(['python', 'ferenda-build.py', 'all', 'buildclient', '--clientname=foo', '--serverport=3456', '--processes=2'], stderr=PIPE) bar = Popen(['python', 'ferenda-build.py', 'all', 'buildclient', '--clientname=bar', '--serverport=3456', '--processes=2'], stderr=PIPE) sleep(2) # to allow both clients to spin up so that one won't # be hogging all the jobs (since they have 2 procs # each, that will lead to duplicated pids). NOTE: this # does not guarantee anything try: # then, in-process, push jobs to that queue and watch them # return results argv = ["test", "pid", "--all", "--buildqueue", "--serverport=3456", "--loglevel=CRITICAL"] res = manager.run(argv) # same tests as for RunMultiproc.test_run_single_all args = [x[0] for x in res] pids = [x[1] for x in res] # we're not guaranteed any particular order, therefore we # compare sets instead of lists self.assertEqual(set(args), set(["arg1", "myarg", "arg2"])) self.assertEqual(3, len(set(pids))) finally: # NOTE: On Win32/py2 just terminate() will not kill children of foo # and bar (of which tree out of four are left hanging due # to us not being able to send DONE signals to them all). # See http://stackoverflow.com/questions/1230669/subprocess-deleting-child-processes-in-windows if sys.platform == "win32": import psutil for child in psutil.Process(foo.pid).children(recursive=True): child.kill() for child in psutil.Process(bar.pid).children(recursive=True): child.kill() for p in foo, bar, queue: p.terminate() p.wait() # to clear up the process table (otherwise p lives on as zombie) if p.stderr: p.stderr.close()
def test_run_single_all_multiprocessing_fail(self): self._enable_repos() # print("running multiproc for pid %s, datadir %s" % (os.getpid(), self.tempdir)) argv = [ "test", "errmethod", "--all", "--processes=3", "--loglevel=CRITICAL" ] res = manager.run(argv) # this doesn't test that errors get reported immediately # (which they do) self.assertEqual(res[0][0], Exception) self.assertEqual(res[1][0], errors.DocumentRemovedError) self.assertEqual(res[2], None) self.assertTrue(os.path.exists("dummyfile.txt"))
def test_config_init(self): # make sure that the sub-config object created by run() is # identical to the config object used by the instance manager.config_loaded = False self._enable_repos() argv = ['test', 'inspect', 'config'] ourcfg = manager._load_config(argv=argv, defaults={'loglevel': 'CRITICAL', 'logfile': None, 'datadir': 'data', 'profile': False, 'test': {'hello': 'world'}}) with patch('ferenda.manager._load_config', return_value=ourcfg): instcfg = manager.run(argv) self.assertIsInstance(instcfg, LayeredConfig) self.assertEqual(id(ourcfg.test), id(instcfg))
def test_server_all(self): # run multiple repos, one after the other, on the same method # and make sure the different methods are run. self._enable_repos() # create two out-of-process clients foo = Popen([ 'python', 'ferenda-build.py', 'all', 'buildclient', '--clientname=foo', '--processes=2' ], stderr=PIPE) bar = Popen([ 'python', 'ferenda-build.py', 'all', 'buildclient', '--clientname=bar', '--processes=2' ], stderr=PIPE) try: # run an in-process server argv = [ "all", "pid", "--all", "--buildserver", "--loglevel=CRITICAL" ] res = manager.run(argv) # we should have two main sets of results, the first from # Testrepo, the second from Testrepo2 self.assertEqual(2, len(res)) # same tests as for RunMultiproc.test_run_single_all args1 = [x[0] for x in res[0]] pids1 = [x[1] for x in res[0]] self.assertEqual(set(args1), set(["arg1", "myarg", "arg2"])) self.assertEqual(3, len(set(pids1))) args2 = [x[0] for x in res[1]] pids2 = [x[1] for x in res[1]] self.assertEqual(set(args2), set(["repo2:arg1", "repo2:myarg", "repo2:arg2"])) self.assertEqual(3, len(set(pids2))) finally: for p in foo, bar: p.terminate() p.wait() p.stderr.close()
def test_config_init(self): # make sure that the sub-config object created by run() is # identical to the config object used by the instance manager.config_loaded = False self._enable_repos() argv = ['test', 'inspect', 'config'] ourcfg = manager._load_config(argv=argv, defaults={ 'loglevel': 'CRITICAL', 'logfile': None, 'datadir': 'data', 'profile': False, 'test': { 'hello': 'world' } }) with patch('ferenda.manager._load_config', return_value=ourcfg): instcfg = manager.run(argv) self.assertIsInstance(instcfg, LayeredConfig) self.assertEqual(id(ourcfg.test), id(instcfg))
def test_run_makeresources_defaultconfig(self): util.resource_extract(ResourceLoader(), 'scripts/ferenda.template.ini', "ferenda.ini", {'storetype': 'SQLITE', 'storelocation': 'data/ferenda.sqlite', 'storerepository': 'ferenda', 'indextype': 'WHOOSH', 'indexlocation': 'data/whooshindex', 'sitename': 'Test'}) self._enable_repos() s = os.sep got = manager.run(['all', 'makeresources', '--loglevel=CRITICAL']) want = {'xml':[s.join(['rsrc', 'resources.xml'])], 'json': [s.join(['rsrc','api','context.json']), s.join(['rsrc','api','common.json']), s.join(['rsrc','api','terms.json'])], 'img': [s.join(['rsrc', 'img', 'test.png'])], 'css': [s.join(['rsrc', 'css', 'ferenda.css']), s.join(['rsrc', 'css', 'test.css'])], 'js': [s.join(['rsrc', 'js', 'ferenda.js']), s.join(['rsrc', 'js', 'test.js'])]} self.assertEqual(want, got)
def test_run_all_allmethods(self): self._enable_repos() argv = ["all", "all", "--magic=more"] s = os.sep want = OrderedDict( [('download', OrderedDict([('test','test download ok (magic=more)'), ('test2', 'test2 download ok (magic=more)')])), ('parse', OrderedDict([('test', ['test parse arg1', 'test parse myarg', 'test parse arg2']), ('test2', ['test2 parse arg1', 'test2 parse myarg', 'test2 parse arg2'])])), ('relate', OrderedDict([('test', ['test relate arg1', 'test relate myarg', 'test relate arg2']), ('test2', ['test2 relate arg1', 'test2 relate myarg', 'test2 relate arg2'])])), ('makeresources', {'css':[s.join(['rsrc', 'css','test.css']), s.join(['rsrc', 'css','other.css'])], 'js':[s.join(['rsrc', 'js','test.js'])], 'xml':[s.join(['rsrc', 'resources.xml'])]}), ('generate', OrderedDict([('test', ['test generate arg1', 'test generate myarg', 'test generate arg2']), ('test2', ['test2 generate arg1', 'test2 generate myarg', 'test2 generate arg2'])])), ('toc', OrderedDict([('test','test toc ok'), ('test2', 'test2 toc ok')])), ('news', OrderedDict([('test','test news ok'), ('test2', 'test2 news ok')])), ('frontpage', True)]) got = manager.run(argv) self.maxDiff = None self.assertEqual(want,got)
def test_delayed_config(self): # Make sure configuration values gets stored properly in the instance created by run() self._enable_repos() self.assertEqual(".html", manager.run(['test', 'inspect', 'downloaded_suffix'])) self.assertEqual("file", manager.run(['test', 'inspect', 'storage_policy'])) self.assertEqual(self.tempdir, manager.run(['test', 'inspect', 'config', 'datadir'])) self.assertEqual( [".html"], manager.run(['test', 'inspect', 'store', 'downloaded_suffixes'])) self.assertEqual( "file", manager.run(['test', 'inspect', 'store', 'storage_policy'])) self.assertEqual(self.tempdir + os.sep + "test", manager.run(['test', 'inspect', 'store', 'datadir'])) self.assertEqual( ".txt", manager.run(['test2', 'inspect', 'downloaded_suffix'])) self.assertEqual("dir", manager.run(['test2', 'inspect', 'storage_policy'])) self.assertEqual( self.tempdir, manager.run(['test2', 'inspect', 'config', 'datadir'])) self.assertEqual( [".txt"], manager.run(['test2', 'inspect', 'store', 'downloaded_suffixes'])) self.assertEqual( "dir", manager.run(['test2', 'inspect', 'store', 'storage_policy'])) self.assertEqual(self.tempdir + os.sep + "test2", manager.run(['test2', 'inspect', 'store', 'datadir']))
def test_noconfig(self): os.unlink("ferenda.ini") with self.assertRaises(errors.ConfigurationError): manager.run(["test", "mymethod", "myarg"])
def test_run_all(self): self._enable_repos() argv = ["all", "mymethod", "myarg"] self.assertEqual(manager.run(argv), ["ok!", "yeah!"])
def test_run_all_allmethods(self): self._enable_repos() argv = ["all", "all", "--magic=more"] s = os.sep want = OrderedDict([ ('download', OrderedDict([('test', 'test download ok (magic=more)'), ('test2', 'test2 download ok (magic=more)')])), ('parse', OrderedDict([ ('test', ['test parse arg1', 'test parse myarg', 'test parse arg2']), ('test2', [ 'test2 parse arg1', 'test2 parse myarg', 'test2 parse arg2' ]) ])), ('relate', OrderedDict([('test', [ 'test relate arg1', 'test relate myarg', 'test relate arg2' ]), ('test2', [ 'test2 relate arg1', 'test2 relate myarg', 'test2 relate arg2' ])])), ('makeresources', { 'css': [ s.join(['rsrc', 'css', 'test.css']), s.join(['rsrc', 'css', 'other.css']) ], 'js': [s.join(['rsrc', 'js', 'test.js'])], 'img': [s.join(['rsrc', 'img', 'test.png'])], 'json': [ s.join(['rsrc', 'api', 'context.json']), s.join(['rsrc', 'api', 'common.json']), s.join(['rsrc', 'api', 'terms.json']) ], 'xml': [s.join(['rsrc', 'resources.xml'])] }), ('toc', OrderedDict([('test', 'test toc ok'), ('test2', 'test2 toc ok')])), ('generate', OrderedDict([('test', [ 'test generate arg1', 'test generate myarg', 'test generate arg2' ]), ('test2', [ 'test2 generate arg1', 'test2 generate myarg', 'test2 generate arg2' ])])), ('transformlinks', OrderedDict([('test', [ 'test transformlinks arg1', 'test transformlinks myarg', 'test transformlinks arg2' ]), ('test2', [ 'test2 transformlinks arg1', 'test2 transformlinks myarg', 'test2 transformlinks arg2' ])])), ('news', OrderedDict([('test', 'test news ok'), ('test2', 'test2 news ok')])), ('frontpage', True) ]) got = manager.run(argv) self.maxDiff = None self.assertEqual(want, got) # 7. add an unrelated command line flag and verify that this # does not interfere with the processing argv.append('--downloadmax=50') got = manager.run(argv) self.assertEqual(want, got)
def test_run_all_all(self): self._enable_repos() argv = ["all", "mymethod", "--all"] # FIXME: Check to see if proper setup and teardown on Testrepo/Testrepo2 was properly called (how?) self.assertEqual(manager.run(argv), [[None, "ok!", None], [None, "yeah!", None]])
def test_delayed_config(self): # Make sure configuration values gets stored properly in the instance created by run() self._enable_repos() self.assertEqual(".html", manager.run(['test', 'inspect', 'downloaded_suffix'])) self.assertEqual("file", manager.run(['test', 'inspect', 'storage_policy'])) self.assertEqual(self.tempdir, manager.run(['test', 'inspect', 'config', 'datadir'])) self.assertEqual([".html"], manager.run(['test', 'inspect', 'store', 'downloaded_suffixes'])) self.assertEqual("file", manager.run(['test', 'inspect', 'store', 'storage_policy'])) self.assertEqual(self.tempdir+os.sep+"test", manager.run(['test','inspect','store', 'datadir'])) self.assertEqual(".txt", manager.run(['test2', 'inspect', 'downloaded_suffix'])) self.assertEqual("dir", manager.run(['test2', 'inspect', 'storage_policy'])) self.assertEqual(self.tempdir, manager.run(['test2', 'inspect', 'config', 'datadir'])) self.assertEqual([".txt"], manager.run(['test2', 'inspect', 'store', 'downloaded_suffixes'])) self.assertEqual("dir", manager.run(['test2', 'inspect', 'store','storage_policy'])) self.assertEqual(self.tempdir+os.sep+"test2", manager.run(['test2', 'inspect', 'store', 'datadir']))
def test_custom_docstore(self): self._enable_repos() got = manager.run(['test2', 'callstore']) self.assertEqual("CustomStore OK", got)
def test_run_ctrlc(self): self._enable_repos() argv = ["test", "keyboardinterrupt", "--all"] with self.assertRaises(KeyboardInterrupt): manager.run(argv)
#!/usr/bin/env python # -*- coding: utf-8 -*- from __future__ import unicode_literals import sys # use when developing the ferenda source code # import os # sys.path.append(os.path.normpath(os.getcwd() + os.sep + os.pardir)) from ferenda import manager if __name__ == '__main__': manager.run(sys.argv[1:])
def test_run_ctrlc_multiprocessing(self): self._enable_repos() argv = ["test", "keyboardinterrupt", "--all", "--processes=2"] with self.assertRaises(KeyboardInterrupt): manager.run(argv)
#!/usr/share/ferenda/.virtualenv/bin/python # -*- coding: utf-8 -*- from __future__ import unicode_literals import sys import os sys.path.append(os.path.normpath(os.getcwd() + os.sep + os.pardir)) from ferenda import manager if __name__ == '__main__': manager.run(sys.argv[1:])
def test_run_single_all(self): self._enable_repos() argv = ["test","mymethod","--all"] with patch("example.Testrepo.setup", return_value=False): self.assertEqual(manager.run(argv), [])