class SchedulerTest(TestCase): @classmethod def setUpClass(cls): setup(['-m', 'mongodb://localhost'], do_not_parse_config=True) super(SchedulerTest, cls).setUpClass() def setUp(self): self.host = Host(host="example.com") self.host.save() @mock.patch("toddler.rabbit_sender.send_message_sync") def test_scheduling(self, send_message): from toddler.scheduler import Scheduler s = Scheduler(rabbitmq_url="amqp://localhost", exchange='CrawlRequest', routing_key='CrawlRequest') s.schedule_jobs_for_hosts() host = Host.objects(host=self.host.host).first() self.assertEqual( host.last_crawl_job_date.date(), datetime.utcnow().date() ) # self.assertTrue(send_message.called) @mock.patch("toddler.rabbit_sender.send_message_sync") def test_analysis_task_delay_queue(self, send_message): from toddler.scheduler import AnalysisTaskDelayQueueObserver a = AnalysisTaskDelayQueueObserver(rabbitmq_url="amqp://localhost", queue='AnalysisTaskDelayQueue', exchange="AnalysisTask", routing_key='AnalysisTask') d = { "delay_reason": "test reason", "timeout": (datetime.utcnow() - timedelta(hours=1)).isoformat(), "message": { "test": "test" } } a.process_task( ujson.dumps(d).encode("utf8") ) self.assertTrue(send_message.called) self.assertEqual(send_message.call_args[0][1], ujson.dumps({"test": "test"}))
def main(*argv): """ = Configuration import = check "--help" Run command: python -m toddler.tools.configimport :param argv: :return: """ parser = argparse.ArgumentParser(argv, description="ConfigImport v{}".format(__version__)) parser.add_argument("-t", "--type", help="Config type", choices=["crawl"]) if len(argv) > 0: args = setup(argv, argument_parser=parser, do_not_parse_config=True) else: args = setup(argument_parser=parser, do_not_parse_config=True) print(Style.DIM + Fore.BLUE + "ConfigImport v{}".format(__version__)) with open(args.config) as config_file: print(Fore.BLUE + "Opened file:" + Fore.RESET + " {}".format(args.config) + Fore.RESET) if args.type == "crawl": print(Style.BRIGHT + Fore.BLUE + "Importing crawlConfig") from toddler.imports.nimbuscrawl import get_configuration from toddler.models import Host config_content = config_file.read() config = get_configuration(config_content) for host_name, crawl_config in config: host = Host.objects(host=host_name).first() if host is None: host = Host(host=host_name) host.config["crawlConfig"] = crawl_config host.save() print(Fore.GREEN + "+ Added config for host {}".format(host_name))
def test_scheduling(self, send_message): from toddler.scheduler import Scheduler s = Scheduler(rabbitmq_url="amqp://localhost", exchange='CrawlRequest', routing_key='CrawlRequest') s.schedule_jobs_for_hosts() host = Host.objects(host=self.host.host).first() self.assertEqual( host.last_crawl_job_date.date(), datetime.utcnow().date() )
def test_configimport_script(self, mock_open): from toddler.decorators import _reset_already_run from toddler import setup _reset_already_run(setup) argv = ['--config', 'test.xml', "--type", "crawl", "--mongo-url", "mongodb://localhost/test"] import io def fopen(*args, **kwargs): return io.StringIO(self.example_xml) mock_open.side_effect = fopen from toddler.tools.configimport import main from toddler.models import Host main(*argv) host = Host.objects(host="www.lesiteimmo.com").first() """:type: Host""" self.assertEqual(host.host, "www.lesiteimmo.com") self.assertGreater(len(host.config['crawlConfig']), 0) self.pattern_asserts(host.config['crawlConfig'])
def __init__(self, hostname): self.host = Host.objects(host=hostname).first()
def get_host(self, hostname): return Host.objects(host=hostname).first()
def setUp(self): self.host = Host(host="example.com") self.host.save()