def test_scheduling(self, send_message): from toddler.scheduler import Scheduler s = Scheduler(rabbitmq_url="amqp://localhost", exchange='CrawlRequest', routing_key='CrawlRequest') s.schedule_jobs_for_hosts() host = Host.objects(host=self.host.host).first() self.assertEqual( host.last_crawl_job_date.date(), datetime.utcnow().date() )
def main(*argv): """ = Configuration import = check "--help" Run command: python -m toddler.tools.configimport :param argv: :return: """ parser = argparse.ArgumentParser(argv, description="ConfigImport v{}".format(__version__)) parser.add_argument("-t", "--type", help="Config type", choices=["crawl"]) if len(argv) > 0: args = setup(argv, argument_parser=parser, do_not_parse_config=True) else: args = setup(argument_parser=parser, do_not_parse_config=True) print(Style.DIM + Fore.BLUE + "ConfigImport v{}".format(__version__)) with open(args.config) as config_file: print(Fore.BLUE + "Opened file:" + Fore.RESET + " {}".format(args.config) + Fore.RESET) if args.type == "crawl": print(Style.BRIGHT + Fore.BLUE + "Importing crawlConfig") from toddler.imports.nimbuscrawl import get_configuration from toddler.models import Host config_content = config_file.read() config = get_configuration(config_content) for host_name, crawl_config in config: host = Host.objects(host=host_name).first() if host is None: host = Host(host=host_name) host.config["crawlConfig"] = crawl_config host.save() print(Fore.GREEN + "+ Added config for host {}".format(host_name))
def test_configimport_script(self, mock_open): from toddler.decorators import _reset_already_run from toddler import setup _reset_already_run(setup) argv = ['--config', 'test.xml', "--type", "crawl", "--mongo-url", "mongodb://localhost/test"] import io def fopen(*args, **kwargs): return io.StringIO(self.example_xml) mock_open.side_effect = fopen from toddler.tools.configimport import main from toddler.models import Host main(*argv) host = Host.objects(host="www.lesiteimmo.com").first() """:type: Host""" self.assertEqual(host.host, "www.lesiteimmo.com") self.assertGreater(len(host.config['crawlConfig']), 0) self.pattern_asserts(host.config['crawlConfig'])
def __init__(self, hostname): self.host = Host.objects(host=hostname).first()
def get_host(self, hostname): return Host.objects(host=hostname).first()