Beispiel #1
0
def execute(args):
    if args.verbose:
        logger.setLevel(logging.DEBUG)
    if args.command == "spider":
        write_spider(url=args.url, depth=args.depth, filename=args.output)
    if args.command == "walker":
        write_walker(start_dir=args.start_dir, filename=args.output)
    elif args.command == "clear":
        clear(service=args.service)
    elif args.command == "from-thredds":
        feed_from_thredds(
            service=args.service,
            catalog_url=args.catalog_url,
            depth=args.depth,
            maxrecords=args.maxrecords,
            batch_size=args.batch_size,
        )
    elif args.command == "from-walker":
        feed_from_walker(
            service=args.service, start_dir=args.start_dir, maxrecords=args.maxrecords, batch_size=args.batch_size
        )
    elif args.command == "from-spider":
        feed_from_spider(
            service=args.service, url=args.url, depth=args.depth, maxrecords=args.maxrecords, batch_size=args.batch_size
        )
    logger.info("Done.")
Beispiel #2
0
def test_write_walker():
    client.write_walker(start_dir=TESTDATA_PATH, filename="/tmp/out_walker.csv")