Esempio n. 1
0
def execute(args):
    if args.verbose:
        logger.setLevel(logging.DEBUG)
    if args.command == "spider":
        write_spider(url=args.url, depth=args.depth, filename=args.output)
    if args.command == "walker":
        write_walker(start_dir=args.start_dir, filename=args.output)
    elif args.command == "clear":
        clear(service=args.service)
    elif args.command == "from-thredds":
        feed_from_thredds(
            service=args.service,
            catalog_url=args.catalog_url,
            depth=args.depth,
            maxrecords=args.maxrecords,
            batch_size=args.batch_size,
        )
    elif args.command == "from-walker":
        feed_from_walker(
            service=args.service, start_dir=args.start_dir, maxrecords=args.maxrecords, batch_size=args.batch_size
        )
    elif args.command == "from-spider":
        feed_from_spider(
            service=args.service, url=args.url, depth=args.depth, maxrecords=args.maxrecords, batch_size=args.batch_size
        )
    logger.info("Done.")
Esempio n. 2
0
def test_write_spider():
    page="http://ensemblesrt3.dmi.dk/data/CORDEX/AFR-44/KNMI/MOHC-HadGEM2-ES/rcp45/r1i1p1/KNMI-RACMO22T/v1/mon/tasmax/"
    client.write_spider(page, depth=1, filename="/tmp/out_spider.csv")