def execute(args): if args.verbose: logger.setLevel(logging.DEBUG) if args.command == "spider": write_spider(url=args.url, depth=args.depth, filename=args.output) if args.command == "walker": write_walker(start_dir=args.start_dir, filename=args.output) elif args.command == "clear": clear(service=args.service) elif args.command == "from-thredds": feed_from_thredds( service=args.service, catalog_url=args.catalog_url, depth=args.depth, maxrecords=args.maxrecords, batch_size=args.batch_size, ) elif args.command == "from-walker": feed_from_walker( service=args.service, start_dir=args.start_dir, maxrecords=args.maxrecords, batch_size=args.batch_size ) elif args.command == "from-spider": feed_from_spider( service=args.service, url=args.url, depth=args.depth, maxrecords=args.maxrecords, batch_size=args.batch_size ) logger.info("Done.")
def test_write_spider(): page="http://ensemblesrt3.dmi.dk/data/CORDEX/AFR-44/KNMI/MOHC-HadGEM2-ES/rcp45/r1i1p1/KNMI-RACMO22T/v1/mon/tasmax/" client.write_spider(page, depth=1, filename="/tmp/out_spider.csv")