def main(config, path, threads, overwrite): with open(config, 'rb') as fh: config = yaml.load(fh) if path is not None: config['path'] = path if threads is not None: config['threads'] = threads if overwrite is not None: config['overwrite'] = overwrite # TODO: validate config format mfk = MetaFolderKrauler(config) mfk.run()
def crawl_to_metafolder(config): configure_logging() mfk = MetaFolderKrauler(config) mfk.metafolder # Show storage location for output return mfk.run()