def main(config, path, threads, overwrite):
with open(config, 'rb') as fh:
config = yaml.load(fh)
if path is not None:
config['path'] = path
if threads is not None:
config['threads'] = threads
if overwrite is not None:
config['overwrite'] = overwrite
# TODO: validate config format
mfk = MetaFolderKrauler(config)
mfk.run()
def crawl_to_metafolder(config):
configure_logging()
mfk = MetaFolderKrauler(config)
mfk.metafolder # Show storage location for output
return mfk.run()