Exemple #1
0
def main(config, path, threads, overwrite):
    with open(config, 'rb') as fh:
        config = yaml.load(fh)

    if path is not None:
        config['path'] = path

    if threads is not None:
        config['threads'] = threads

    if overwrite is not None:
        config['overwrite'] = overwrite

    # TODO: validate config format
    mfk = MetaFolderKrauler(config)
    mfk.run()
Exemple #2
0
def crawl_to_metafolder(config):
    configure_logging()
    mfk = MetaFolderKrauler(config)
    mfk.metafolder  # Show storage location for output
    return mfk.run()