Пример #1
0
Файл: cli.py Проект: e6/krauler
#!/usr/bin/python
import yaml
import click

from krauler.util import configure_logging
from krauler.mf import MetaFolderKrauler

configure_logging()


@click.command()
@click.argument('config', type=click.Path(exists=True))
@click.option('--path', type=click.Path(), default=None)
@click.option('--threads', '-t', type=int, default=None)
@click.option('--overwrite', '-o', type=bool, default=None)
def main(config, path, threads, overwrite):
    with open(config, 'rb') as fh:
        config = yaml.load(fh)

    if path is not None:
        config['path'] = path

    if threads is not None:
        config['threads'] = threads

    if overwrite is not None:
        config['overwrite'] = overwrite

    # TODO: validate config format
    mfk = MetaFolderKrauler(config)
    mfk.run()
Пример #2
0
def crawl_to_metafolder(config):
    configure_logging()
    mfk = MetaFolderKrauler(config)
    mfk.metafolder  # Show storage location for output
    return mfk.run()