def ingest_cmd(driver_manager, config_file, year, queue_size, save_tasks, load_tasks, dry_run, executor): index = driver_manager.index if config_file: config = load_config_from_file(index, config_file) variable_params = get_variable_params(config) source_type, output_type = make_output_type(driver_manager, config) tasks = create_task_list(driver_manager, output_type, year, source_type, config) elif load_tasks: config, tasks = load_tasks_(load_tasks) source_type, output_type = make_output_type(driver_manager, config) else: click.echo('Must specify exactly one of --config-file, --load-tasks') return 1 if dry_run: check_existing_files( get_filename(config, task['tile_index'], task['tile'].sources) for task in tasks) return 0 if save_tasks: save_tasks_(config, tasks, save_tasks) return 0 successful, failed = process_tasks(driver_manager, config, source_type, output_type, tasks, queue_size, executor) click.echo('%d successful, %d failed' % (successful, failed)) return 0
def ingest_cmd(index, config_file, year, queue_size, save_tasks, load_tasks, dry_run, allow_product_changes, executor): # pylint: disable=too-many-locals if config_file: config = load_config_from_file(config_file) elif load_tasks: config, tasks = load_tasks_(load_tasks) else: click.echo('Must specify exactly one of --config-file, --load-tasks') sys.exit(-1) try: # ignore the added filename key which is not part of the schema filename = config['filename'] del config['filename'] IngestorConfig.validate(config) config['filename'] = filename except InvalidDocException as e: exception, = e.args _LOG.error(exception.message) sys.exit(-1) if config_file: driver = get_driver_from_config(config) source_type, output_type = ensure_output_type( index, config, driver.format, allow_product_changes=allow_product_changes) tasks = create_task_list(index, output_type, year, source_type, config) elif load_tasks: driver = get_driver_from_config(config) source_type, output_type = ensure_output_type( index, config, driver.format, allow_product_changes=allow_product_changes) if dry_run: check_existing_files( get_filename(config, task['tile_index'], task['tile'].sources) for task in tasks) elif save_tasks: save_tasks_(config, tasks, save_tasks) else: successful, failed = process_tasks(index, config, source_type, output_type, tasks, queue_size, executor) click.echo('%d successful, %d failed' % (successful, failed)) sys.exit(failed)
def ingest_cmd(index, config_file, year, queue_size, save_tasks, load_tasks, dry_run, allow_product_changes, executor): # pylint: disable=too-many-locals try: if config_file: config, tasks = load_config_from_file(config_file), None elif load_tasks: config, tasks = load_tasks_(load_tasks) else: click.echo( 'Must specify exactly one of --config-file, --load-tasks') sys.exit(-1) except InvalidDocException as e: exception, = e.args _LOG.error(exception.message) sys.exit(-1) driver = get_driver_from_config(config) try: source_type, output_type = ensure_output_type( index, config, driver.format, allow_product_changes=allow_product_changes) except ValueError as e: _LOG.error(str(e)) sys.exit(-1) if tasks is None: tasks = create_task_list(index, output_type, year, source_type, config) if dry_run: check_existing_files( get_filename(config, task['tile_index'], task['tile'].sources) for task in tasks) elif save_tasks: save_tasks_(config, tasks, save_tasks) else: successful, failed = process_tasks(index, config, source_type, output_type, tasks, queue_size, executor) click.echo('%d successful, %d failed' % (successful, failed)) sys.exit(failed)
def ingest_cmd(index, config_file, year, queue_size, save_tasks, load_tasks, dry_run, executor, allow_product_changes): # pylint: disable=too-many-locals if config_file: config = load_config_from_file(config_file) driver = get_driver_from_config(config) source_type, output_type = ensure_output_type( index, config, driver.format, allow_product_changes=allow_product_changes) tasks = create_task_list(index, output_type, year, source_type, config) elif load_tasks: config, tasks = load_tasks_(load_tasks) driver = get_driver_from_config(config) source_type, output_type = ensure_output_type( index, config, driver.format, allow_product_changes=allow_product_changes) else: click.echo('Must specify exactly one of --config-file, --load-tasks') sys.exit(-1) if dry_run: check_existing_files( get_filename(config, task['tile_index'], task['tile'].sources) for task in tasks) elif save_tasks: save_tasks_(config, tasks, save_tasks) else: successful, failed = process_tasks(index, config, source_type, output_type, tasks, queue_size, executor) click.echo('%d successful, %d failed' % (successful, failed)) sys.exit(failed)