def ingest_cmd(driver_manager, config_file, year, queue_size, save_tasks,
load_tasks, dry_run, executor):
index = driver_manager.index
if config_file:
config = load_config_from_file(index, config_file)
variable_params = get_variable_params(config)
source_type, output_type = make_output_type(driver_manager, config)
tasks = create_task_list(driver_manager, output_type, year,
source_type, config)
elif load_tasks:
config, tasks = load_tasks_(load_tasks)
source_type, output_type = make_output_type(driver_manager, config)
else:
click.echo('Must specify exactly one of --config-file, --load-tasks')
return 1
if dry_run:
check_existing_files(
get_filename(config, task['tile_index'], task['tile'].sources)
for task in tasks)
return 0
if save_tasks:
save_tasks_(config, tasks, save_tasks)
return 0
successful, failed = process_tasks(driver_manager, config, source_type,
output_type, tasks, queue_size,
executor)
click.echo('%d successful, %d failed' % (successful, failed))
return 0
def ingest_cmd(index, config_file, year, queue_size, save_tasks, load_tasks,
dry_run, allow_product_changes, executor):
# pylint: disable=too-many-locals
if config_file:
config = load_config_from_file(config_file)
elif load_tasks:
config, tasks = load_tasks_(load_tasks)
else:
click.echo('Must specify exactly one of --config-file, --load-tasks')
sys.exit(-1)
try:
# ignore the added filename key which is not part of the schema
filename = config['filename']
del config['filename']
IngestorConfig.validate(config)
config['filename'] = filename
except InvalidDocException as e:
exception, = e.args
_LOG.error(exception.message)
sys.exit(-1)
if config_file:
driver = get_driver_from_config(config)
source_type, output_type = ensure_output_type(
index,
config,
driver.format,
allow_product_changes=allow_product_changes)
tasks = create_task_list(index, output_type, year, source_type, config)
elif load_tasks:
driver = get_driver_from_config(config)
source_type, output_type = ensure_output_type(
index,
config,
driver.format,
allow_product_changes=allow_product_changes)
if dry_run:
check_existing_files(
get_filename(config, task['tile_index'], task['tile'].sources)
for task in tasks)
elif save_tasks:
save_tasks_(config, tasks, save_tasks)
else:
successful, failed = process_tasks(index, config, source_type,
output_type, tasks, queue_size,
executor)
click.echo('%d successful, %d failed' % (successful, failed))
sys.exit(failed)
def ingest_cmd(index, config_file, year, queue_size, save_tasks, load_tasks,
dry_run, allow_product_changes, executor):
# pylint: disable=too-many-locals
try:
if config_file:
config, tasks = load_config_from_file(config_file), None
elif load_tasks:
config, tasks = load_tasks_(load_tasks)
else:
click.echo(
'Must specify exactly one of --config-file, --load-tasks')
sys.exit(-1)
except InvalidDocException as e:
exception, = e.args
_LOG.error(exception.message)
sys.exit(-1)
driver = get_driver_from_config(config)
try:
source_type, output_type = ensure_output_type(
index,
config,
driver.format,
allow_product_changes=allow_product_changes)
except ValueError as e:
_LOG.error(str(e))
sys.exit(-1)
if tasks is None:
tasks = create_task_list(index, output_type, year, source_type, config)
if dry_run:
check_existing_files(
get_filename(config, task['tile_index'], task['tile'].sources)
for task in tasks)
elif save_tasks:
save_tasks_(config, tasks, save_tasks)
else:
successful, failed = process_tasks(index, config, source_type,
output_type, tasks, queue_size,
executor)
click.echo('%d successful, %d failed' % (successful, failed))
sys.exit(failed)
def ingest_cmd(index, config_file, year, queue_size, save_tasks, load_tasks,
dry_run, executor, allow_product_changes):
# pylint: disable=too-many-locals
if config_file:
config = load_config_from_file(config_file)
driver = get_driver_from_config(config)
source_type, output_type = ensure_output_type(
index,
config,
driver.format,
allow_product_changes=allow_product_changes)
tasks = create_task_list(index, output_type, year, source_type, config)
elif load_tasks:
config, tasks = load_tasks_(load_tasks)
driver = get_driver_from_config(config)
source_type, output_type = ensure_output_type(
index,
config,
driver.format,
allow_product_changes=allow_product_changes)
else:
click.echo('Must specify exactly one of --config-file, --load-tasks')
sys.exit(-1)
if dry_run:
check_existing_files(
get_filename(config, task['tile_index'], task['tile'].sources)
for task in tasks)
elif save_tasks:
save_tasks_(config, tasks, save_tasks)
else:
successful, failed = process_tasks(index, config, source_type,
output_type, tasks, queue_size,
executor)
click.echo('%d successful, %d failed' % (successful, failed))
sys.exit(failed)
