def app_main(db_index, config, tasks, executor, **opts):
"""
make_config => config
config => make_tasks => tasks
"""
from pickle import dumps
unused(db_index, opts, config)
click.echo('Using executor {}'.format(repr(executor)))
task_runner = wrap_task(run_task, config['op'])
click.echo('Task function size: {}'.format(len(dumps(task_runner))))
run_tasks(tasks, executor, task_runner, queue_size=10)
return 0
def main(num_tasks, op, qsub=None, runner=None):
""" test app for qsub/runner
"""
if qsub is not None:
click.echo(repr(qsub))
return qsub('--op', op, num_tasks)
click.echo(datacube.__file__)
click.echo('PWD:' + str(pathlib.Path('.').absolute()))
click.echo('num_tasks: ' + str(num_tasks))
if runner.start() is False:
click.echo('Failed to launch worker pool')
return 1
runner(task_generator(num_tasks), wrap_task(run_task, op),
log_completed_task)
click.echo('Shutting down worker pool')
runner.stop()
click.echo('All done!')
return 0
def test_wrap_task():
def task_with_args(task, a, b):
return (task, a, b)
assert task_with_args(1, 2, 'a') == (1, 2, 'a')
assert wrap_task(task_with_args, 'a', 'b')(0) == (0, 'a', 'b')