def run(index,
dry_run: bool,
tag: str,
task_desc_file: str,
qsub: QSubLauncher,
runner: TaskRunner,
*args, **kwargs):
_LOG.info('Starting Fractional Cover processing...')
_LOG.info('Tag: %r', tag)
task_desc = serialise.load_structure(Path(task_desc_file), TaskDescription)
config, tasks = task_app.load_tasks(task_desc.runtime_state.task_serialisation_path)
if dry_run:
task_app.check_existing_files((task['filename'] for task in tasks))
return 0
task_func = partial(do_fc_task, config)
process_func = partial(process_result, index)
try:
runner(task_desc, tasks, task_func, process_func)
_LOG.info("Runner finished normally, triggering shutdown.")
finally:
runner.stop()
def run(index, dry_run: bool, task_desc_file: str, runner: TaskRunner, qsub):
_LOG.info('Starting DEA Stacker processing...')
task_desc = serialise.load_structure(Path(task_desc_file), TaskDescription)
config, tasks = task_app.load_tasks(
task_desc.runtime_state.task_serialisation_path)
if dry_run:
task_app.check_existing_files((task['filename'] for task in tasks))
return
task_func = partial(stacker.do_stack_task, config)
process_func = partial(stacker.process_result, index)
try:
runner(task_desc, tasks, task_func, process_func)
_LOG.info("Runner finished normally, triggering shutdown.")
finally:
runner.stop()
def run(index, dry_run: bool, input_filename: str, runner: TaskRunner,
skip_indexing: bool, **kwargs):
config, tasks = task_app.load_tasks(input_filename)
work_dir = Path(input_filename).parent
# TODO: Get rid of this completely
task_desc = TaskDescription(
type_='fc',
task_dt=datetime.utcnow().astimezone(timezone.utc),
events_path=work_dir,
logs_path=work_dir,
jobs_path=work_dir,
parameters=None,
runtime_state=None,
)
if dry_run:
_LOG.info('Starting Fractional Cover Dry Run...')
task_app.check_existing_files(
(task['filename_dataset'] for task in tasks))
return 0
_LOG.info('Starting Fractional Cover processing...')
task_func = partial(_do_fc_task, config)
if skip_indexing:
process_func = _skip_indexing_and_only_log
else:
process_func = partial(_index_datasets, index)
try:
runner(task_desc, tasks, task_func, process_func)
_LOG.info("Runner finished normally, triggering shutdown.")
except Exception as err:
if "Error 104" in err:
_LOG.info(
"Processing completed and shutdown was initiated. Exception: %s",
str(err))
else:
_LOG.info("Exception during processing: %s", err)
finally:
runner.stop()
return 0
def run_tasks(self, tasks, runner=None, task_slice=None):
from digitalearthau.qsub import TaskRunner
from digitalearthau.runners.model import TaskDescription, DefaultJobParameters
if task_slice is not None:
tasks = islice(tasks, task_slice.start, task_slice.stop,
task_slice.step)
output_driver = self._partially_applied_output_driver()
task_runner = partial(execute_task,
output_driver=output_driver,
chunking=self.computation.get('chunking', {}))
# does not need to be thorough for now
task_desc = TaskDescription(
type_='datacube_stats',
task_dt=datetime.utcnow().replace(tzinfo=tz.tzutc()),
events_path=Path(self.location) / 'events',
logs_path=Path(self.location) / 'logs',
jobs_path=Path(self.location) / 'jobs',
parameters=DefaultJobParameters(query={},
source_products=[],
output_products=[]))
task_desc.logs_path.mkdir(parents=True, exist_ok=True)
task_desc.events_path.mkdir(parents=True, exist_ok=True)
task_desc.jobs_path.mkdir(parents=True, exist_ok=True)
if runner is None:
runner = TaskRunner()
result = runner(task_desc, tasks, task_runner)
_LOG.debug('Stopping runner.')
runner.stop()
_LOG.debug('Runner stopped.')
return result
def run(index, input_filename: str, runner: TaskRunner, skip_indexing: bool,
redirect_outputs: str, **kwargs):
"""
Process WOfS tasks from a task file.
"""
config, tasks = task_app.load_tasks(input_filename)
work_dir = Path(input_filename).parent
if redirect_outputs is not None:
tasks = _prepend_path_to_tasks(redirect_outputs, tasks)
# TODO: Get rid of this completely
task_desc = TaskDescription(
type_='wofs',
task_dt=datetime.utcnow().astimezone(timezone.utc),
events_path=work_dir,
logs_path=work_dir,
jobs_path=work_dir,
parameters=None,
runtime_state=None,
)
_LOG.info('Starting WOfS processing...')
task_func = partial(_do_wofs_task, config)
if skip_indexing:
process_func = _skip_indexing_and_only_log
else:
process_func = partial(_index_datasets, index)
try:
runner(task_desc, tasks, task_func, process_func)
_LOG.info("Runner finished normally, triggering shutdown.")
finally:
runner.stop()
# TODO: Check for failures and return error state
sys.exit(0)