def run(index, dry_run: bool, tag: str, task_desc_file: str, qsub: QSubLauncher, runner: TaskRunner, *args, **kwargs): _LOG.info('Starting Fractional Cover processing...') _LOG.info('Tag: %r', tag) task_desc = serialise.load_structure(Path(task_desc_file), TaskDescription) config, tasks = task_app.load_tasks(task_desc.runtime_state.task_serialisation_path) if dry_run: task_app.check_existing_files((task['filename'] for task in tasks)) return 0 task_func = partial(do_fc_task, config) process_func = partial(process_result, index) try: runner(task_desc, tasks, task_func, process_func) _LOG.info("Runner finished normally, triggering shutdown.") finally: runner.stop()
def run(index, dry_run: bool, task_desc_file: str, runner: TaskRunner, qsub): _LOG.info('Starting DEA Stacker processing...') task_desc = serialise.load_structure(Path(task_desc_file), TaskDescription) config, tasks = task_app.load_tasks( task_desc.runtime_state.task_serialisation_path) if dry_run: task_app.check_existing_files((task['filename'] for task in tasks)) return task_func = partial(stacker.do_stack_task, config) process_func = partial(stacker.process_result, index) try: runner(task_desc, tasks, task_func, process_func) _LOG.info("Runner finished normally, triggering shutdown.") finally: runner.stop()
def run(index, dry_run: bool, input_filename: str, runner: TaskRunner, skip_indexing: bool, **kwargs): config, tasks = task_app.load_tasks(input_filename) work_dir = Path(input_filename).parent # TODO: Get rid of this completely task_desc = TaskDescription( type_='fc', task_dt=datetime.utcnow().astimezone(timezone.utc), events_path=work_dir, logs_path=work_dir, jobs_path=work_dir, parameters=None, runtime_state=None, ) if dry_run: _LOG.info('Starting Fractional Cover Dry Run...') task_app.check_existing_files( (task['filename_dataset'] for task in tasks)) return 0 _LOG.info('Starting Fractional Cover processing...') task_func = partial(_do_fc_task, config) if skip_indexing: process_func = _skip_indexing_and_only_log else: process_func = partial(_index_datasets, index) try: runner(task_desc, tasks, task_func, process_func) _LOG.info("Runner finished normally, triggering shutdown.") except Exception as err: if "Error 104" in err: _LOG.info( "Processing completed and shutdown was initiated. Exception: %s", str(err)) else: _LOG.info("Exception during processing: %s", err) finally: runner.stop() return 0
def run_tasks(self, tasks, runner=None, task_slice=None): from digitalearthau.qsub import TaskRunner from digitalearthau.runners.model import TaskDescription, DefaultJobParameters if task_slice is not None: tasks = islice(tasks, task_slice.start, task_slice.stop, task_slice.step) output_driver = self._partially_applied_output_driver() task_runner = partial(execute_task, output_driver=output_driver, chunking=self.computation.get('chunking', {})) # does not need to be thorough for now task_desc = TaskDescription( type_='datacube_stats', task_dt=datetime.utcnow().replace(tzinfo=tz.tzutc()), events_path=Path(self.location) / 'events', logs_path=Path(self.location) / 'logs', jobs_path=Path(self.location) / 'jobs', parameters=DefaultJobParameters(query={}, source_products=[], output_products=[])) task_desc.logs_path.mkdir(parents=True, exist_ok=True) task_desc.events_path.mkdir(parents=True, exist_ok=True) task_desc.jobs_path.mkdir(parents=True, exist_ok=True) if runner is None: runner = TaskRunner() result = runner(task_desc, tasks, task_runner) _LOG.debug('Stopping runner.') runner.stop() _LOG.debug('Runner stopped.') return result
def run(index, input_filename: str, runner: TaskRunner, skip_indexing: bool, redirect_outputs: str, **kwargs): """ Process WOfS tasks from a task file. """ config, tasks = task_app.load_tasks(input_filename) work_dir = Path(input_filename).parent if redirect_outputs is not None: tasks = _prepend_path_to_tasks(redirect_outputs, tasks) # TODO: Get rid of this completely task_desc = TaskDescription( type_='wofs', task_dt=datetime.utcnow().astimezone(timezone.utc), events_path=work_dir, logs_path=work_dir, jobs_path=work_dir, parameters=None, runtime_state=None, ) _LOG.info('Starting WOfS processing...') task_func = partial(_do_wofs_task, config) if skip_indexing: process_func = _skip_indexing_and_only_log else: process_func = partial(_index_datasets, index) try: runner(task_desc, tasks, task_func, process_func) _LOG.info("Runner finished normally, triggering shutdown.") finally: runner.stop() # TODO: Check for failures and return error state sys.exit(0)