def test_dump_load_task_structure(tmpdir):
# Dump to json and reload, check equality.
d = Path(str(tmpdir))
task_description = TaskDescription(
type_="reproject",
task_dt=datetime.datetime.utcnow(),
events_path=d.joinpath('events'),
logs_path=d.joinpath('logs'),
parameters=DefaultJobParameters(
query={'time': [2013, 2015]},
source_products=['ls5_nbar_albers'],
output_products=['ls5_nbar_waterman_butterfly'],
),
# Task-app framework
runtime_state=TaskAppState(
config_path=Path('config.test.yaml'),
task_serialisation_path=d.joinpath('generated-tasks.pickle'),
)
)
serialised_file = d.joinpath('task_description.json')
serialise.dump_structure(serialised_file, task_description)
result = serialise.load_structure(serialised_file, expected_type=TaskDescription)
assert result == task_description
def run(self, runner, task_file=None, task_slice=None):
if task_file:
tasks = unpickle_stream(task_file)
else:
tasks = self.generate_tasks(self.configure_outputs())
if task_slice is not None:
tasks = islice(tasks, task_slice.start, task_slice.stop,
task_slice.step)
app_info = _get_app_metadata(self.config_file)
output_driver = partial(self.output_driver,
output_path=self.location,
app_info=app_info,
storage=self.storage,
global_attributes=self.global_attributes,
var_attributes=self.var_attributes)
task_runner = partial(execute_task,
output_driver=output_driver,
chunking=self.computation.get('chunking', {}))
# does not need to be thorough for now
task_desc = TaskDescription(
type_='datacube_stats',
task_dt=datetime.utcnow().replace(tzinfo=tz.tzutc()),
events_path=Path(self.location),
logs_path=Path(self.location),
parameters=DefaultJobParameters(query={},
source_products=[],
output_products=[]))
result = runner(task_desc, tasks, task_runner)
_LOG.debug('Stopping runner.')
runner.stop()
_LOG.debug('Runner stopped.')
return result
def run_tasks(self, tasks, runner=None, task_slice=None):
from digitalearthau.qsub import TaskRunner
from digitalearthau.runners.model import TaskDescription, DefaultJobParameters
if task_slice is not None:
tasks = islice(tasks, task_slice.start, task_slice.stop,
task_slice.step)
output_driver = self._partially_applied_output_driver()
task_runner = partial(execute_task,
output_driver=output_driver,
chunking=self.computation.get('chunking', {}))
# does not need to be thorough for now
task_desc = TaskDescription(
type_='datacube_stats',
task_dt=datetime.utcnow().replace(tzinfo=tz.tzutc()),
events_path=Path(self.location) / 'events',
logs_path=Path(self.location) / 'logs',
jobs_path=Path(self.location) / 'jobs',
parameters=DefaultJobParameters(query={},
source_products=[],
output_products=[]))
task_desc.logs_path.mkdir(parents=True, exist_ok=True)
task_desc.events_path.mkdir(parents=True, exist_ok=True)
task_desc.jobs_path.mkdir(parents=True, exist_ok=True)
if runner is None:
runner = TaskRunner()
result = runner(task_desc, tasks, task_runner)
_LOG.debug('Stopping runner.')
runner.stop()
_LOG.debug('Runner stopped.')
return result