def generate(index: Index, task_desc_file: str, no_qsub: bool, tag: str,
email_options: str, email_id: str, dry_run: bool):
"""
Generate Tasks into file and Queue PBS job to process them.
If dry run is enabled, do not add the new products to the database.
"""
_LOG.info('Tag: %s', tag)
config, task_desc = _make_config_and_description(index,
Path(task_desc_file),
dry_run)
num_tasks_saved = task_app.save_tasks(
config, _make_fc_tasks(index, config,
query=task_desc.parameters.query),
task_desc.runtime_state.task_serialisation_path)
_LOG.info('Found %d tasks', num_tasks_saved)
if not num_tasks_saved:
_LOG.info("No tasks. Finishing.")
return 0
nodes, walltime = _estimate_job_size(num_tasks_saved)
_LOG.info('Will request %d nodes and %s time', nodes, walltime)
if no_qsub:
_LOG.info('Skipping submission due to --no-qsub')
return 0
# If dry run is not enabled just pass verbose option
dry_run_option = '--dry-run' if dry_run else '-v'
extra_qsub_args = '-M {0} -m {1}'.format(email_id, email_options)
# Append email options and email id to the PbsParameters dict key, extra_qsub_args
task_desc.runtime_state.pbs_parameters.extra_qsub_args.extend(
extra_qsub_args.split(' '))
submit_subjob(
name='run',
task_desc=task_desc,
command=[
'run',
'-vv',
'--task-desc',
str(task_desc_file),
'--celery',
'pbs-launch',
'--tag',
tag,
dry_run_option,
],
qsub_params=dict(name='fc-run-{}'.format(tag),
mem='medium',
wd=True,
nodes=nodes,
walltime=walltime),
)
def generate(index: Index,
task_desc_file: str,
no_qsub: bool,
tag: str):
_LOG.info('Tag: %s', tag)
config, task_desc = _make_config_and_description(index, Path(task_desc_file))
num_tasks_saved = task_app.save_tasks(
config,
make_fc_tasks(index, config, query=task_desc.parameters.query),
task_desc.runtime_state.task_serialisation_path
)
_LOG.info('Found %d tasks', num_tasks_saved)
if not num_tasks_saved:
_LOG.info("No tasks. Finishing.")
return 0
nodes, walltime = estimate_job_size(num_tasks_saved)
_LOG.info('Will request %d nodes and %s time', nodes, walltime)
if no_qsub:
_LOG.info('Skipping submission due to --no-qsub')
return 0
submit_subjob(
name='run',
task_desc=task_desc,
command=[
'run',
'-vv',
'--task-desc', str(task_desc_file),
'--celery', 'pbs-launch',
'--tag', tag,
],
qsub_params=dict(
name='fc-run-{}'.format(tag),
mem='small',
wd=True,
nodes=nodes,
walltime=walltime
),
)
def submit(index: Index,
app_config: str,
project: str,
queue: str,
no_qsub: bool,
time_range: Tuple[datetime, datetime],
tag: str):
_LOG.info('Tag: %s', tag)
app_config_path = Path(app_config).resolve()
app_config = paths.read_document(app_config_path)
task_desc, task_path = init_task_app(
job_type="fc",
source_products=[app_config['source_product']],
output_products=[app_config['output_product']],
# TODO: Use @datacube.ui.click.parsed_search_expressions to allow params other than time from the cli?
datacube_query_args=Query(index=index, time=time_range).search_terms,
app_config_path=app_config_path,
pbs_project=project,
pbs_queue=queue
)
_LOG.info("Created task description: %s", task_path)
if no_qsub:
_LOG.info('Skipping submission due to --no-qsub')
return 0
submit_subjob(
name='generate',
task_desc=task_desc,
command=[
'generate', '-v', '-v',
'--task-desc', str(task_path),
'--tag', tag
],
qsub_params=dict(
mem='20G',
wd=True,
ncpus=1,
walltime='1h',
name='fc-generate-{}'.format(tag)
)
)
def submit(index: Index, app_config: str, project: str, queue: str,
no_qsub: bool, time_range: Tuple[datetime, datetime], tag: str,
email_options: str, email_id: str, dry_run: bool):
"""
Kick off two stage PBS job
Stage 1 (Generate task file):
The task-app machinery loads a config file, from a path specified on the
command line, into a dict.
If dry is enabled, a dummy DatasetType is created for tasks generation without indexing
the product in the database.
If dry run is disabled, generate tasks into file and queue PBS job to process them.
Stage 2 (Run):
During normal run, following are performed:
1) Tasks shall be yielded for dispatch to workers.
2) Load data
3) Run FC algorithm
4) Attach metadata
5) Write output files and
6) Finally index the newly created FC output netCDF files
If dry run is enabled, application only prepares a list of output files to be created and does not
record anything in the database.
"""
_LOG.info('Tag: %s', tag)
app_config_path = Path(app_config).resolve()
app_config = paths.read_document(app_config_path)
if not time_range or not all(time_range):
query_args = Query(index=index).search_terms
else:
query_args = Query(index=index, time=time_range).search_terms
task_desc, task_path = init_task_app(
job_type="fc",
source_products=[app_config['source_product']],
output_products=[app_config['output_product']],
# TODO: Use @datacube.ui.click.parsed_search_expressions to allow params other than time from the cli?
datacube_query_args=query_args,
app_config_path=app_config_path,
pbs_project=project,
pbs_queue=queue)
_LOG.info("Created task description: %s", task_path)
if no_qsub:
_LOG.info('Skipping submission due to --no-qsub')
return 0
# If dry run is not enabled just pass verbose option
dry_run_option = '--dry-run' if dry_run else '-v'
extra_qsub_args = '-M {0} -m {1}'.format(email_id, email_options)
# Append email options and email id to the PbsParameters dict key, extra_qsub_args
task_desc.runtime_state.pbs_parameters.extra_qsub_args.extend(
extra_qsub_args.split(' '))
submit_subjob(name='generate',
task_desc=task_desc,
command=[
'generate',
'-vv',
'--task-desc',
str(task_path),
'--tag',
tag,
'--log-queries',
'--email-id',
email_id,
'--email-options',
email_options,
dry_run_option,
],
qsub_params=dict(name='fc-generate-{}'.format(tag),
mem='medium',
wd=True,
nodes=1,
walltime='1h'))