def generate(index: Index, task_desc_file: str, no_qsub: bool, tag: str, email_options: str, email_id: str, dry_run: bool): """ Generate Tasks into file and Queue PBS job to process them. If dry run is enabled, do not add the new products to the database. """ _LOG.info('Tag: %s', tag) config, task_desc = _make_config_and_description(index, Path(task_desc_file), dry_run) num_tasks_saved = task_app.save_tasks( config, _make_fc_tasks(index, config, query=task_desc.parameters.query), task_desc.runtime_state.task_serialisation_path) _LOG.info('Found %d tasks', num_tasks_saved) if not num_tasks_saved: _LOG.info("No tasks. Finishing.") return 0 nodes, walltime = _estimate_job_size(num_tasks_saved) _LOG.info('Will request %d nodes and %s time', nodes, walltime) if no_qsub: _LOG.info('Skipping submission due to --no-qsub') return 0 # If dry run is not enabled just pass verbose option dry_run_option = '--dry-run' if dry_run else '-v' extra_qsub_args = '-M {0} -m {1}'.format(email_id, email_options) # Append email options and email id to the PbsParameters dict key, extra_qsub_args task_desc.runtime_state.pbs_parameters.extra_qsub_args.extend( extra_qsub_args.split(' ')) submit_subjob( name='run', task_desc=task_desc, command=[ 'run', '-vv', '--task-desc', str(task_desc_file), '--celery', 'pbs-launch', '--tag', tag, dry_run_option, ], qsub_params=dict(name='fc-run-{}'.format(tag), mem='medium', wd=True, nodes=nodes, walltime=walltime), )
def generate(index: Index, task_desc_file: str, no_qsub: bool, tag: str): _LOG.info('Tag: %s', tag) config, task_desc = _make_config_and_description(index, Path(task_desc_file)) num_tasks_saved = task_app.save_tasks( config, make_fc_tasks(index, config, query=task_desc.parameters.query), task_desc.runtime_state.task_serialisation_path ) _LOG.info('Found %d tasks', num_tasks_saved) if not num_tasks_saved: _LOG.info("No tasks. Finishing.") return 0 nodes, walltime = estimate_job_size(num_tasks_saved) _LOG.info('Will request %d nodes and %s time', nodes, walltime) if no_qsub: _LOG.info('Skipping submission due to --no-qsub') return 0 submit_subjob( name='run', task_desc=task_desc, command=[ 'run', '-vv', '--task-desc', str(task_desc_file), '--celery', 'pbs-launch', '--tag', tag, ], qsub_params=dict( name='fc-run-{}'.format(tag), mem='small', wd=True, nodes=nodes, walltime=walltime ), )
def submit(index: Index, app_config: str, project: str, queue: str, no_qsub: bool, time_range: Tuple[datetime, datetime], tag: str): _LOG.info('Tag: %s', tag) app_config_path = Path(app_config).resolve() app_config = paths.read_document(app_config_path) task_desc, task_path = init_task_app( job_type="fc", source_products=[app_config['source_product']], output_products=[app_config['output_product']], # TODO: Use @datacube.ui.click.parsed_search_expressions to allow params other than time from the cli? datacube_query_args=Query(index=index, time=time_range).search_terms, app_config_path=app_config_path, pbs_project=project, pbs_queue=queue ) _LOG.info("Created task description: %s", task_path) if no_qsub: _LOG.info('Skipping submission due to --no-qsub') return 0 submit_subjob( name='generate', task_desc=task_desc, command=[ 'generate', '-v', '-v', '--task-desc', str(task_path), '--tag', tag ], qsub_params=dict( mem='20G', wd=True, ncpus=1, walltime='1h', name='fc-generate-{}'.format(tag) ) )
def submit(index: Index, app_config: str, project: str, queue: str, no_qsub: bool, time_range: Tuple[datetime, datetime], tag: str, email_options: str, email_id: str, dry_run: bool): """ Kick off two stage PBS job Stage 1 (Generate task file): The task-app machinery loads a config file, from a path specified on the command line, into a dict. If dry is enabled, a dummy DatasetType is created for tasks generation without indexing the product in the database. If dry run is disabled, generate tasks into file and queue PBS job to process them. Stage 2 (Run): During normal run, following are performed: 1) Tasks shall be yielded for dispatch to workers. 2) Load data 3) Run FC algorithm 4) Attach metadata 5) Write output files and 6) Finally index the newly created FC output netCDF files If dry run is enabled, application only prepares a list of output files to be created and does not record anything in the database. """ _LOG.info('Tag: %s', tag) app_config_path = Path(app_config).resolve() app_config = paths.read_document(app_config_path) if not time_range or not all(time_range): query_args = Query(index=index).search_terms else: query_args = Query(index=index, time=time_range).search_terms task_desc, task_path = init_task_app( job_type="fc", source_products=[app_config['source_product']], output_products=[app_config['output_product']], # TODO: Use @datacube.ui.click.parsed_search_expressions to allow params other than time from the cli? datacube_query_args=query_args, app_config_path=app_config_path, pbs_project=project, pbs_queue=queue) _LOG.info("Created task description: %s", task_path) if no_qsub: _LOG.info('Skipping submission due to --no-qsub') return 0 # If dry run is not enabled just pass verbose option dry_run_option = '--dry-run' if dry_run else '-v' extra_qsub_args = '-M {0} -m {1}'.format(email_id, email_options) # Append email options and email id to the PbsParameters dict key, extra_qsub_args task_desc.runtime_state.pbs_parameters.extra_qsub_args.extend( extra_qsub_args.split(' ')) submit_subjob(name='generate', task_desc=task_desc, command=[ 'generate', '-vv', '--task-desc', str(task_path), '--tag', tag, '--log-queries', '--email-id', email_id, '--email-options', email_options, dry_run_option, ], qsub_params=dict(name='fc-generate-{}'.format(tag), mem='medium', wd=True, nodes=1, walltime='1h'))