def test_background_optional(self):
"""Background inputs are optional if source monitoring is false."""
self.config_dict["source_monitoring"]["monitor"] = False
del self.config_dict["inputs"]["background"]
config_yaml = yaml.as_document(self.config_dict)
pipeline_config = PipelineConfig(config_yaml)
pipeline_config.validate()
def test_background_for_source_monitoring(self):
"""Background input images must be provided if source monitoring is true."""
self.config_dict["source_monitoring"]["monitor"] = True
del self.config_dict["inputs"]["background"]
config_yaml = yaml.as_document(self.config_dict)
with self.assertRaises(PipelineConfigError):
pipeline_config = PipelineConfig(config_yaml)
pipeline_config.validate()
def test_input_files_exist(self):
# add a fake input file to each input list
for input_type in PipelineConfig._REQUIRED_INPUT_TYPES:
input_file_list = self.config_dict["inputs"][input_type]
input_file_list.append(input_file_list[0].replace("01", "0x"))
config_yaml = yaml.as_document(self.config_dict)
with self.assertRaises(PipelineConfigError):
pipeline_config = PipelineConfig(config_yaml)
pipeline_config.validate()
def test_minimum_two_inputs(self):
for input_type in PipelineConfig._REQUIRED_INPUT_TYPES:
self.config_dict["inputs"][input_type] = [
self.config_dict["inputs"][input_type][0],
]
config_yaml = yaml.as_document(self.config_dict)
with self.assertRaises(PipelineConfigError):
pipeline_config = PipelineConfig(config_yaml)
pipeline_config.validate()
def test_nr_files_differs(self):
for input_type in PipelineConfig._REQUIRED_INPUT_TYPES:
with self.subTest(input_type=input_type):
# add a new unique input file
input_file_list = self.config_dict["inputs"][input_type]
input_file_list.append(input_file_list[0].replace("01", "0x"))
config_yaml = yaml.as_document(self.config_dict)
with self.assertRaises(PipelineConfigError):
pipeline_config = PipelineConfig(config_yaml)
pipeline_config.validate()
def test_duplicated_files(self):
for input_type in PipelineConfig._REQUIRED_INPUT_TYPES:
with self.subTest(input_type=input_type):
# duplicate the first input file
input_file_list = self.config_dict["inputs"][input_type]
input_file_list[1] = input_file_list[0]
config_yaml = yaml.as_document(self.config_dict)
with self.assertRaises(PipelineConfigError):
pipeline_config = PipelineConfig(config_yaml)
pipeline_config.validate()
def test_maximum_input_images(self):
max_files = settings.MAX_PIPERUN_IMAGES
user = AnonymousUser()
n_files_to_add = max_files - len(
self.config_dict["inputs"]["image"]) + 1
for input_type in PipelineConfig._REQUIRED_INPUT_TYPES:
input_file_list = self.config_dict["inputs"][input_type]
input_file_list.extend(
[str(uuid.uuid4()) for _ in range(n_files_to_add)])
config_yaml = yaml.as_document(self.config_dict)
with self.assertRaises(PipelineConfigError):
pipeline_config = PipelineConfig(config_yaml)
pipeline_config.validate(user=user) # type: ignore[arg-type]
def test_input_glob(self):
"""Test simple glob expressions, one for each input"""
config_yaml_original = yaml.as_document(self.config_dict)
pipeline_config_original = PipelineConfig(config_yaml_original)
pipeline_config_original.validate()
# replace the inputs with glob expressions
self.config_dict["inputs"]["image"] = {
"glob": "vast_pipeline/tests/data/epoch??.fits"
}
self.config_dict["inputs"]["selavy"] = {
"glob": "vast_pipeline/tests/data/epoch??.selavy.components.txt"
}
self.config_dict["inputs"]["noise"] = {
"glob": "vast_pipeline/tests/data/epoch??.noiseMap.fits"
}
self.config_dict["inputs"]["background"] = {
"glob": "vast_pipeline/tests/data/epoch??.meanMap.fits"
}
config_yaml_globs = yaml.as_document(self.config_dict)
pipeline_config_globs = PipelineConfig(config_yaml_globs)
pipeline_config_globs.validate()
# after validation, the glob expressions should be resolved and be identical to
# the original config
self.assertDictEqual(pipeline_config_original._yaml.data,
pipeline_config_globs._yaml.data)
def test_association_method_value(self):
# test valid options
for method in PipelineConfig._VALID_ASSOC_METHODS:
with self.subTest(method=method):
self.config_dict["source_association"]["method"] = method
config_yaml = yaml.as_document(self.config_dict)
pipeline_config = PipelineConfig(config_yaml)
pipeline_config.validate()
# test invalid option
method = "foo"
with self.subTest(method=method):
self.config_dict["source_association"]["method"] = method
config_yaml = yaml.as_document(self.config_dict)
with self.assertRaises(PipelineConfigError):
pipeline_config = PipelineConfig(config_yaml)
pipeline_config.validate()
def test_input_multiple_globs(self):
"""Test multiple consecutive glob expressions"""
config_yaml_original = yaml.as_document(self.config_dict)
pipeline_config_original = PipelineConfig(config_yaml_original)
pipeline_config_original.validate()
# replace the inputs with glob expressions
self.config_dict["inputs"]["image"] = {
"glob": [
"vast_pipeline/tests/data/epoch0[12].fits",
"vast_pipeline/tests/data/epoch0[34].fits",
],
}
self.config_dict["inputs"]["selavy"] = {
"glob": [
"vast_pipeline/tests/data/epoch0[12].selavy.components.txt",
"vast_pipeline/tests/data/epoch0[34].selavy.components.txt",
],
}
self.config_dict["inputs"]["noise"] = {
"glob": [
"vast_pipeline/tests/data/epoch0[12].noiseMap.fits",
"vast_pipeline/tests/data/epoch0[34].noiseMap.fits",
],
}
self.config_dict["inputs"]["background"] = {
"glob": [
"vast_pipeline/tests/data/epoch0[12].meanMap.fits",
"vast_pipeline/tests/data/epoch0[34].meanMap.fits",
],
}
config_yaml_globs = yaml.as_document(self.config_dict)
pipeline_config_globs = PipelineConfig(config_yaml_globs)
pipeline_config_globs.validate()
# after validation, the glob expressions should be resolved and be identical to
# the original config
self.assertDictEqual(pipeline_config_original._yaml.data,
pipeline_config_globs._yaml.data)
def handle(self, *args, **options) -> None:
"""
Handle function of the command.
Args:
*args: Variable length argument list.
**options: Variable length options.
Returns:
None
"""
# configure logging
if options['verbosity'] > 1:
# set root logger to use the DEBUG level
root_logger = logging.getLogger('')
root_logger.setLevel(logging.DEBUG)
# set the traceback on
options['traceback'] = True
piperuns = options['piperuns']
for piperun in piperuns:
p_run_name = get_p_run_name(piperun)
try:
p_run = Run.objects.get(name=p_run_name)
except Run.DoesNotExist:
raise CommandError(f'Pipeline run {p_run_name} does not exist')
if p_run.status not in ['END', 'ERR']:
raise CommandError(
f"Run {p_run_name} does not have an 'END' or 'ERR' status."
" Unable to run restore.")
path = p_run.path
pipeline = Pipeline(name=p_run_name,
config_path=os.path.join(path, 'config.yaml'))
try:
# update pipeline run status to restoring
prev_status = p_run.status
pipeline.set_status('RES')
prev_config_file = os.path.join(p_run.path, 'config.yaml.bak')
if os.path.isfile(prev_config_file):
shutil.copy(
prev_config_file,
prev_config_file.replace('.yaml.bak', '.bak.yaml'))
prev_config_file = prev_config_file.replace(
'.yaml.bak', '.bak.yaml')
prev_config = PipelineConfig.from_file(prev_config_file)
os.remove(prev_config_file)
else:
raise CommandError('Previous config file does not exist.'
' Cannot restore pipeline run.')
bak_files = {}
for i in [
'associations', 'bands', 'images', 'measurement_pairs',
'relations', 'skyregions', 'sources', 'config'
]:
if i == 'config':
f_name = os.path.join(p_run.path, f'{i}.yaml.bak')
else:
f_name = os.path.join(p_run.path, f'{i}.parquet.bak')
if os.path.isfile(f_name):
bak_files[i] = f_name
else:
raise CommandError(f'File {f_name} does not exist.'
' Cannot restore pipeline run.')
logger_msg = "Will restore the run to the following config:\n"
logger.info(logger_msg + prev_config._yaml.as_yaml())
user_continue = True if options['no_confirm'] else yesno(
"Would you like to restore the run")
if user_continue:
restore_pipe(p_run, bak_files, prev_config)
pipeline.set_status('END')
logger.info('Restore complete.')
else:
pipeline.set_status(prev_status)
logger.info('No actions performed.')
except Exception as e:
logger.error('Restoring failed!')
logger.error(e)
pipeline.set_status('ERR')
def test_source_finder_value(self):
self.config_dict["measurements"]["source_finder"] = "foo"
config_yaml = yaml.as_document(self.config_dict)
with self.assertRaises(PipelineConfigError):
pipeline_config = PipelineConfig(config_yaml)
pipeline_config.validate()
def test_valid_config(self):
config_yaml = yaml.as_document(self.config_dict)
pipeline_config = PipelineConfig(config_yaml)
pipeline_config.validate()
def test_input_globs_epoch_mode(self):
"""Test glob expressions with user-defined epochs."""
# modify the config to define arbitrary epochs, i.e. "epoch-mode"
self.config_dict["inputs"]["image"] = {
"A": [
"vast_pipeline/tests/data/epoch01.fits",
"vast_pipeline/tests/data/epoch02.fits",
],
"B": [
"vast_pipeline/tests/data/epoch03.fits",
"vast_pipeline/tests/data/epoch04.fits",
],
}
self.config_dict["inputs"]["selavy"] = {
"A": [
"vast_pipeline/tests/data/epoch01.selavy.components.txt",
"vast_pipeline/tests/data/epoch02.selavy.components.txt",
],
"B": [
"vast_pipeline/tests/data/epoch03.selavy.components.txt",
"vast_pipeline/tests/data/epoch04.selavy.components.txt",
],
}
self.config_dict["inputs"]["noise"] = {
"A": [
"vast_pipeline/tests/data/epoch01.noiseMap.fits",
"vast_pipeline/tests/data/epoch02.noiseMap.fits",
],
"B": [
"vast_pipeline/tests/data/epoch03.noiseMap.fits",
"vast_pipeline/tests/data/epoch04.noiseMap.fits",
],
}
self.config_dict["inputs"]["background"] = {
"A": [
"vast_pipeline/tests/data/epoch01.meanMap.fits",
"vast_pipeline/tests/data/epoch02.meanMap.fits",
],
"B": [
"vast_pipeline/tests/data/epoch03.meanMap.fits",
"vast_pipeline/tests/data/epoch04.meanMap.fits",
],
}
config_yaml_original = yaml.as_document(self.config_dict)
pipeline_config_original = PipelineConfig(config_yaml_original)
pipeline_config_original.validate()
# replace the inputs with glob expressions
self.config_dict["inputs"]["image"] = {
"A": {
"glob": "vast_pipeline/tests/data/epoch0[12].fits",
},
"B": {
"glob": "vast_pipeline/tests/data/epoch0[34].fits",
},
}
self.config_dict["inputs"]["selavy"] = {
"A": {
"glob":
"vast_pipeline/tests/data/epoch0[12].selavy.components.txt",
},
"B": {
"glob":
"vast_pipeline/tests/data/epoch0[34].selavy.components.txt",
},
}
self.config_dict["inputs"]["noise"] = {
"A": {
"glob": "vast_pipeline/tests/data/epoch0[12].noiseMap.fits",
},
"B": {
"glob": "vast_pipeline/tests/data/epoch0[34].noiseMap.fits",
},
}
self.config_dict["inputs"]["background"] = {
"A": {
"glob": "vast_pipeline/tests/data/epoch0[12].meanMap.fits",
},
"B": {
"glob": "vast_pipeline/tests/data/epoch0[34].meanMap.fits",
},
}
config_yaml_globs = yaml.as_document(self.config_dict)
pipeline_config_globs = PipelineConfig(config_yaml_globs)
pipeline_config_globs.validate()
# after validation, the glob expressions should be resolved and be identical to
# the original config
self.assertDictEqual(pipeline_config_original._yaml.data,
pipeline_config_globs._yaml.data)