def test_config_station(self):
"""Test something with the yaml config..."""
myfilename = "/tmp/mytestfile"
with patch('aapp_runner.read_aapp_config.load_config_from_file',
return_value=self.config_complete):
cfg_obj = AappRunnerConfig(myfilename, 'norrkoping', 'xl-band')
result = cfg_obj.config
self.assertEqual(result['station'], 'norrkoping')
myconfig = self.config_complete.copy()
with patch('aapp_runner.read_aapp_config.load_config_from_file',
return_value=myconfig):
with pytest.raises(StationError) as exec_info:
cfg_obj = AappRunnerConfig(myfilename, 'dundee', 'xl-band')
exception_raised = exec_info.value
self.assertEqual(
str(exception_raised),
"Station from command line: dundee does not match with configured station: norrkoping"
)
del myconfig['station']
with patch('aapp_runner.read_aapp_config.load_config_from_file',
return_value=myconfig):
cfg_obj = AappRunnerConfig(myfilename, 'dundee', 'xl-band')
result = cfg_obj.config
self.assertEqual(result['station'], 'dundee')
def test_mandatory_variables(self, file_options, static_config):
"""Test that the configuration contains mandatory variables"""
static_config.return_value = True
file_options.return_value = True
myfilename = "/tmp/mytestfile"
myconfig = self.config_mandatory.copy()
with patch('aapp_runner.read_aapp_config.load_config_from_file',
return_value=myconfig):
cfg_obj = AappRunnerConfig(myfilename, 'norrkoping', 'xl-band')
cfg_obj.check_config()
result = cfg_obj.config
expected_dict = {
'aapp_processes': {
'xl-band': {
'description':
'Text describing this processing config',
'name':
'xl-band',
'subscribe_topics':
['/XLBANDANTENNA/HRPT/L0', '/XLBANDANTENNA/METOP/L0'],
'publish_sift_format':
'/{format:s}/{data_processing_level:s}/polar/direct_readout',
'aapp_prefix':
'/disk2/AAPP',
'aapp_environment_file':
'ATOVS_ENV8',
'aapp_outdir_base':
'/disk2/aapp-runner-data',
'aapp_outdir_format':
'{satellite_name:s}_{start_time:%Y%m%d}_{start_time:%H%M}_{orbit_number:05d}',
'aapp_log_files_archive_dir':
'/disk2/aapp-runner-log',
'aapp_log_outdir_format':
'{satellite_name:s}_{start_time:%Y%m%d}_{start_time:%H%M}_{orbit_number:05d}',
'aapp_log_files_archive_length':
1,
'rename_aapp_compose':
'{data_type:s}_{satellite_name:s}_{start_time:%Y%m%d}_{start_time:%H%M}_{orbit_number:5d}.{data_level:s}',
'rename_aapp_files': [{
'avhrr': {
'aapp_file': 'hrpt.l1b',
'data_type': 'hrpt',
'data_level': 'l1b'
}
}],
'aapp_workdir':
'/tmp/myworkdir/'
}
},
'environment': 'xl-band',
'station': 'norrkoping'
}
self.assertDictEqual(expected_dict, result)
def test_check_if_scene_is_unique_return_value(self, config):
"""Test checking if the current scene is unique or if it has been processed earlier."""
config.return_value = self.config_complete
myfilename = "/tmp/mytestfile"
aapp_run_config = AappRunnerConfig(myfilename, 'norrkoping', 'xl-band')
aapp_config = AappL1Config(aapp_run_config.config, 'xl-band')
aapp_config['platform_name'] = 'metop03'
aapp_config['collection_area_id'] = 'euron1'
aapp_config['starttime'] = datetime(2022, 1, 8, 12, 49, 50)
aapp_config['endtime'] = datetime(2022, 1, 8, 13, 0, 26)
aapp_config.job_register = {}
result = check_if_scene_is_unique(aapp_config)
assert result
aapp_config.job_register = {
'metop03': [(datetime(2022, 1, 8, 12, 49,
50), datetime(2022, 1, 8, 13, 0,
26), 'euron1')]
}
# An EARS scene (same platform and overlapping time interval and over
# the same area of interest) arrives shortly after:
aapp_config['platform_name'] = 'metop03'
aapp_config['collection_area_id'] = 'euron1'
aapp_config['starttime'] = datetime(2022, 1, 8, 12, 50)
aapp_config['endtime'] = datetime(2022, 1, 8, 13, 0)
result = check_if_scene_is_unique(aapp_config)
assert not result
def test_update_process_config_sensors_and_filenames_input_uri(
self, config):
"""Test update the process-config for sensors and filenames when input is an uri (one file)."""
config.return_value = self.config_complete
myfilename = "/tmp/mytestfile"
aapp_run_config = AappRunnerConfig(myfilename, 'norrkoping', 'xl-band')
aapp_config = AappL1Config(aapp_run_config.config, 'xl-band')
inmsg = self.input_msg_uri
config = aapp_config
assert config.config.get('process_mhs') is None
result = generate_process_config(inmsg, config)
assert result
assert config.config['process_mhs']
assert config.config['process_amsua']
assert config.config['process_avhrr']
assert config.config['process_amsub']
assert config.config['process_hirs']
assert config.config['process_msu'] is False
assert config.config.get('input_amsub_file') is None
assert config.config.get('input_msu_file') is None
expected_filename = "/san1/polar_in/direct_readout/hrpt/20211213122608_NOAA_18.hmf"
self.assertEqual(config.config['input_avhrr_file'], expected_filename)
self.assertEqual(config.config['input_amsua_file'], expected_filename)
self.assertEqual(config.config['input_mhs_file'], expected_filename)
self.assertEqual(config.config['input_hirs_file'], expected_filename)
def test_update_process_config_sensors_and_filenames_input_dataset(
self, config):
"""Test update the process-config for sensors and filenames when input is a dataset."""
config.return_value = self.config_complete
myfilename = "/tmp/mytestfile"
aapp_run_config = AappRunnerConfig(myfilename, 'norrkoping', 'xl-band')
aapp_config = AappL1Config(aapp_run_config.config, 'xl-band')
inmsg = self.input_msg_dset
config = aapp_config
assert config.config.get('process_mhs') is None
result = generate_process_config(inmsg, config)
assert result
assert config.config['process_mhs']
assert config.config['process_amsua']
assert config.config['process_avhrr']
assert config.config['process_amsub'] is False
assert config.config['process_hirs'] is False
assert config.config['process_msu'] is False
assert config.config.get('input_amsub_file') is None
assert config.config.get('input_hirs_file') is None
assert config.config.get('input_msu_file') is None
expected_filename = "/san1/polar_in/direct_readout/metop/AVHR_HRP_00_M01_20211209085803Z_20211209091327Z_N_O_20211209085817Z"
self.assertEqual(config.config['input_avhrr_file'], expected_filename)
expected_filename = "/san1/polar_in/direct_readout/metop/AMSA_HRP_00_M01_20211209085800Z_20211209091312Z_N_O_20211209085817Z"
self.assertEqual(config.config['input_amsua_file'], expected_filename)
expected_filename = "/san1/polar_in/direct_readout/metop/MHSx_HRP_00_M01_20211209085800Z_20211209091317Z_N_O_20211209085817Z"
self.assertEqual(config.config['input_mhs_file'], expected_filename)
def test_check_if_scene_is_unique_logging(self, config):
"""Test the logging when checking if the current scene is unique."""
config.return_value = self.config_complete
myfilename = "/tmp/mytestfile"
aapp_run_config = AappRunnerConfig(myfilename, 'norrkoping', 'xl-band')
aapp_config = AappL1Config(aapp_run_config.config, 'xl-band')
aapp_config.job_register = {
'metop03': [(datetime(2022, 1, 8, 12, 49,
50), datetime(2022, 1, 8, 13, 0,
26), 'euron1')]
}
# An EARS scene (same platform and overlapping time interval and over
# the same area of interest) arrives shortly after:
aapp_config['platform_name'] = 'metop03'
aapp_config['collection_area_id'] = 'euron1'
aapp_config['starttime'] = datetime(2022, 1, 8, 12, 50)
aapp_config['endtime'] = datetime(2022, 1, 8, 13, 0)
expected_logging = [
'INFO:aapp_runner.helper_functions:first message',
'INFO:aapp_runner.helper_functions:Processing of scene metop03 2022-01-08 12:49:50 2022-01-08 13:00:26 with overlapping time has been launched previously. Skip it!'
]
with self.assertLogs('aapp_runner.helper_functions',
level='INFO') as cm:
logging.getLogger('aapp_runner.helper_functions').info(
'first message')
_ = check_if_scene_is_unique(aapp_config)
self.assertEqual(cm.output, expected_logging)
with self.assertLogs('aapp_runner.helper_functions',
level='WARNING') as cm:
logging.getLogger('aapp_runner.helper_functions').warning(
'first message')
_ = check_if_scene_is_unique(aapp_config)
self.assertEqual(len(cm.output), 1)
# Scene is different (different satellite) from previous:
aapp_config['platform_name'] = 'metop01'
aapp_config['collection_area_id'] = 'euron1'
aapp_config['starttime'] = datetime(2022, 1, 8, 12, 50)
aapp_config['endtime'] = datetime(2022, 1, 8, 13, 0)
with self.assertLogs('aapp_runner.helper_functions',
level='INFO') as cm:
logging.getLogger('aapp_runner.helper_functions').info(
'first message')
result = check_if_scene_is_unique(aapp_config)
assert result
self.assertEqual(len(cm.output), 1)
def test_config_aapp_processes(self, config):
"""Test something with the yaml config..."""
myconfig = self.config_complete.copy()
del myconfig['aapp_processes']
config.return_value = myconfig
myfilename = "/tmp/mytestfile"
with pytest.raises(AappProcessKeyMissing) as exec_info:
cfg_obj = AappRunnerConfig(myfilename, 'norrkoping', 'xl-band')
exception_raised = exec_info.value
self.assertEqual(
str(exception_raised),
"Can not find main section 'aapp_processes' in yaml file. Please check your config."
)
def test_config_env_mismatch(self, file_options, static_config, config):
"""Test something with the yaml config..."""
config.return_value = self.config_complete
static_config.return_value = True
file_options.return_value = True
myfilename = "/tmp/mytestfile"
with pytest.raises(EnvironmentError) as exec_info:
cfg_obj = AappRunnerConfig(myfilename, 'norrkoping', 'unknown')
exception_raised = exec_info.value
self.assertEqual(
str(exception_raised),
"Environment unknown not configured in config. Please check.")
def test_aapp_workdir(self):
"""Test that the aapp-process-env configuration contains an aapp_workdir setting"""
from aapp_runner.read_aapp_config import load_config_from_file
myfilename = "/tmp/mytestfile"
myconfig = self.config_mandatory.copy()
del myconfig['aapp_processes']['xl-band']['aapp_workdir']
with patch('aapp_runner.read_aapp_config.load_config_from_file',
return_value=myconfig):
with pytest.raises(AappWorkDirNotSet) as exec_info:
AappRunnerConfig(myfilename, 'norrkoping', 'xl-band')
exception_raised = exec_info.value
self.assertEqual(
str(exception_raised),
"You must give either 'aapp_workdir' or 'working_dir in config.")
args = read_arguments()
station_name = args.station
environment = args.environment
config_filename = args.config_file
log_file = args.log
log_config = args.log_config
verbose = args.verbose
nameservers = args.nameservers
publish_port = args.publish_port
if not os.path.isfile(config_filename):
print("ERROR! Can not find config file: {}".format(config_filename))
print("Exits!")
sys.exit()
aapp_run_config = AappRunnerConfig(config_filename, station_name,
environment)
aapp_run_config.check_config()
config = aapp_run_config.config
if log_config:
setup_logging_from_config(log_config)
else:
try:
LOG = setup_logging(config, log_file, verbose)
except Exception:
print("Logging setup failed. Check your config")
sys.exit()
try:
aapp_config = AappL1Config(config, environment)
except Exception as err:
def test_read_config(self, file_options, static_config, config):
"""Test loading and initialising the yaml config"""
config.return_value = self.config_complete
static_config.return_value = True
file_options.return_value = True
myfilename = "/tmp/mytestfile"
cfg_obj = AappRunnerConfig(myfilename, 'norrkoping', 'xl-band')
cfg_obj.check_config()
result = cfg_obj.config
log_conf = result['logging']
log_expected = {
'log_rotation_days': 1,
'log_rotation_backup': 30,
'logging_mode': 'DEBUG'
}
self.assertDictEqual(log_expected, log_conf)
aapp_static_conf = result['aapp_static_configuration']
aapp_static_expected = {
'decommutation_files': {
'hirs_file': 'hrsn.l1b',
'amsua_file': 'aman.l1b',
'amsub_file': 'ambn.l1b',
'mhs_file': 'ambn.l1b',
'avhrr_file': 'hrpt.l1b',
'msu_file': 'msun.l1b'
},
'supported_noaa_satellites': ['NOAA-18', 'NOAA-19'],
'supported_metop_satellites': ['Metop-C', 'Metop-B', 'Metop-A'],
'platform_name_aliases': {
'NOAA-19': 'noaa19',
'NOAA-18': 'noaa18',
'Metop-A': 'metop02',
'Metop-B': 'metop01',
'Metop-C': 'metop03',
'METOP-A': 'metop02',
'METOP-B': 'metop01',
'METOP-C': 'metop03',
'METOP A': 'metop02',
'METOP B': 'metop01',
'METOP C': 'metop03',
'M01': 'metop01',
'M02': 'metop02',
'M03': 'metop03'
},
'satellite_sensor_name_aliases': {
'amsua': 'amsu-a',
'amsub': 'amsu-b',
'hirs': 'hirs/4',
'mhs': 'mhs',
'avhrr': 'avhrr/3'
},
'tle_platform_name_aliases': {
'NOAA-19': 'NOAA 19',
'noaa19': 'NOAA 19',
'NOAA-18': 'NOAA 18',
'Metop-A': 'METOP-A',
'Metop-B': 'METOP-B',
'Metop-C': 'METOP-C'
},
'sensor_name_converter': {
'amsua': 'amsu-a',
'amsub': 'amsu-b',
'hirs': 'hirs/4',
'mhs': 'mhs',
'avhrr': 'avhrr/3',
'amsu-a': 'amsua',
'amsu-b': 'amsub',
'hirs/4': 'hirs',
'hirs/3': 'hirs',
'avhrr/3': 'avhrr'
}
}
self.assertDictEqual(aapp_static_expected, aapp_static_conf)
aapp_proc_config = result['aapp_processes']
aapp_proc_expected = {
'xl-band': {
'description':
'Text describing this processing config',
'name':
'xl-band',
'subscribe_topics':
['/XLBANDANTENNA/HRPT/L0', '/XLBANDANTENNA/METOP/L0'],
'tle_indir':
'/disk2/AAPP/orbelems',
'tle_archive_dir':
'{tle_indir:s}/tle-archive/{timestamp:%Y%m}',
'tle_infile_format':
'tle{timestamp:%y%m%d}.txt',
'download_tle_files':
False,
'tle_download': [{
'url':
'http://www.celestrak.com/NORAD/elements/weather.txt'
}, {
'url':
'http://oiswww.eumetsat.org/metopTLEs/html/data_out/latest_m01_tle.txt'
}],
'tle_file_to_data_diff_limit_days':
3,
'locktime_before_rerun':
10,
'collection_area_id':
'euron1',
'publish_sift_format':
'/{format:s}/{data_processing_level:s}/polar/direct_readout',
'keep_orbit_number_from_message':
True,
'aapp_prefix':
'/disk2/AAPP',
'aapp_environment_file':
'ATOVS_ENV8',
'aapp_workdir':
'/run/shm/aapp-workdir',
'working_dir':
'<path to working dir>',
'use_dyn_work_dir':
True,
'aapp_outdir_base':
'/disk2/aapp-runner-data',
'aapp_outdir_format':
'{satellite_name:s}_{start_time:%Y%m%d}_{start_time:%H%M}_{orbit_number:05d}',
'passlength_threshold':
5,
'aapp_log_files_archive_dir':
'/disk2/aapp-runner-log',
'aapp_log_outdir_format':
'{satellite_name:s}_{start_time:%Y%m%d}_{start_time:%H%M}_{orbit_number:05d}',
'aapp_log_files_archive_length':
1,
'message_providing_server':
'satproc2.met.no',
'do_ana_correction':
True,
'do_atovpp':
True,
'do_avh2hirs':
True,
'instrument_skipped_in_processing': [{
'NOAA-15': ['hirs/3', 'amsu-a', 'amsu-b']
}, {
'METOP-C': ['hirs/4']
}],
'rename_aapp_compose':
'{data_type:s}_{satellite_name:s}_{start_time:%Y%m%d}_{start_time:%H%M}_{orbit_number:5d}.{data_level:s}',
'rename_aapp_files': [
{
'avhrr': {
'aapp_file': 'hrpt.l1b',
'data_type': 'hrpt',
'data_level': 'l1b'
}
},
{
'hirs': {
'aapp_file': 'hrsn.l1b',
'data_type': 'hirsl1b',
'data_level': 'l1b'
}
},
{
'hirs': {
'aapp_file': 'hrsn.l1c',
'data_type': 'hirsl1c',
'data_level': 'l1c'
}
},
{
'hirs': {
'aapp_file': 'hirs.l1d',
'data_type': 'hirsl1d',
'data_level': 'l1d'
}
},
{
'amsua': {
'aapp_file': 'aman.l1b',
'data_type': 'amsual1b',
'data_level': 'l1b'
}
},
{
'amsub': {
'aapp_file': 'ambn.l1b',
'data_type': 'amsubl1b',
'data_level': 'l1b'
}
},
{
'mhs': {
'aapp_file': 'ambn.l1b',
'data_type': 'mhsl1b',
'data_level': 'l1b'
}
},
],
'monitor_message': {
'send': True,
'topic': 'some fantastic topic'
}
}
}
self.assertDictEqual(aapp_proc_config, aapp_proc_expected)
self.assertEqual(result['station'], 'norrkoping')
self.assertEqual(result['environment'], 'xl-band')