def main():
test_path = utils.get_test_path()
source_path = f"{test_path}/source_data/"
_load_test_data(source_path)
log_file = f"{source_path}Mace_Head.log"
raw_files = {
'radar': f"{source_path}raw_mira_radar.mmclx",
'lidar': f"{source_path}raw_chm15k_lidar.nc",
}
""""
We know these fail at the moment:
for name, file in raw_files.items():
check_metadata(file, log_file)
"""
calibrated_files = {
'radar': f"{source_path}radar.nc",
'lidar': f"{source_path}lidar.nc",
}
site_meta = {'name': 'Mace Head', 'altitude': 13.0}
uuid_radar = mira2nc(raw_files['radar'],
calibrated_files['radar'],
site_meta,
uuid='kissa')
assert uuid_radar == 'kissa'
uuid_lidar = ceilo2nc(raw_files['lidar'], calibrated_files['lidar'],
site_meta)
for name, file in calibrated_files.items():
check_metadata(file, log_file)
check_data_quality(file, log_file)
api.check_attributes(calibrated_files['radar'], site_meta)
api.check_is_valid_uuid(uuid_lidar)
input_files = {
'radar': calibrated_files['radar'],
'lidar': calibrated_files['lidar'],
'mwr': f"{source_path}hatpro_mwr.nc",
'model': f"{source_path}ecmwf_model.nc",
}
categorize_file = f"{source_path}categorize.nc"
uuid_categorize = generate_categorize(input_files, categorize_file)
check_metadata(categorize_file, log_file)
check_data_quality(categorize_file, log_file)
api.check_is_valid_uuid(uuid_categorize)
api.check_source_file_uuids(categorize_file, (uuid_lidar, uuid_radar))
product_file_types = ['classification', 'iwc', 'lwc', 'drizzle']
for file in product_file_types:
product_file, uuid_product = _process_product_file(
file, source_path, categorize_file)
check_metadata(product_file, log_file)
check_data_quality(product_file, log_file)
api.check_is_valid_uuid(uuid_product)
api.check_attributes(product_file, site_meta)
api.check_source_file_uuids(product_file, (uuid_categorize, ))
def check_metadata(file, log_file=None):
"""Runs metadata checks for the given file.
Args:
file (str): Name of the file to be tested.
log_file (str, optional): Name of the log file where warning
messages are stored.
"""
test_path = utils.get_test_path()
script = f"{test_path}/meta_qc.py"
subprocess.call([script, file, _validate_log_file(log_file)])
def check_data_quality(file, log_file=None):
"""Runs data quality checks for the given file.
Args:
file (str): Name of the file to be tested.
log_file (str, optional): Name of the log file where warning
messages are stored.
"""
test_path = utils.get_test_path()
script = f"{test_path}/data_qc.py"
print(f"Checking data quality of {file}")
subprocess.call([script, file, _validate_log_file(log_file)])
def main():
run_unit_tests()
test_path = utils.get_test_path()
source_path = f"{test_path}/source_data/"
_load_test_data(source_path)
prefix = '20190517_mace-head_'
log_file = f"{source_path}Mace_Head.log"
raw_files = {
'radar': f"{source_path}{prefix}mira_raw.nc",
'lidar': f"{source_path}{prefix}chm15k_raw.nc",
}
""""
We know these fail at the moment:
for name, file in raw_files.items():
check_metadata(file, log_file)
"""
calibrated_files = {
'radar': f"{source_path}radar.nc",
'lidar': f"{source_path}lidar.nc",
}
site_meta = {'name': 'Mace Head', 'altitude': 13}
if PROCESS:
mira2nc(raw_files['radar'], calibrated_files['radar'], site_meta)
ceilo2nc(raw_files['lidar'], calibrated_files['lidar'], site_meta)
for name, file in calibrated_files.items():
check_metadata(file, log_file)
check_data_quality(file, log_file)
input_files = {
'radar': calibrated_files['radar'],
'lidar': calibrated_files['lidar'],
'mwr': f"{source_path}{prefix}hatpro.nc",
'model': f"{source_path}{prefix}ecmwf.nc",
}
categorize_file = f"{source_path}categorize.nc"
if PROCESS:
generate_categorize(input_files, categorize_file)
check_metadata(categorize_file, log_file)
check_data_quality(categorize_file, log_file)
product_file_types = ['iwc', 'lwc', 'drizzle', 'classification']
for file in product_file_types:
product_file = _process_product_file(file, source_path, categorize_file)
check_metadata(product_file, log_file)
check_data_quality(product_file, log_file)
def check_metadata(file, log_file=None):
"""Runs metadata checks for the given file.
Args:
file (str): Name of the file to be tested.
log_file (str, optional): Name of the log file where warning
messages are stored.
"""
test_path = utils.get_test_path()
script = f"{test_path}/meta_qc.py"
print(f"Checking metadata of {file}")
try:
subprocess.check_call([script, file, _validate_log_file(log_file)])
except subprocess.CalledProcessError:
raise