def test_date_argument(self):
temp_file = NamedTemporaryFile()
ceilo2nc(self.daily_temp_file.name, temp_file.name, self.site_meta, date="2020-10-22")
nc = netCDF4.Dataset(temp_file.name)
assert len(nc.variables["time"]) == 20
assert nc.year == "2020"
assert nc.month == "10"
assert nc.day == "22"
nc.close()
with pytest.raises(ValidTimeStampError):
ceilo2nc(
self.daily_temp_file.name, self.temp_file.name, self.site_meta, date="2020-10-23"
)
def test_date_argument():
daily_file = NamedTemporaryFile()
temp_file = NamedTemporaryFile()
concat_lib.concatenate_files(FILES,
daily_file.name,
concat_dimension="profile")
ceilo2nc(daily_file.name, temp_file.name, SITE_META, date="2021-08-30")
nc = netCDF4.Dataset(temp_file.name)
assert len(nc.variables["time"]) == 12
assert np.all(np.diff(nc.variables["time"][:]) > 0)
assert nc.year == "2021"
assert nc.month == "08"
assert nc.day == "30"
nc.close()
def main():
run_unit_tests()
test_path = utils.get_test_path()
source_path = f"{test_path}/source_data/"
_load_test_data(source_path)
prefix = '20190517_mace-head_'
log_file = f"{source_path}Mace_Head.log"
raw_files = {
'radar': f"{source_path}{prefix}mira_raw.nc",
'lidar': f"{source_path}{prefix}chm15k_raw.nc",
}
""""
We know these fail at the moment:
for name, file in raw_files.items():
check_metadata(file, log_file)
"""
calibrated_files = {
'radar': f"{source_path}radar.nc",
'lidar': f"{source_path}lidar.nc",
}
site_meta = {'name': 'Mace Head', 'altitude': 13}
if PROCESS:
mira2nc(raw_files['radar'], calibrated_files['radar'], site_meta)
ceilo2nc(raw_files['lidar'], calibrated_files['lidar'], site_meta)
for name, file in calibrated_files.items():
check_metadata(file, log_file)
check_data_quality(file, log_file)
input_files = {
'radar': calibrated_files['radar'],
'lidar': calibrated_files['lidar'],
'mwr': f"{source_path}{prefix}hatpro.nc",
'model': f"{source_path}{prefix}ecmwf.nc",
}
categorize_file = f"{source_path}categorize.nc"
if PROCESS:
generate_categorize(input_files, categorize_file)
check_metadata(categorize_file, log_file)
check_data_quality(categorize_file, log_file)
product_file_types = ['iwc', 'lwc', 'drizzle', 'classification']
for file in product_file_types:
product_file = _process_product_file(file, source_path, categorize_file)
check_metadata(product_file, log_file)
check_data_quality(product_file, log_file)
class TestCategorize(Check):
date = "2021-11-20"
site_meta = {
"name": "Munich",
"altitude": 538,
"latitude": 48.5,
"longitude": 11.5
}
radar_file = NamedTemporaryFile()
lidar_file = NamedTemporaryFile()
uuid_radar = mira2nc(f"{filepath}raw_mira_radar.mmclx", radar_file.name,
site_meta)
uuid_lidar = ceilo2nc(f"{filepath}raw_chm15k_lidar.nc", lidar_file.name,
site_meta)
input_files = {
"radar": radar_file.name,
"lidar": lidar_file.name,
"mwr": f"{filepath}hatpro_mwr.nc",
"model": f"{filepath}ecmwf_model.nc",
}
temp_file = NamedTemporaryFile()
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=DeprecationWarning)
uuid = generate_categorize(input_files, temp_file.name)
def test_global_attributes(self):
nc = netCDF4.Dataset(self.temp_file.name)
assert nc.title == "Cloud categorization products from Munich"
nc.close()
def main():
test_path = utils.get_test_path()
source_path = f"{test_path}/source_data/"
_load_test_data(source_path)
log_file = f"{source_path}Mace_Head.log"
raw_files = {
'radar': f"{source_path}raw_mira_radar.mmclx",
'lidar': f"{source_path}raw_chm15k_lidar.nc",
}
""""
We know these fail at the moment:
for name, file in raw_files.items():
check_metadata(file, log_file)
"""
calibrated_files = {
'radar': f"{source_path}radar.nc",
'lidar': f"{source_path}lidar.nc",
}
site_meta = {'name': 'Mace Head', 'altitude': 13.0}
uuid_radar = mira2nc(raw_files['radar'],
calibrated_files['radar'],
site_meta,
uuid='kissa')
assert uuid_radar == 'kissa'
uuid_lidar = ceilo2nc(raw_files['lidar'], calibrated_files['lidar'],
site_meta)
for name, file in calibrated_files.items():
check_metadata(file, log_file)
check_data_quality(file, log_file)
api.check_attributes(calibrated_files['radar'], site_meta)
api.check_is_valid_uuid(uuid_lidar)
input_files = {
'radar': calibrated_files['radar'],
'lidar': calibrated_files['lidar'],
'mwr': f"{source_path}hatpro_mwr.nc",
'model': f"{source_path}ecmwf_model.nc",
}
categorize_file = f"{source_path}categorize.nc"
uuid_categorize = generate_categorize(input_files, categorize_file)
check_metadata(categorize_file, log_file)
check_data_quality(categorize_file, log_file)
api.check_is_valid_uuid(uuid_categorize)
api.check_source_file_uuids(categorize_file, (uuid_lidar, uuid_radar))
product_file_types = ['classification', 'iwc', 'lwc', 'drizzle']
for file in product_file_types:
product_file, uuid_product = _process_product_file(
file, source_path, categorize_file)
check_metadata(product_file, log_file)
check_data_quality(product_file, log_file)
api.check_is_valid_uuid(uuid_product)
api.check_attributes(product_file, site_meta)
api.check_source_file_uuids(product_file, (uuid_categorize, ))
class TestWithRealData(Check):
files = glob.glob(f"{SCRIPT_PATH}/data/chm15k/*.nc")
date = "2020-10-22"
site_meta = SITE_META
daily_temp_file = NamedTemporaryFile()
temp_file = NamedTemporaryFile()
concat_lib.concatenate_files(files, daily_temp_file.name)
uuid = ceilo2nc(daily_temp_file.name, temp_file.name, site_meta)
def test_variable_names(self):
keys = {
"beta",
"beta_raw",
"beta_smooth",
"calibration_factor",
"range",
"height",
"zenith_angle",
"time",
"altitude",
"latitude",
"longitude",
"wavelength",
}
assert set(self.nc.variables.keys()) == keys
def test_common_lidar(self):
lidar_fun = LidarFun(self.nc, self.site_meta, self.date, self.uuid)
for name, method in LidarFun.__dict__.items():
if "test_" in name:
getattr(lidar_fun, name)()
def test_variable_values(self):
assert self.nc.variables["wavelength"][:] == 1064
assert self.nc.variables["zenith_angle"][:] == 0
def test_comments(self):
for key in ("beta", "beta_smooth"):
assert "SNR threshold applied: 5" in self.nc.variables[key].comment
def test_global_attributes(self):
assert self.nc.source == "Lufft CHM15k"
assert self.nc.title == f'CHM15k ceilometer from {self.site_meta["name"]}'
def test_date_argument(self):
temp_file = NamedTemporaryFile()
ceilo2nc(self.daily_temp_file.name, temp_file.name, self.site_meta, date="2020-10-22")
nc = netCDF4.Dataset(temp_file.name)
assert len(nc.variables["time"]) == 20
assert nc.year == "2020"
assert nc.month == "10"
assert nc.day == "22"
nc.close()
with pytest.raises(ValidTimeStampError):
ceilo2nc(
self.daily_temp_file.name, self.temp_file.name, self.site_meta, date="2020-10-23"
)
def main():
test_path = Path(__file__).parent
source_path = f"{test_path}/source_data/"
raw_files = {
"radar": f"{source_path}raw_mira_radar.mmclx",
"lidar": f"{source_path}raw_chm15k_lidar.nc",
}
calibrated_files = {
"radar": f"{source_path}radar.nc",
"lidar": f"{source_path}lidar.nc"
}
site_meta = {
"name": "Munich",
"altitude": 538,
"latitude": 48.148,
"longitude": 11.573
}
uuid_radar = mira2nc(raw_files["radar"],
calibrated_files["radar"],
site_meta,
uuid="kissa")
assert uuid_radar == "kissa"
lidar_meta = site_meta.copy()
uuid_lidar = ceilo2nc(raw_files["lidar"], calibrated_files["lidar"],
lidar_meta)
for _, filename in calibrated_files.items():
_run_tests(filename)
_check_attributes(calibrated_files["radar"], site_meta)
_check_is_valid_uuid(uuid_lidar)
input_files = {
"radar": calibrated_files["radar"],
"lidar": calibrated_files["lidar"],
"mwr": f"{source_path}hatpro_mwr.nc",
"model": f"{source_path}ecmwf_model.nc",
}
uuid_model, uuid_mwr = _get_uuids(input_files)
categorize_file = f"{source_path}categorize.nc"
uuid_categorize = generate_categorize(input_files, categorize_file)
_run_tests(categorize_file)
_check_is_valid_uuid(uuid_categorize)
_check_source_file_uuids(categorize_file,
(uuid_lidar, uuid_radar, uuid_model, uuid_mwr))
product_file_types = ["classification", "iwc", "lwc", "drizzle"]
for file in product_file_types:
product_file, uuid_product = _process_product_file(
file, source_path, categorize_file)
_run_tests(product_file)
_check_is_valid_uuid(uuid_product)
_check_attributes(product_file, site_meta)
_check_source_file_uuids(product_file, (uuid_categorize, ))
class TestCl61d(Check):
site_meta = SITE_META
date = "2021-08-29"
daily_file = NamedTemporaryFile()
concat_lib.concatenate_files(FILES,
daily_file.name,
concat_dimension="profile")
temp_file = NamedTemporaryFile()
uuid = ceilo2nc(daily_file.name, temp_file.name, site_meta, date=date)
def test_variable_names(self):
keys = {
"beta",
"beta_smooth",
"calibration_factor",
"range",
"height",
"zenith_angle",
"time",
"depolarisation",
"altitude",
"latitude",
"longitude",
"wavelength",
}
assert set(self.nc.variables.keys()) == keys
def test_common_lidar(self):
lidar_fun = LidarFun(self.nc, self.site_meta, self.date, self.uuid)
for name, method in LidarFun.__dict__.items():
if "test_" in name:
getattr(lidar_fun, name)()
def test_variable_values(self):
assert abs(self.nc.variables["wavelength"][:] - 910.55) < 0.001
assert self.nc.variables["zenith_angle"][:] == 3.0
assert ma.max(self.nc.variables["depolarisation"][:]) < 1
assert ma.min(self.nc.variables["depolarisation"][:]) > -0.1
def test_comments(self):
assert "SNR threshold applied: 5" in self.nc.variables["beta"].comment
def test_global_attributes(self):
assert self.nc.source == "Vaisala CL61d"
assert self.nc.title == f'CL61d ceilometer from {self.site_meta["name"]}'
def test_date_argument(self):
temp_file = NamedTemporaryFile()
input = f"{SCRIPT_PATH}/data/vaisala/cl31_badtime.DAT"
ceilo2nc(input, temp_file.name, self.site_meta, date="2020-04-10")
nc = netCDF4.Dataset(temp_file.name)
assert len(nc.variables["time"]) == 2
assert nc.year == "2020"
assert nc.month == "04"
assert nc.day == "10"
nc.close()
ceilo2nc(input, temp_file.name, self.site_meta, date="2020-04-11")
nc = netCDF4.Dataset(temp_file.name)
assert len(nc.variables["time"]) == 2
assert nc.year == "2020"
assert nc.month == "04"
assert nc.day == "11"
nc.close()
with pytest.raises(ValidTimeStampError):
ceilo2nc(input, temp_file.name, self.site_meta, date="2020-04-12")
class TestCL51(Check):
site_meta = {
"name": "Kumpula",
"altitude": 123,
"latitude": 45.0,
"longitude": 22.0
}
date = "2020-11-15"
input = f"{SCRIPT_PATH}/data/vaisala/cl51.DAT"
temp_file = NamedTemporaryFile()
uuid = ceilo2nc(input, temp_file.name, site_meta)
def test_variable_names(self):
keys = {
"beta",
"beta_raw",
"beta_smooth",
"calibration_factor",
"range",
"height",
"zenith_angle",
"time",
"altitude",
"latitude",
"longitude",
"wavelength",
}
assert set(self.nc.variables.keys()) == keys
def test_common_lidar(self):
lidar_fun = LidarFun(self.nc, self.site_meta, self.date, self.uuid)
for name, method in LidarFun.__dict__.items():
if "test_" in name:
getattr(lidar_fun, name)()
def test_variable_values(self):
assert self.nc.variables["wavelength"][:] == 910.0
assert self.nc.variables["zenith_angle"][:] == 4.5
assert np.all(np.diff(self.nc.variables["time"][:]) > 0)
def test_comments(self):
for key in ("beta", "beta_smooth"):
assert "SNR threshold applied: 5" in self.nc.variables[key].comment
def test_global_attributes(self):
assert self.nc.source == "Vaisala CL51"
assert self.nc.title == f'CL51 ceilometer from {self.site_meta["name"]}'
def test_date_argument(self):
temp_file = NamedTemporaryFile()
ceilo2nc(self.input, temp_file.name, self.site_meta, date="2020-11-15")
nc = netCDF4.Dataset(temp_file.name)
assert len(nc.variables["time"]) == 2
assert nc.year == "2020"
assert nc.month == "11"
assert nc.day == "15"
nc.close()
with pytest.raises(ValidTimeStampError):
ceilo2nc(self.input,
temp_file.name,
self.site_meta,
date="2021-09-15")
