def test_date_argument(self): temp_file = NamedTemporaryFile() ceilo2nc(self.daily_temp_file.name, temp_file.name, self.site_meta, date="2020-10-22") nc = netCDF4.Dataset(temp_file.name) assert len(nc.variables["time"]) == 20 assert nc.year == "2020" assert nc.month == "10" assert nc.day == "22" nc.close() with pytest.raises(ValidTimeStampError): ceilo2nc( self.daily_temp_file.name, self.temp_file.name, self.site_meta, date="2020-10-23" )
def test_date_argument(): daily_file = NamedTemporaryFile() temp_file = NamedTemporaryFile() concat_lib.concatenate_files(FILES, daily_file.name, concat_dimension="profile") ceilo2nc(daily_file.name, temp_file.name, SITE_META, date="2021-08-30") nc = netCDF4.Dataset(temp_file.name) assert len(nc.variables["time"]) == 12 assert np.all(np.diff(nc.variables["time"][:]) > 0) assert nc.year == "2021" assert nc.month == "08" assert nc.day == "30" nc.close()
def main(): run_unit_tests() test_path = utils.get_test_path() source_path = f"{test_path}/source_data/" _load_test_data(source_path) prefix = '20190517_mace-head_' log_file = f"{source_path}Mace_Head.log" raw_files = { 'radar': f"{source_path}{prefix}mira_raw.nc", 'lidar': f"{source_path}{prefix}chm15k_raw.nc", } """" We know these fail at the moment: for name, file in raw_files.items(): check_metadata(file, log_file) """ calibrated_files = { 'radar': f"{source_path}radar.nc", 'lidar': f"{source_path}lidar.nc", } site_meta = {'name': 'Mace Head', 'altitude': 13} if PROCESS: mira2nc(raw_files['radar'], calibrated_files['radar'], site_meta) ceilo2nc(raw_files['lidar'], calibrated_files['lidar'], site_meta) for name, file in calibrated_files.items(): check_metadata(file, log_file) check_data_quality(file, log_file) input_files = { 'radar': calibrated_files['radar'], 'lidar': calibrated_files['lidar'], 'mwr': f"{source_path}{prefix}hatpro.nc", 'model': f"{source_path}{prefix}ecmwf.nc", } categorize_file = f"{source_path}categorize.nc" if PROCESS: generate_categorize(input_files, categorize_file) check_metadata(categorize_file, log_file) check_data_quality(categorize_file, log_file) product_file_types = ['iwc', 'lwc', 'drizzle', 'classification'] for file in product_file_types: product_file = _process_product_file(file, source_path, categorize_file) check_metadata(product_file, log_file) check_data_quality(product_file, log_file)
class TestCategorize(Check): date = "2021-11-20" site_meta = { "name": "Munich", "altitude": 538, "latitude": 48.5, "longitude": 11.5 } radar_file = NamedTemporaryFile() lidar_file = NamedTemporaryFile() uuid_radar = mira2nc(f"{filepath}raw_mira_radar.mmclx", radar_file.name, site_meta) uuid_lidar = ceilo2nc(f"{filepath}raw_chm15k_lidar.nc", lidar_file.name, site_meta) input_files = { "radar": radar_file.name, "lidar": lidar_file.name, "mwr": f"{filepath}hatpro_mwr.nc", "model": f"{filepath}ecmwf_model.nc", } temp_file = NamedTemporaryFile() with warnings.catch_warnings(): warnings.simplefilter("ignore", category=DeprecationWarning) uuid = generate_categorize(input_files, temp_file.name) def test_global_attributes(self): nc = netCDF4.Dataset(self.temp_file.name) assert nc.title == "Cloud categorization products from Munich" nc.close()
def main(): test_path = utils.get_test_path() source_path = f"{test_path}/source_data/" _load_test_data(source_path) log_file = f"{source_path}Mace_Head.log" raw_files = { 'radar': f"{source_path}raw_mira_radar.mmclx", 'lidar': f"{source_path}raw_chm15k_lidar.nc", } """" We know these fail at the moment: for name, file in raw_files.items(): check_metadata(file, log_file) """ calibrated_files = { 'radar': f"{source_path}radar.nc", 'lidar': f"{source_path}lidar.nc", } site_meta = {'name': 'Mace Head', 'altitude': 13.0} uuid_radar = mira2nc(raw_files['radar'], calibrated_files['radar'], site_meta, uuid='kissa') assert uuid_radar == 'kissa' uuid_lidar = ceilo2nc(raw_files['lidar'], calibrated_files['lidar'], site_meta) for name, file in calibrated_files.items(): check_metadata(file, log_file) check_data_quality(file, log_file) api.check_attributes(calibrated_files['radar'], site_meta) api.check_is_valid_uuid(uuid_lidar) input_files = { 'radar': calibrated_files['radar'], 'lidar': calibrated_files['lidar'], 'mwr': f"{source_path}hatpro_mwr.nc", 'model': f"{source_path}ecmwf_model.nc", } categorize_file = f"{source_path}categorize.nc" uuid_categorize = generate_categorize(input_files, categorize_file) check_metadata(categorize_file, log_file) check_data_quality(categorize_file, log_file) api.check_is_valid_uuid(uuid_categorize) api.check_source_file_uuids(categorize_file, (uuid_lidar, uuid_radar)) product_file_types = ['classification', 'iwc', 'lwc', 'drizzle'] for file in product_file_types: product_file, uuid_product = _process_product_file( file, source_path, categorize_file) check_metadata(product_file, log_file) check_data_quality(product_file, log_file) api.check_is_valid_uuid(uuid_product) api.check_attributes(product_file, site_meta) api.check_source_file_uuids(product_file, (uuid_categorize, ))
class TestWithRealData(Check): files = glob.glob(f"{SCRIPT_PATH}/data/chm15k/*.nc") date = "2020-10-22" site_meta = SITE_META daily_temp_file = NamedTemporaryFile() temp_file = NamedTemporaryFile() concat_lib.concatenate_files(files, daily_temp_file.name) uuid = ceilo2nc(daily_temp_file.name, temp_file.name, site_meta) def test_variable_names(self): keys = { "beta", "beta_raw", "beta_smooth", "calibration_factor", "range", "height", "zenith_angle", "time", "altitude", "latitude", "longitude", "wavelength", } assert set(self.nc.variables.keys()) == keys def test_common_lidar(self): lidar_fun = LidarFun(self.nc, self.site_meta, self.date, self.uuid) for name, method in LidarFun.__dict__.items(): if "test_" in name: getattr(lidar_fun, name)() def test_variable_values(self): assert self.nc.variables["wavelength"][:] == 1064 assert self.nc.variables["zenith_angle"][:] == 0 def test_comments(self): for key in ("beta", "beta_smooth"): assert "SNR threshold applied: 5" in self.nc.variables[key].comment def test_global_attributes(self): assert self.nc.source == "Lufft CHM15k" assert self.nc.title == f'CHM15k ceilometer from {self.site_meta["name"]}' def test_date_argument(self): temp_file = NamedTemporaryFile() ceilo2nc(self.daily_temp_file.name, temp_file.name, self.site_meta, date="2020-10-22") nc = netCDF4.Dataset(temp_file.name) assert len(nc.variables["time"]) == 20 assert nc.year == "2020" assert nc.month == "10" assert nc.day == "22" nc.close() with pytest.raises(ValidTimeStampError): ceilo2nc( self.daily_temp_file.name, self.temp_file.name, self.site_meta, date="2020-10-23" )
def main(): test_path = Path(__file__).parent source_path = f"{test_path}/source_data/" raw_files = { "radar": f"{source_path}raw_mira_radar.mmclx", "lidar": f"{source_path}raw_chm15k_lidar.nc", } calibrated_files = { "radar": f"{source_path}radar.nc", "lidar": f"{source_path}lidar.nc" } site_meta = { "name": "Munich", "altitude": 538, "latitude": 48.148, "longitude": 11.573 } uuid_radar = mira2nc(raw_files["radar"], calibrated_files["radar"], site_meta, uuid="kissa") assert uuid_radar == "kissa" lidar_meta = site_meta.copy() uuid_lidar = ceilo2nc(raw_files["lidar"], calibrated_files["lidar"], lidar_meta) for _, filename in calibrated_files.items(): _run_tests(filename) _check_attributes(calibrated_files["radar"], site_meta) _check_is_valid_uuid(uuid_lidar) input_files = { "radar": calibrated_files["radar"], "lidar": calibrated_files["lidar"], "mwr": f"{source_path}hatpro_mwr.nc", "model": f"{source_path}ecmwf_model.nc", } uuid_model, uuid_mwr = _get_uuids(input_files) categorize_file = f"{source_path}categorize.nc" uuid_categorize = generate_categorize(input_files, categorize_file) _run_tests(categorize_file) _check_is_valid_uuid(uuid_categorize) _check_source_file_uuids(categorize_file, (uuid_lidar, uuid_radar, uuid_model, uuid_mwr)) product_file_types = ["classification", "iwc", "lwc", "drizzle"] for file in product_file_types: product_file, uuid_product = _process_product_file( file, source_path, categorize_file) _run_tests(product_file) _check_is_valid_uuid(uuid_product) _check_attributes(product_file, site_meta) _check_source_file_uuids(product_file, (uuid_categorize, ))
class TestCl61d(Check): site_meta = SITE_META date = "2021-08-29" daily_file = NamedTemporaryFile() concat_lib.concatenate_files(FILES, daily_file.name, concat_dimension="profile") temp_file = NamedTemporaryFile() uuid = ceilo2nc(daily_file.name, temp_file.name, site_meta, date=date) def test_variable_names(self): keys = { "beta", "beta_smooth", "calibration_factor", "range", "height", "zenith_angle", "time", "depolarisation", "altitude", "latitude", "longitude", "wavelength", } assert set(self.nc.variables.keys()) == keys def test_common_lidar(self): lidar_fun = LidarFun(self.nc, self.site_meta, self.date, self.uuid) for name, method in LidarFun.__dict__.items(): if "test_" in name: getattr(lidar_fun, name)() def test_variable_values(self): assert abs(self.nc.variables["wavelength"][:] - 910.55) < 0.001 assert self.nc.variables["zenith_angle"][:] == 3.0 assert ma.max(self.nc.variables["depolarisation"][:]) < 1 assert ma.min(self.nc.variables["depolarisation"][:]) > -0.1 def test_comments(self): assert "SNR threshold applied: 5" in self.nc.variables["beta"].comment def test_global_attributes(self): assert self.nc.source == "Vaisala CL61d" assert self.nc.title == f'CL61d ceilometer from {self.site_meta["name"]}'
def test_date_argument(self): temp_file = NamedTemporaryFile() input = f"{SCRIPT_PATH}/data/vaisala/cl31_badtime.DAT" ceilo2nc(input, temp_file.name, self.site_meta, date="2020-04-10") nc = netCDF4.Dataset(temp_file.name) assert len(nc.variables["time"]) == 2 assert nc.year == "2020" assert nc.month == "04" assert nc.day == "10" nc.close() ceilo2nc(input, temp_file.name, self.site_meta, date="2020-04-11") nc = netCDF4.Dataset(temp_file.name) assert len(nc.variables["time"]) == 2 assert nc.year == "2020" assert nc.month == "04" assert nc.day == "11" nc.close() with pytest.raises(ValidTimeStampError): ceilo2nc(input, temp_file.name, self.site_meta, date="2020-04-12")
class TestCL51(Check): site_meta = { "name": "Kumpula", "altitude": 123, "latitude": 45.0, "longitude": 22.0 } date = "2020-11-15" input = f"{SCRIPT_PATH}/data/vaisala/cl51.DAT" temp_file = NamedTemporaryFile() uuid = ceilo2nc(input, temp_file.name, site_meta) def test_variable_names(self): keys = { "beta", "beta_raw", "beta_smooth", "calibration_factor", "range", "height", "zenith_angle", "time", "altitude", "latitude", "longitude", "wavelength", } assert set(self.nc.variables.keys()) == keys def test_common_lidar(self): lidar_fun = LidarFun(self.nc, self.site_meta, self.date, self.uuid) for name, method in LidarFun.__dict__.items(): if "test_" in name: getattr(lidar_fun, name)() def test_variable_values(self): assert self.nc.variables["wavelength"][:] == 910.0 assert self.nc.variables["zenith_angle"][:] == 4.5 assert np.all(np.diff(self.nc.variables["time"][:]) > 0) def test_comments(self): for key in ("beta", "beta_smooth"): assert "SNR threshold applied: 5" in self.nc.variables[key].comment def test_global_attributes(self): assert self.nc.source == "Vaisala CL51" assert self.nc.title == f'CL51 ceilometer from {self.site_meta["name"]}' def test_date_argument(self): temp_file = NamedTemporaryFile() ceilo2nc(self.input, temp_file.name, self.site_meta, date="2020-11-15") nc = netCDF4.Dataset(temp_file.name) assert len(nc.variables["time"]) == 2 assert nc.year == "2020" assert nc.month == "11" assert nc.day == "15" nc.close() with pytest.raises(ValidTimeStampError): ceilo2nc(self.input, temp_file.name, self.site_meta, date="2021-09-15")