def add_source_attribute(attributes: dict, data: dict): """Adds source attribute.""" variables = { "radar": ( "v", "width", "v_sigma", "ldr", "Z", "zdr", "sldr", "radar_frequency", "nyquist_velocity", "rain_rate", ), "lidar": ("beta", "lidar_wavelength"), "mwr": ("lwp", ), "model": ("uwind", "vwind", "Tw", "q", "pressure", "temperature"), } for instrument, keys in variables.items(): source = data[instrument].dataset.source for key in keys: if key in attributes: attributes[key] = attributes[key]._replace(source=source) else: attributes[key] = MetaData(source=source) return attributes
def test_set_attributes(self): obj = CloudnetArray(self.time, 'test_name') meta = MetaData(long_name='the long name', units='g m-3') obj.set_attributes(meta) for key, value in zip(['long_name', 'units'], ['the long name', 'g m-3']): assert hasattr(obj, key) assert getattr(obj, key) == value
def test_add_time_attribute(): attr = MetaData(long_name='Some name', units='xy') attributes = {'kissa': attr} date = ['2020', '01', '12'] new_attributes = output.add_time_attribute(attributes, date) assert new_attributes['time'].units == 'hours since 2020-01-12 00:00:00' assert new_attributes['kissa'].units == 'xy'
def test_set_attributes(self): obj = CloudnetArray(self.time, "test_name") meta = MetaData(long_name="the long name", units="g m-3") obj.set_attributes(meta) for key, value in zip(["long_name", "units"], ["the long name", "g m-3"]): assert hasattr(obj, key) assert getattr(obj, key) == value
def test_add_time_attribute(): attr = MetaData(long_name="Some name", units="xy") attributes = {"kissa": attr} date = ["2020", "01", "12"] new_attributes = output.add_time_attribute(attributes, date) assert new_attributes[ "time"].units == "hours since 2020-01-12 00:00:00 +00:00" assert new_attributes["kissa"].units == "xy"
def test_set_attributes(fake_nc_file): nc = netCDF4.Dataset(fake_nc_file) test_var = nc.variables['time'] obj = CloudnetArray(test_var, 'test_name') meta = MetaData(long_name='the long name', units='g m-3') obj.set_attributes(meta) for key, value in zip(['long_name', 'units'], ['the long name', 'g m-3']): assert hasattr(obj, key) assert getattr(obj, key) == value
def add_time_attribute(attributes: dict, date: list, key: str = "time") -> dict: """Adds time attribute with correct units.""" date_str = "-".join(date) units = f"hours since {date_str} 00:00:00 +00:00" if key not in attributes: attributes[key] = MetaData(units=units) else: attributes[key] = attributes[key]._replace(units=units) return attributes
def add_time_attribute(attributes: dict, date: list) -> dict: """"Adds time attribute with correct units. Args: attributes: Attributes of variables. date: Date as ['YYYY', 'MM', 'DD']. Returns: dict: Same attributes with 'time' attribute added. """ date = '-'.join(date) attributes['time'] = MetaData(units=f'hours since {date} 00:00:00') return attributes
cloudnet_array.rebin_data(self.time, time_grid) snr_gain = self._estimate_snr_gain(time_grid, self.time) self.time = time_grid return snr_gain @staticmethod def _estimate_snr_gain(time_sparse: np.ndarray, time_dense: np.ndarray) -> float: """Returns factor for SNR (dB) increase when data is binned.""" binning_ratio = utils.mdiff(time_sparse) / utils.mdiff(time_dense) return np.sqrt(binning_ratio) def _init_mira_date(self) -> List[str]: time_stamps = self.getvar('time') return utils.seconds2date(time_stamps[0], self.epoch)[:3] ATTRIBUTES = { 'Ze': MetaData( long_name= 'Radar reflectivity factor (uncorrected), vertical polarization', units='dBZ', ), 'SNR': MetaData( long_name='Signal-to-noise ratio', units='dB', ) }
'Value 3: Drizzle or rain coexisting with cloud liquid droplets.\n' 'Value 4: Ice particles.\n' 'Value 5: Ice coexisting with supercooled liquid droplets.\n' 'Value 6: Melting ice particles.\n' 'Value 7: Melting ice particles coexisting with cloud liquid droplets.\n' 'Value 8: Aerosol particles, no cloud or precipitation.\n' 'Value 9: Insects, no cloud or precipitation.\n' 'Value 10: Aerosol coexisting with insects, no cloud or precipitation.'), 'detection_status': ('\n' 'Value 0: Clear sky.\n' 'Value 1: Good radar and lidar echos.\n' 'Value 2: Good radar echo only.\n' 'Value 3: Radar echo, corrected for liquid attenuation.\n' 'Value 4: Lidar echo only.\n' 'Value 5: Radar echo, uncorrected for liquid attenuation.\n' 'Value 6: Radar ground clutter.\n' 'Value 7: Lidar clear-air molecular scattering.'), } CLASSIFICATION_ATTRIBUTES = { 'target_classification': MetaData(long_name='Target classification', comment=COMMENTS['target_classification'], definition=DEFINITIONS['target_classification']), 'detection_status': MetaData(long_name='Radar and lidar detection status', comment=COMMENTS['detection_status'], definition=DEFINITIONS['detection_status']) }
line = linecache.getline(full_path, first_empty_line + n) if not utils.is_empty_line(line): line = linecache.getline(full_path, first_empty_line + n + 1) break if "CL" in line: return "cl31_or_cl51" if "CT" in line: return "ct25k" raise RuntimeError("Error: Unknown ceilo model.") ATTRIBUTES = { "depolarisation": MetaData( long_name="Lidar volume linear depolarisation ratio", units="1", comment= "SNR-screened lidar volume linear depolarisation ratio at 910.55 nm.", ), "scale": MetaData(long_name="Scale", units="%", comment="100 (%) is normal."), "software_level": MetaData( long_name="Software level ID", units="1", ), "laser_temperature": MetaData( long_name="Laser temperature", units="C", ), "window_transmission":
DEFINITIONS = { 'retrieval_method': ('\n' 'Value 0: Linear Regression\n' 'Value 1: Quadratic Regression\n' 'Value 2: Neural Network'), 'quality_flag': ('\n' 'Bit 0: Rain information (0=no rain, 1=raining)\n' 'Bit 1/2: Quality level (0=Not evaluated, 1=high, 2=medium, 3=low)\n' 'Bit 3/4: Reason for reduced quality'), } ATTRIBUTES = { 'time': MetaData( units='seconds since 2001-01-01 00:00:00', long_name='sample time', ), 'file_code': MetaData( long_name='File code', comment='RPG HATPRO software version.', ), 'program_number': MetaData(long_name='Program number', ), 'retrieval_method': MetaData(long_name='Retrieval method', definition=DEFINITIONS['retrieval_method']), 'LWP': MetaData(long_name='Liquid water path', units='g m-2'), 'zenith': MetaData(
output.add_file_type(rootgrp, 'lidar') if hasattr(ceilo, 'dataset'): output.copy_variables(ceilo.dataset, rootgrp, ('wavelength', )) rootgrp.title = f"Ceilometer file from {location}" rootgrp.year, rootgrp.month, rootgrp.day = ceilo.date rootgrp.location = location rootgrp.history = f"{utils.get_time()} - ceilometer file created" rootgrp.source = ceilo.model rootgrp.close() return uuid ATTRIBUTES = { 'beta': MetaData(long_name='Attenuated backscatter coefficient', units='sr-1 m-1', comment='Range corrected, SNR screened, attenuated backscatter.'), 'beta_raw': MetaData(long_name='Raw attenuated backscatter coefficient', units='sr-1 m-1', comment="Range corrected, attenuated backscatter."), 'beta_smooth': MetaData( long_name='Smoothed attenuated backscatter coefficient', units='sr-1 m-1', comment=('Range corrected, SNR screened backscatter coefficient.\n' 'Weak background is smoothed using Gaussian 2D-kernel.')), 'scale': MetaData(long_name='Scale', units='%', comment='100 (%) is normal.'), 'software_level': MetaData(
self.date = self.get_date() self.source = 'BASTA' def screen_data(self, keymap: dict) -> None: """Saves only valid pixels.""" mask = self.getvar('background_mask') for key in keymap.values(): self.data[key].mask_indices(np.where(mask != 1)) def validate_date(self, expected_date: str) -> None: """Validates expected data.""" date_units = self.dataset.variables['time'].units date = date_units.split()[2] if expected_date != date: raise ValueError('Basta date not what expected.') def add_geolocation(self): """Adds geo info.""" for key in ('latitude', 'longitude', 'altitude'): if key not in self.data.keys(): # Not provided by user value = ma.median(self.getvar(key)) self.append_data(value.astype(float), key) ATTRIBUTES = { 'Ze': MetaData( long_name='Radar reflectivity factor', units='dBZ', ), }
" ice or liquid cloud has not been corrected.\n" "Value 3: Good radar and lidar echos.\n" "Value 4: No radar echo but rain or liquid cloud beneath mean that attenuation that would\n" " be experienced is unknown.\n" "Value 5: Good radar echo only.\n" "Value 6: No radar echo but known attenuation.\n" "Value 7: Radar echo corrected for liquid attenuation using microwave radiometer data.\n" "Value 8: Radar ground clutter.\n" "Value 9: Lidar clear-air molecular scattering." ), } CLASSIFICATION_ATTRIBUTES = { "target_classification": MetaData( long_name="Target classification", comment=COMMENTS["target_classification"], definition=DEFINITIONS["target_classification"], units="1", ), "detection_status": MetaData( long_name="Radar and lidar detection status", comment=COMMENTS["detection_status"], definition=DEFINITIONS["detection_status"], units="1", ), "cloud_top_height_amsl": MetaData( long_name="Height of cloud top above mean sea level", units="m", ), "cloud_base_height_amsl": MetaData( long_name="Height of cloud base above mean sea level", units="m",
'Value 0: Square\n' 'Value 1: Parzen\n' 'Value 2: Blackman\n' 'Value 3: Welch\n' 'Value 4: Slepian2\n' 'Value 5: Slepian3'), 'quality_flag': ('\n' 'Bit 0: ADC saturation.\n' 'Bit 1: Spectral width too high.\n' 'Bit 2: No transmission power levelling.') } RPG_ATTRIBUTES = { 'file_code': MetaData( long_name='File code', comment='Indicates the RPG software version.', ), 'program_number': MetaData(long_name='Program number', ), 'model_number': MetaData(long_name='Model number', definition=DEFINITIONS['model_number']), 'antenna_separation': MetaData( long_name='Antenna separation', units='m', ), 'antenna_diameter': MetaData( long_name='Antenna diameter', units='m', ),
if not utils.isscalar(results[key]): results[key][classification.is_rain, :] = 0 return results def _append_data(drizzle_data, results): """Save retrieved fields to the drizzle_data object.""" for key, value in results.items(): value = ma.masked_where(value == 0, value) drizzle_data.append_data(value, key) DRIZZLE_ATTRIBUTES = { 'drizzle_N': MetaData(long_name='Drizzle number concentration', units='m-3', ancillary_variables='drizzle_N_error'), 'drizzle_N_error': MetaData(long_name='Random error in drizzle number concentration', units='dB'), 'drizzle_lwc': MetaData(long_name='Drizzle liquid water content', units='kg m-3', ancillary_variables='drizzle_lwc_error drizzle_lwc_bias'), 'drizzle_lwc_error': MetaData( long_name='Random error in drizzle liquid water content', units='dB', ), 'drizzle_lwc_bias': MetaData(
"Value 0: Square\n" "Value 1: Parzen\n" "Value 2: Blackman\n" "Value 3: Welch\n" "Value 4: Slepian2\n" "Value 5: Slepian3"), "quality_flag": ("\n" "Bit 0: ADC saturation.\n" "Bit 1: Spectral width too high.\n" "Bit 2: No transmission power levelling."), } RPG_ATTRIBUTES = { # LDR-mode radars: "ldr": MetaData(long_name="Linear depolarisation ratio", units="dB"), "rho_cx": MetaData(long_name="Co-cross-channel correlation coefficient", units="1"), "phi_cx": MetaData(long_name="Co-cross-channel differential phase", units="rad"), # STSR-mode radars "zdr": MetaData(long_name="Differential reflectivity", units="dB"), "rho_hv": MetaData(long_name="Correlation coefficient", units="1"), "phi_dp": MetaData(long_name="Differential phase", units="rad"), "sldr": MetaData(long_name="Slanted linear depolarisation ratio", units="dB"), "srho_hv": MetaData(long_name="Slanted correlation coefficient", units="1"),
f"Filtering {n_removed} profiles due to varying zenith / azimuth angle" ) self.append_data(zenith, "zenith_angle") for key in ("elevation", "azimuth_velocity"): del self.data[key] return list(is_stable_profile) def _init_mira_date(self) -> List[str]: time_stamps = self.getvar("time") return utils.seconds2date(time_stamps[0], self.epoch)[:3] ATTRIBUTES = { "SNR": MetaData( long_name="Signal-to-noise ratio", units="dB", ), "nfft": MetaData( long_name="Number of FFT points", units="1", ), "nave": MetaData( long_name= "Number of spectral averages (not accounting for overlapping FFTs)", units="1", ), "rg0": MetaData(long_name="Number of lowest range gates", units="1"), "prf":
"Value 3: Adiabatic retrieval: new cloud pixels where cloud top has been\n" " adjusted to match liquid water path from microwave radiometer because\n" " layer is not detected by radar.\n" "Value 4: No retrieval: either no liquid water path is available or liquid water\n" " path is uncertain.\n" "Value 5: No retrieval: liquid water layer detected only by the lidar and liquid\n" " water path is unavailable or uncertain: cloud top may be higher than\n" " diagnosed cloud top since lidar signal has been attenuated.\n" "Value 6: Rain present: cloud extent is difficult to ascertain and liquid water\n" " path also uncertain.") } LWC_ATTRIBUTES = { "lwc": MetaData(long_name="Liquid water content", comment=COMMENTS["lwc"], ancillary_variables="lwc_error"), "lwc_error": MetaData( long_name= "Random error in liquid water content, one standard deviation", comment=COMMENTS["lwc_error"], units="dB", ), "lwc_retrieval_status": MetaData( long_name="Liquid water content retrieval status", comment=COMMENTS["lwc_retrieval_status"], definition=DEFINITIONS["lwc_retrieval_status"], units="1", ),
} DEFINITIONS = { 'lwc_retrieval_status': ('\n' 'Value 0: No liquid water detected.\n' 'Value 1: Reliable retrieval.\n' 'Value 2: Cloud pixel whose top has been adjusted so that the theoretical\n' ' liquid water path would match observation.\n' 'Value 3: New cloud pixel introduced so that the theoretical liquid\n' ' water path would match observation.\n' 'Value 4: Rain present: cloud extent is difficult to ascertain and liquid\n' ' water path also uncertain.'), } LWC_ATTRIBUTES = { 'lwc': MetaData(long_name='Liquid water content', comment=COMMENTS['lwc'], ancillary_variables='lwc_error'), 'lwc_error': MetaData(long_name= 'Random error in liquid water content, one standard deviation', comment=COMMENTS['lwc_error'], units='dB'), 'lwc_retrieval_status': MetaData(long_name='Liquid water content retrieval status', comment=COMMENTS['lwc_retrieval_status'], definition=DEFINITIONS['lwc_retrieval_status']), }
"Value 3: Retrieval performed but radar corrected for liquid attenuation using\n" " radiometer liquid water path which is not always accurate.\n" "Value 4: Ice detected only by the lidar.\n" "Value 5: Ice detected by radar but rain below so no retrieval performed\n" " due to very uncertain attenuation.\n" "Value 6: Clear sky above rain and wet-bulb temperature less than 0degC:\n" " if rain attenuation is strong, ice could be present but undetected.\n" "Value 7: Drizzle or rain that would have been classified as ice if the wet-bulb\n" " temperature were less than 0degC: may be ice if temperature is in error" ) } IWC_ATTRIBUTES = { "iwc": MetaData( long_name="Ice water content", units="kg m-3", ancillary_variables="iwc_error iwc_sensitivity iwc_bias", ), "iwc_error": MetaData( long_name="Random error in ice water content", units="dB", ), "iwc_bias": MetaData( long_name="Possible bias in ice water content", units="dB", comment=COMMENTS["iwc_bias"] ), "iwc_sensitivity": MetaData( long_name="Minimum detectable ice water content", units="kg m-3", comment=COMMENTS["iwc_sensitivity"], ), "iwc_retrieval_status": MetaData(
return results def _append_data(drizzle_data: DrizzleSource, results: dict): """Save retrieved fields to the drizzle_data object.""" for key, value in results.items(): if key != "drizzle_retrieval_status": value = ma.masked_where(value == 0, value) drizzle_data.append_data(value, key) DRIZZLE_ATTRIBUTES = { "drizzle_N": MetaData( long_name="Drizzle number concentration", units="m-3", ancillary_variables="drizzle_N_error drizzle_N_bias", ), "drizzle_N_error": MetaData(long_name="Random error in drizzle number concentration", units="dB"), "drizzle_N_bias": MetaData( long_name="Possible bias in drizzle number concentration", units="dB", ), "drizzle_lwc": MetaData( long_name="Drizzle liquid water content", units="kg m-3", ancillary_variables="drizzle_lwc_error drizzle_lwc_bias",
DEFINITIONS = { "retrieval_method": ("\n" "Value 0: Linear Regression\n" "Value 1: Quadratic Regression\n" "Value 2: Neural Network"), "quality_flag": ("\n" "Bit 0: Rain information (0=no rain, 1=raining)\n" "Bit 1/2: Quality level (0=Not evaluated, 1=high, 2=medium, 3=low)\n" "Bit 3/4: Reason for reduced quality"), } ATTRIBUTES = { "file_code": MetaData(long_name="File code", comment="RPG HATPRO software version.", units="1"), "program_number": MetaData(long_name="Program number", ), "retrieval_method": MetaData(long_name="Retrieval method", definition=DEFINITIONS["retrieval_method"], units="1"), "quality_flag": MetaData( long_name="Quality flag", definition=DEFINITIONS["quality_flag"], units="1", comment= "Quality information as an 8 bit array. See RPG HATPRO manual for more\n" "information.",
' data which can be inaccurate.' 'Value 3: Unreliable retrieval: Uncorrected liquid attenuation due to\n' ' missing liquid water path data.' 'Value 4: No retrieval: Ice detected only by the lidar.\n' 'Value 5: No retrieval: Rain below the detected ice leads to large\n' ' uncertainties.\n' 'Value 6: Clear sky above rain and wet-bulb temperature less than 0degC: ' ' if rain attenuation is strong, ice could be present but undetected.\n' 'Value 7: Drizzle or rain that would have been classified as ice if the\n' ' wet-bulb temperature were less than 0degC.') } IWC_ATTRIBUTES = { 'iwc': MetaData( long_name='Ice water content', units='kg m-3', comment=COMMENTS['iwc'], ancillary_variables='iwc_sensitivity iwc_bias' ), 'iwc_error': MetaData( long_name='Random error in ice water content, one standard deviation', units='dB', comment=COMMENTS['iwc_error'] ), 'iwc_bias': MetaData( long_name='Possible bias in ice water content, one standard deviation', units='dB', comment=COMMENTS['iwc_bias'] ), 'iwc_sensitivity': MetaData( long_name='Minimum detectable ice water content', units='kg m-3',
array1 = nc_file1.variables[key][:] array2 = nc_file2.variables[key][:] assert_array_equal(array1, array2) return array1 def _close(*args) -> None: for arg in args: arg.close() ATTRIBUTES = { "depolarisation": MetaData( long_name="Lidar volume linear depolarisation ratio", units="1", comment= "SNR-screened lidar volume linear depolarisation ratio at 532 nm.", ), "depolarisation_raw": MetaData( long_name="Lidar volume linear depolarisation ratio", units="1", comment= "Non-screened lidar volume linear depolarisation ratio at 532 nm.", ), "calibration_factor": MetaData( long_name="Attenuated backscatter calibration factor", units="1", comment="Calibration factor applied.", ),
'detection_status': ('\n' 'Value 0: Clear sky.\n' 'Value 1: Good radar and lidar echos.\n' 'Value 2: Good radar echo only.\n' 'Value 3: Radar echo, corrected for liquid attenuation.\n' 'Value 4: Lidar echo only.\n' 'Value 5: Radar echo, uncorrected for liquid attenuation.\n' 'Value 6: Radar ground clutter.\n' 'Value 7: Lidar clear-air molecular scattering.'), } CLASSIFICATION_ATTRIBUTES = { 'target_classification': MetaData(long_name='Target classification', comment=COMMENTS['target_classification'], definition=DEFINITIONS['target_classification']), 'detection_status': MetaData(long_name='Radar and lidar detection status', comment=COMMENTS['detection_status'], definition=DEFINITIONS['detection_status']), 'cloud_top_height_amsl': MetaData( long_name='Height of cloud top above mean sea level', units='m', ), 'cloud_base_height_amsl': MetaData( long_name='Height of cloud base above mean sea level', units='m', ),
'Bit 3: The lidar echo is due to clear-air molecular scattering.\n' 'Bit 4: Liquid water cloud, rainfall or melting ice below this pixel\n' ' will have caused radar and lidar attenuation; if bit 5 is set then\n' ' a correction for the radar attenuation has been performed;\n' ' otherwise do not trust the absolute values of reflectivity factor.\n' ' No correction is performed for lidar attenuation.\n' 'Bit 5: Radar reflectivity has been corrected for liquid-water attenuation\n' ' using the microwave radiometer measurements of liquid water path\n' ' and the lidar estimation of the location of liquid water cloud;\n' ' be aware that errors in reflectivity may result.'), } CATEGORIZE_ATTRIBUTES = { 'Z': MetaData(long_name='Radar reflectivity factor', units='dBZ', comment=COMMENTS['Z'], ancillary_variables='Z_error Z_bias Z_sensitivity'), 'Z_error': MetaData(long_name='Error in radar reflectivity factor', units='dB', comment=COMMENTS['Z_error']), 'Z_bias': MetaData(long_name='Bias in radar reflectivity factor', units='dB', comment=COMMENTS['bias']), 'Z_sensitivity': MetaData(long_name='Minimum detectable radar reflectivity', units='dBZ', comment=COMMENTS['Z_sensitivity']), 'Zh': MetaData(
def _parse_int(row: np.ndarray) -> np.ndarray: values = ma.masked_all((len(row),)) for ind, value in enumerate(row): try: value = int(value) if value != 0: values[ind] = value except ValueError: pass return values ATTRIBUTES = { "velocity": MetaData( long_name="Center fall velocity of precipitation particles", units="m s-1", comment="Predefined velocity classes.", ), "velocity_spread": MetaData( long_name="Width of velocity interval", units="m s-1", comment="Bin size of each velocity interval.", ), "velocity_bnds": MetaData( long_name="Velocity bounds", units="m s-1", comment="Upper and lower bounds of velocity interval.", ), "diameter": MetaData( long_name="Center diameter of precipitation particles", units="m",
" will have caused radar and lidar attenuation; if bit 5 is set then\n" " a correction for the radar attenuation has been performed;\n" " otherwise do not trust the absolute values of reflectivity factor.\n" " No correction is performed for lidar attenuation.\n" "Bit 5: Radar reflectivity has been corrected for liquid-water attenuation\n" " using the microwave radiometer measurements of liquid water path\n" " and the lidar estimation of the location of liquid water cloud;\n" " be aware that errors in reflectivity may result.\n"), } CATEGORIZE_ATTRIBUTES = { # Radar variables "Z": MetaData( long_name="Radar reflectivity factor", units="dBZ", comment=COMMENTS["Z"], ancillary_variables="Z_error Z_bias Z_sensitivity", ), "Z_error": MetaData(long_name="Error in radar reflectivity factor", units="dB", comment=COMMENTS["Z_error"]), "Z_bias": MetaData(long_name="Bias in radar reflectivity factor", units="dB", comment=COMMENTS["bias"]), "Z_sensitivity": MetaData( long_name="Minimum detectable radar reflectivity", units="dBZ", comment=COMMENTS["Z_sensitivity"],