def __init__(self,context,MCsteps=1000,parallel_cores=1):
self._measurement_function_factory = InterpolationFactory()
self.prop = PropagateUnc(context, MCsteps, parallel_cores=parallel_cores)
self.templ = DataTemplates(context=context)
self.writer=HypernetsWriter(context)
self.plot=Plotting(context)
self.context=context
def test_return_directory_context_nottoarchive(self):
context = Context()
context.set_config_value("archive_directory", "directory")
context.set_config_value("site", "site")
context.set_config_value("time", dt(2020, 4, 5, 11, 23, 4, 5))
context.set_config_value("to_archive", False)
hw = HypernetsWriter(context)
self.assertEqual(os.path.join("directory"), hw.return_directory())
def test_write_csv(self, mock_write, mock_dir, mock_fmt):
ds = Dataset()
ds.attrs["product_name"] = "test"
hw = HypernetsWriter()
hw.write(ds)
mock_write.assert_called_once_with(
ds, os.path.join("directory", "test.csv"))
def test__write_netcdf(self):
ds = MagicMock()
path = "test.nc"
HypernetsWriter._write_netcdf(ds, path)
ds.to_netcdf.assert_called_once_with(path,
encoding={},
engine='netcdf4',
format='netCDF4')
def test_write_netcdf(self, mock_write, mock_dir, mock_fmt):
ds = Dataset()
ds.attrs["product_name"] = "test"
hw = HypernetsWriter()
hw.write(ds)
mock_write.assert_called_once_with(ds,
os.path.join(
"directory", "test.nc"),
compression_level=None)
def test_fill_ds(self):
ds = Dataset()
ds["array_variable1"] = DatasetUtil.create_variable([7, 8], np.float32)
ds["array_variable2"] = DatasetUtil.create_variable([7, 8], np.float32)
ds["array_variable1"][2, 3] = np.nan
ds["array_variable2"][2, 3] = np.nan
HypernetsWriter.fill_ds(ds)
self.assertTrue(np.all(ds["array_variable1"] == 9.96921E36))
self.assertTrue(np.all(ds["array_variable2"] == 9.96921E36))
def __init__(self, context):
dir_path = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
self.path_ascii = os.path.join(dir_path, 'calibration_files_ascii',
'HYPSTAR_cal')
self.path_netcdf = os.path.join(
dir_path, 'hypernets_processor/calibration/calibration_files',
'HYPSTAR_cal')
context.set_config_value("product_format", "netcdf")
self.templ = DataTemplates(context)
self.writer = HypernetsWriter(context)
self.context = context
def __init__(self, context, MCsteps=1000, parallel_cores=1):
self._measurement_function_factory = ProtocolFactory(context=context)
self.prop = PropagateUnc(context,
MCsteps,
parallel_cores=parallel_cores)
self.templ = DataTemplates(context=context)
self.writer = HypernetsWriter(context)
self.avg = Average(context)
self.calibrate = Calibrate(context)
self.plot = Plotting(context)
self.context = context
self.rh = RhymerHypstar(context)
self.rhp = RhymerProcessing(context)
self.rhs = RhymerShared(context)
def __init__(self, context):
self.context = context
self.model = self.context.get_config_value("model").split(',')
self.templ = DataTemplates(context)
self.writer = HypernetsWriter(context=context)
cckeys = [
'mapping_vis_a', 'mapping_vis_b', 'mapping_vis_c', 'mapping_vis_d',
'mapping_vis_e', 'mapping_vis_f'
]
ccvalues = []
for i in range(len(cckeys)):
ccvalues.append(self.context.get_config_value(cckeys[i]))
self.cc_vis = dict(zip(cckeys, ccvalues))
class ArchiveDB(dataset.Database):
"""
Class for handling Archive Database in memory, inherits from dataset.Databases
:type url: str
:param url: database url
:type context: hypernets_processor.context.Context
:param context: processor context
"""
def __init__(self, url, context):
self.context = context
self.writer = HypernetsWriter(context)
super().__init__(url)
def archive_product(self, ds, path):
"""
Adds product to archive database
:type ds: xarray.dataset
:param ds: product to archive
:type path: str
:param path: path product is being written to
"""
tbl = self.get_table("products")
tbl.insert(
dict(
product_name=ds.attrs["product_name"],
product_path=path,
product_level="",
datetime=self.context.get_config_value("time"),
sequence_name=self.context.get_config_value("sequence_name"),
sequence_path=self.context.get_config_value("sequence_path"),
site_id=ds.attrs["site_id"],
system_id=ds.attrs["system_id"],
plot_path=self.writer.return_plot_directory(),
image_path=self.writer.return_image_directory(),
# solar_zenith_angle_min=ds.attrs["solar_zenith_angle_min"],
# solar_zenith_angle_max=ds.attrs["solar_zenith_angle_max"],
# solar_azimuth_angle_min=ds.attrs["solar_azimuth_angle_min"],
# solar_azimuth_angle_max=ds.attrs["solar_azimuth_angle_max"],
# viewing_zenith_angle_min=ds.attrs["viewing_zenith_angle_min"],
# viewing_zenith_angle_max=ds.attrs["viewing_zenith_angle_max"],
# viewing_azimuth_angle_min=ds.attrs["viewing_azimuth_angle_min"],
# viewing_azimuth_angle_max=ds.attrs["viewing_azimuth_angle_max"],
))
def __init__(self, context, path=None):
self.context = context
if path is None:
self.path = HypernetsWriter(context).return_plot_directory()
if not os.path.exists(self.path):
os.makedirs(self.path)
pass
def __init__(self, context):
self.context = context
self.templ = DataTemplates(context=context)
self.writer = HypernetsWriter(context)
self.avg = Average(context)
self.intp = Interpolate(context, MCsteps=1000)
self.plot = Plotting(context)
self.rhymeranc = RhymerAncillary(context)
self.rhymerproc = RhymerProcessing(context)
self.rhymershared = RhymerShared(context)
def create_sample_W_L1B_file(output_directory, n_sequence, site):
dsb = HypernetsDSBuilder()
dataset = dsb.create_ds_template(
{"wavelength": N_WAVELENGTHS, "sequence": n_sequence}, "W_L1B"
)
# wavelength data
# todo - fix issue with wavelength assignment as coordinate
dataset["wavelength"].data = np.concatenate(
(np.arange(400, 1000, 3), np.arange(1000, 1700 + 10, 10))
)
dataset["bandwidth"].data = np.random.normal(1.0, 0.5, N_WAVELENGTHS)
# geometry data
dataset["viewing_azimuth_angle"].data = np.linspace(30, 60, n_sequence)
dataset["viewing_zenith_angle"].data = np.linspace(30, 60, n_sequence)
dataset["solar_azimuth_angle"].data = np.linspace(30, 60, n_sequence)
dataset["solar_zenith_angle"].data = np.linspace(30, 60, n_sequence)
# observation data
dataset["upwelling_radiance"].data = np.round(
np.random.rand(N_WAVELENGTHS, n_sequence), 3
)
# dataset["u_random_reflectance"].data = np.random.normal(1.0, 0.5, (N_WAVELENGTHS, n_sequence))
# dataset["u_systematic_reflectance"].data = np.random.normal(1.0, 0.5, (N_WAVELENGTHS,n_sequence))
# dataset["cov_random_reflectance"].data = np.random.normal(1.0, 0.5, (N_WAVELENGTHS, N_WAVELENGTHS))
# dataset["cov_systematic_reflectance"].data = np.random.normal(1.0, 0.5, (N_WAVELENGTHS, N_WAVELENGTHS))
# time data
dataset["acquisition_time"].data = np.arange(10000, 10000 + n_sequence, dtype=int)
# make file name
fu = ProductNameUtil()
filename = fu.create_file_name_l1b("w", site, datetime.datetime.today(), "0.00")
# write file
hw = HypernetsWriter()
hw.write(dataset, pjoin(output_directory, filename))
def create_db_template(self, url, db_format, db_format_defs=DB_DICT_DEFS):
"""
Returns empty Hypernets database
:type url: str
:param url: url of database to create
:type db_format: str
:param db_format: product format string
:type db_format_defs: dict
:param db_format_defs: dictionary of schema_dict/schema_sql for each database format
:return: Empty database
:rtype: dataset.Database
"""
format_def = db_format_defs[db_format]
schema_dict = format_def if isinstance(format_def, dict) else None
schema_sql = format_def if isinstance(format_def, str) else None
db = create_template_db(url,
schema_dict=schema_dict,
schema_sql=schema_sql)
if db_format == "archive":
db.__class__ = ArchiveDB
db.context = self.context
db.writer = HypernetsWriter(self.context)
elif db_format == "anomaly":
db.__class__ = AnomolyDB
db.context = self.context
db.anomalies_dict = ANOMALIES_DICT
elif db_format == "metadata":
db.__class__ = MetadataDB
db.context = self.context
return db
class HypernetsReader:
def __init__(self, context):
self.context = context
self.model = self.context.get_config_value("model").split(',')
self.templ = DataTemplates(context)
self.writer = HypernetsWriter(context=context)
cckeys = [
'mapping_vis_a', 'mapping_vis_b', 'mapping_vis_c', 'mapping_vis_d',
'mapping_vis_e', 'mapping_vis_f'
]
ccvalues = []
for i in range(len(cckeys)):
ccvalues.append(self.context.get_config_value(cckeys[i]))
self.cc_vis = dict(zip(cckeys, ccvalues))
# READ METADATA
# CG - 20200331
# old functions: gen2dict, extract_metadata, read_metadata, read_metadata 2, read_spectra -> NOT USED TO REMOVE
# CG - 20200604
# new functions: read_header, read_data, read_footer, read_seq, read_wavelength
def plot_spectra(self, spectra, dataSpectra):
plt.clf()
plt.title(spectra)
plt.plot([i for i in range(len(dataSpectra))], dataSpectra)
plt.show()
# def save(self, path):
# with open(path, 'w') as f:
# f.write('Dataset length: {} bytes\n'
# 'Timestamp: {} ms\n'
# 'CRC32: {} \n'
# 'Entrance: {}\n'
# 'Radiometer: {}\n'
# 'Exposure time: {} ms\n'
# 'Sensor temperature: {} \'C\n'
# 'Pixel count: {}\n'
# 'Tilt:\n'
# '\tx:{}\u00B1{}\n'
# '\t y:{}\u00B1{}\n'
# '\t z:{}\u00B1{}\n'.format(self.header.total_length, self.header.timestamp, hex(self.crc32[0]),
# self.header.spectrum_type.optics.name,
# self.header.spectrum_type.radiometer.name,
# self.header.exposure_time, self.header.temperature,
# self.header.pixel_count,
# self.header.accel_stats.mean_x,
# self.header.accel_stats.std_x,
# self.header.accel_stats.mean_y,
# self.header.accel_stats.std_y,
# self.header.accel_stats.mean_z,
# self.header.accel_stats.std_z))
def read_header(self, f, headerDef):
header = {}
for headLen, headName, headFormat in headerDef:
data = f.read(headLen)
if len(data) != headLen:
self.context.logger.error(
"Spectra length not similar to header length")
break
continue
# if version_info > (3, 0):
# print("%02X " * headLen % (tuple([b for b in data])))
# else:
# print("%02X " * headLen % (tuple([ord(b) for b in data])))
var, = unpack(headFormat, data)
if headName == "Pixel Count": pixel_count = var
if headName == "Spectrum Type Information":
specInfo = format(ord(data), '#010b')
specInfo = ['1' == a for a in reversed(specInfo[2:])]
# bit 7 for VIS radiometer,
# bit 6 for SWIR,
# bit 4 for radiance,
# bit 3 for irradiance,
# bits 4 and 3 for dark;
strInfo = ""
if specInfo[7]: strInfo += "VIS " # noqa
if specInfo[6]: strInfo += "SWIR " # noqa
if not specInfo[3] and not specInfo[4]:
strInfo += "Dark" # noqa
if specInfo[3] and not specInfo[4]: strInfo += "Irr" # noqa
if specInfo[4] and not specInfo[3]: strInfo += "Rad" # noqa
if specInfo[3] and specInfo[4]: strInfo += "Error" # noqa
self.context.logger.debug("Spectrum Type Info : %s " % strInfo)
header[headName] = var
return header
def read_data(self, f, data_len):
prev = 0
self.context.logger.debug("Reading Data spectra ...")
dataSpectra = []
for i in range(int(data_len)): # Last read data is count
data = f.read(2)
if len(data) != 2:
self.context.logger.error(
"Warning : impossible to read 2 bytes")
break
continue
# Read data as unsigned short
unpackData, = unpack('<H', data)
dataSpectra.append(unpackData)
prev = unpackData
return dataSpectra
def read_footer(self, f, datalength):
# print(f)
self.context.logger.debug("Reading CRC32 ...")
data = f.read(datalength)
unpackData, = unpack('<I', data)
def read_wavelength(self, pixcount, cal_data):
pix = range(pixcount)
wav_coef = cal_data["wavelength_coefficients"]
wav_coef_func = np.poly1d(np.flip(wav_coef))
wvl = wav_coef_func(pix)
self.context.logger.debug("Wavelength range: %s -%s" %
(min(wvl), max(wvl)))
return wvl
def read_series_L(self, seq_dir, series, lat, lon, metadata, flag,
fileformat, cal_data, cal_data_swir):
model_name = self.model
# 1. Read header to create template dataset (including wvl and scan dimensions + end of file!!)
# ----------------------------------------
# scan dimension - to have total number of dimensions
index_scan_total = model_name.index("scan_total")
# series id
# ------------------------------------------
# added to consider concanated files
scanDim = sum(
[int(re.split('_|\.', i)[index_scan_total]) for i in series])
# ------------------------------------------
# added to consider non concanated files
# scanDims = [int(re.split('_|\.', i)[index_scan_total]) for i in series]
# scanDim = sum(scanDims)
## rewrite series
# baseName=["_".join(seriesname.split("_", 7)[:7]) for seriesname in series]
# print(baseName)
# newSeries = []
# for i in series:
# # create dictionary from filename
# seriesAttr = re.split('_|\.', i)[:-1] # remove spe extension
# model = dict(zip(model_name, seriesAttr))
# baseName = '_'.join(seriesAttr)
# nbrScans = int(model["scan_total"])
# n = 1
# while n <= nbrScans:
# new_fname = "{}_{}{}".format(baseName, n, '.spe')
# newSeries.append(new_fname)
# n += 1
# series = newSeries
# -----------------------------------------
# wvl dimensions
FOLDER_NAME = os.path.join(seq_dir, "RADIOMETER/")
f = open(FOLDER_NAME + series[0], "rb")
# Header definition with length, description and decoding format
header = self.read_header(f, HEADER_DEF)
self.context.logger.debug(header)
pixCount = header['Pixel Count']
# if bool(header) == False:
# print("Data corrupt go to next line")
# header = self.read_header(f, HEADER_DEF)
if pixCount == 2048:
wvl = self.read_wavelength(pixCount, cal_data)
elif pixCount == 256:
wvl = self.read_wavelength(pixCount, cal_data_swir)
else:
self.context.logger.error(
"The number of wavelength pixels does not match "
"the expected values for VNIR or SWIR.")
# look for the maximum number of lines to read-- maybe not an elegant way to do?
f.seek(0, 2) # go to end of file
eof = f.tell()
f.close()
# 2. Create template dataset
# -----------------------------------
# use template from variables and metadata in format
ds = self.templ.l0_template_dataset(wvl, scanDim, fileformat)
# Keep track of scan number!
scan_number = 0
# read all spectra (== spe files with concanated files) in a series
for spectra in series:
model = dict(zip(model_name, spectra.split('_')[:-1]))
specBlock = model['series_rep'] + '_' + model[
'series_id'] + '_' + model['vaa'] + '_' + model[
'azimuth_ref'] + '_' + model['vza']
# spectra attributes from metadata file
specattr = dict(metadata[specBlock])
# name of spectra file
acquisitionTime = specattr[spectra]
acquisitionTime = datetime.datetime.strptime(
acquisitionTime + "UTC", '%Y%m%dT%H%M%S%Z')
acquisitionTime = acquisitionTime.replace(tzinfo=timezone.utc)
# -------------------------------------
# # account for non concanated files
# spec = "_".join(spectra.split('_')[:-1]) + ".spe"
# acquisitionTime = specattr[spec]
# print(acquisitionTime)
# acquisitionTime = datetime.datetime.strptime(acquisitionTime + "UTC", '%Y%m%dT%H%M%S%Z')
#
# acquisitionTime = acquisitionTime.replace(tzinfo=timezone.utc)
# model = dict(zip(model_name, str.split(spectra, "_")))
# ________________________________________
# -----------------------
# read the file
# -----------------------
with open(FOLDER_NAME + spectra, "rb") as f:
f.seek(0, 2)
file_size = f.tell()
f.seek(0)
print('file size: {}'.format(file_size))
byte_pointer = 0
chunk_size = 1
chunk_counter = 1
while file_size - byte_pointer:
print('Parsing chunk No {}, size {} bytes, bytes left: {}'.
format(chunk_counter, chunk_size,
file_size - byte_pointer))
chunk_size = unpack('<H', f.read(2))[0]
if chunk_size == 4119:
chunk_size = 4131
f.seek(byte_pointer)
chunk_body = f.read(chunk_size)
spectrum = Spectrum.parse_raw(chunk_body)
spectrum.print_header()
print(spectra, scan_number, pixCount, chunk_size,
len(spectrum.body))
byte_pointer = f.tell()
chunk_counter += 1
# if no header comment those lines
# header = self.read_header(f, HEADER_DEF)
# if bool(header) == False:
# self.context.logger.error("Data corrupt go to next line")
# break
# continue
# # -------------------------------------------------------
#pixCount = spectrum.header.pixel_count
scan = spectrum.body
# should include this back again when crc32 is in the headers!
#crc32 = self.read_footer(f, 4)
# HypernetsReader(self.context).plot_spectra(spectra, scan)
# fill in dataset
# maybe xarray has a better way to do - check merge, concat, ...
series_id = model['series_id']
ds["series_id"][scan_number] = series_id
ds["viewing_azimuth_angle"][scan_number] = model['vaa']
ds["viewing_zenith_angle"][scan_number] = model['vza']
# estimate time based on timestamp
ds["acquisition_time"][
scan_number] = datetime.datetime.timestamp(
acquisitionTime)
# #print(datetime.fromtimestamp(acquisitionTime))
# # didn't use acquisition time from instrument
# # possibility that acquisition time is time since reboot, but how to now reboot time?
# # if we use the metadata time header
# timestamp=header['acquisition_time']
# ts = int(timestamp)/1000
# date_time_str = timereboot+'UTC'
# print(date_time_str)
# date_time_obj = datetime.strptime(date_time_str, '%Y%m%dT%H%M%S%Z')
# print(date_time_obj)
# timereboot = datetime.timestamp(date_time_obj)
# print("timereboot =", timereboot)
# print(datetime.fromtimestamp(timereboot))
# print(datetime.fromtimestamp(int(ts+timereboot)))
# print(datetime.fromtimestamp(int(ts+timereboot))-date_time_obj)
if lat is not None:
ds.attrs["site_latitude"] = lat
ds.attrs["site_longitude"] = lon
ds["solar_zenith_angle"][scan_number] = get_altitude(
float(lat), float(lon), acquisitionTime)
ds["solar_azimuth_angle"][scan_number] = get_azimuth(
float(lat), float(lon), acquisitionTime)
else:
self.context.logger.error(
"Lattitude is not found, using default values instead for lat, lon, sza and saa."
)
ds['quality_flag'][scan_number] = flag
ds['integration_time'][scan_number] = header[
'integration_time']
ds['temperature'][scan_number] = header['temperature']
# accelaration:
# Reference acceleration data contains 3x 16 bit signed integers with X, Y and Z
# acceleration measurements respectively. These are factory-calibrated steady-state
# reference acceleration measurements of the gravity vector when instrument is in
# horizontal position. Due to device manufacturing tolerances, these are
# device-specific and should be applied, when estimating tilt from the measured
# acceleration data. Each measurement is bit count of full range ±19.6 m s−2 .
# Acceleration for each axis can be calculated per Eq. (4).
a = 19.6
b = 2**15
ds['acceleration_x_mean'][
scan_number] = header['acceleration_x_mean'] * a / b
ds['acceleration_x_std'][
scan_number] = header['acceleration_x_std'] * a / b
ds['acceleration_y_mean'][
scan_number] = header['acceleration_y_mean'] * a / b
ds['acceleration_y_std'][
scan_number] = header['acceleration_y_std'] * a / b
ds['acceleration_z_mean'][
scan_number] = header['acceleration_z_mean'] * a / b
ds['acceleration_z_std'][
scan_number] = header['acceleration_z_std'] * a / b
#ds['digital_number'][0:pixCount, scan_number] = scan
scan_number += 1
if f.tell() == eof:
nextLine = False
return ds
def read_series(self,
seq_dir,
series,
lat,
lon,
metadata,
flag,
fileformat,
cal_data=None,
cal_data_swir=None):
model_name = self.model
# 1. Read header to create template dataset (including wvl and scan dimensions + end of file!!)
# ----------------------------------------
# scan dimension - to have total number of dimensions
index_scan_total = model_name.index("scan_total")
# series id
# ------------------------------------------
# added to consider concanated files
scanDim = sum(
[int(re.split('_|\.', i)[index_scan_total]) for i in series])
# ------------------------------------------
# added to consider non concanated files
# scanDims = [int(re.split('_|\.', i)[index_scan_total]) for i in series]
# scanDim = sum(scanDims)
## rewrite series
# baseName=["_".join(seriesname.split("_", 7)[:7]) for seriesname in series]
# print(baseName)
# newSeries = []
# for i in series:
# # create dictionary from filename
# seriesAttr = re.split('_|\.', i)[:-1] # remove spe extension
# model = dict(zip(model_name, seriesAttr))
# baseName = '_'.join(seriesAttr)
# nbrScans = int(model["scan_total"])
# n = 1
# while n <= nbrScans:
# new_fname = "{}_{}{}".format(baseName, n, '.spe')
# newSeries.append(new_fname)
# n += 1
# series = newSeries
# -----------------------------------------
# wvl dimensions
FOLDER_NAME = os.path.join(seq_dir, "RADIOMETER/")
f = open(FOLDER_NAME + series[1], "rb")
# Header definition with length, description and decoding format
header = self.read_header(f, HEADER_DEF)
self.context.logger.debug(header)
pixCount = header['Pixel Count']
# if bool(header) == False:
# print("Data corrupt go to next line")
# header = self.read_header(f, HEADER_DEF)
if pixCount == 2048:
wvl = self.read_wavelength(pixCount, cal_data)
# 2. Create template dataset
# -----------------------------------
# use template from variables and metadata in format
ds = self.templ.l0_template_dataset(wvl, scanDim, fileformat)
else:
self.context.logger.error(
"The number of wavelength pixels does not match "
"the expected values for VNIR.")
# look for the maximum number of lines to read-- maybe not an elegant way to do?
f.seek(0, 2) # go to end of file
eof = f.tell()
f.close()
ds.attrs["source_file"] = str(os.path.basename(seq_dir))
ds["wavelength"] = wvl
# ds["bandwidth"]=wvl
ds["scan"] = np.linspace(1, scanDim, scanDim)
# Keep track of scan number!
scan_number = 0
# read all spectra (== spe files with concanated files) in a series
for spectra in series:
model = dict(zip(model_name, spectra.split('_')[:-1]))
specBlock = model['series_rep'] + '_' + model[
'series_id'] + '_' + model['vaa'] + '_' + model[
'azimuth_ref'] + '_' + model['vza']
# spectra attributes from metadata file
specattr = dict(metadata[specBlock])
# name of spectra file
acquisitionTime = specattr[spectra]
acquisitionTime = datetime.datetime.strptime(
acquisitionTime + "UTC", '%Y%m%dT%H%M%S%Z')
acquisitionTime = acquisitionTime.replace(tzinfo=timezone.utc)
# -------------------------------------
# # account for non concanated files
# spec = "_".join(spectra.split('_')[:-1]) + ".spe"
# acquisitionTime = specattr[spec]
# print(acquisitionTime)
# acquisitionTime = datetime.datetime.strptime(acquisitionTime + "UTC", '%Y%m%dT%H%M%S%Z')
#
# acquisitionTime = acquisitionTime.replace(tzinfo=timezone.utc)
# model = dict(zip(model_name, str.split(spectra, "_")))
# ________________________________________
# -----------------------
# read the file
# -----------------------
f = open(FOLDER_NAME + spectra, "rb")
nextLine = True
while nextLine:
# if no header comment those lines
header = self.read_header(f, HEADER_DEF)
if bool(header) == False:
self.context.logger.error("Data corrupt go to next line")
break
continue
# -------------------------------------------------------
pixCount = header['Pixel Count']
scan = self.read_data(f, pixCount)
# should include this back again when crc32 is in the headers!
crc32 = self.read_footer(f, 4)
# HypernetsReader(self.context).plot_spectra(spectra, scan)
# fill in dataset
# maybe xarray has a better way to do - check merge, concat, ...
series_id = model['series_id']
ds["series_id"][scan_number] = series_id
ds["viewing_azimuth_angle"][scan_number] = model['vaa']
ds["viewing_zenith_angle"][scan_number] = model['vza']
# estimate time based on timestamp
ds["acquisition_time"][
scan_number] = datetime.datetime.timestamp(acquisitionTime)
# #print(datetime.fromtimestamp(acquisitionTime))
# # didn't use acquisition time from instrument
# # possibility that acquisition time is time since reboot, but how to now reboot time?
# # if we use the metadata time header
# timestamp=header['acquisition_time']
# ts = int(timestamp)/1000
# date_time_str = timereboot+'UTC'
# print(date_time_str)
# date_time_obj = datetime.strptime(date_time_str, '%Y%m%dT%H%M%S%Z')
# print(date_time_obj)
# timereboot = datetime.timestamp(date_time_obj)
# print("timereboot =", timereboot)
# print(datetime.fromtimestamp(timereboot))
# print(datetime.fromtimestamp(int(ts+timereboot)))
# print(datetime.fromtimestamp(int(ts+timereboot))-date_time_obj)
if lat is not None:
ds.attrs["site_latitude"] = lat
ds.attrs["site_longitude"] = lon
ds["solar_zenith_angle"][scan_number] = get_altitude(
float(lat), float(lon), acquisitionTime)
ds["solar_azimuth_angle"][scan_number] = get_azimuth(
float(lat), float(lon), acquisitionTime)
else:
self.context.logger.error(
"Lattitude is not found, using default values instead for lat, lon, sza and saa."
)
ds['quality_flag'][scan_number] = flag
ds['integration_time'][scan_number] = header[
'integration_time']
ds['temperature'][scan_number] = header['temperature']
# accelaration:
# Reference acceleration data contains 3x 16 bit signed integers with X, Y and Z
# acceleration measurements respectively. These are factory-calibrated steady-state
# reference acceleration measurements of the gravity vector when instrument is in
# horizontal position. Due to device manufacturing tolerances, these are
# device-specific and should be applied, when estimating tilt from the measured
# acceleration data. Each measurement is bit count of full range ±19.6 m s−2 .
# Acceleration for each axis can be calculated per Eq. (4).
a = 19.6
b = 2**15
ds['acceleration_x_mean'][
scan_number] = header['acceleration_x_mean'] * a / b
ds['acceleration_x_std'][
scan_number] = header['acceleration_x_std'] * a / b
ds['acceleration_y_mean'][
scan_number] = header['acceleration_y_mean'] * a / b
ds['acceleration_y_std'][
scan_number] = header['acceleration_y_std'] * a / b
ds['acceleration_z_mean'][
scan_number] = header['acceleration_z_mean'] * a / b
ds['acceleration_z_std'][
scan_number] = header['acceleration_z_std'] * a / b
ds['digital_number'][0:pixCount, scan_number] = scan
scan_number += 1
if f.tell() == eof:
nextLine = False
return ds
def read_series_L(self, seq_dir, series, lat, lon, metadata, flag,
fileformat, cal_data, cal_data_swir):
FOLDER_NAME = os.path.join(seq_dir, "RADIOMETER/")
model_name = self.model
# read all spectra (== spe files with concanated files) in a series
vnir = []
swir = []
for spectra in series:
self.context.logger.debug("processing " + spectra)
model = dict(zip(model_name, spectra.split('_')[:-1]))
specBlock = model['series_rep']+'_'+model['series_id']+'_'+model['vaa']+'_'+\
model['azimuth_ref']+'_'+model['vza']
# spectra attributes from metadata file
specattr = dict(metadata[specBlock])
# name of spectra file
acquisitionTime = specattr[spectra]
acquisitionTime = datetime.datetime.strptime(
acquisitionTime + "UTC", '%Y%m%dT%H%M%S%Z')
acquisitionTime = acquisitionTime.replace(tzinfo=timezone.utc)
# -----------------------
# read the file
# -----------------------
with open(FOLDER_NAME + spectra, "rb") as f:
f.seek(0, 2)
file_size = f.tell()
f.seek(0)
byte_pointer = 0
chunk_size = 1
chunk_counter = 1
while file_size - byte_pointer:
self.context.logger.debug(
'Parsing chunk No {}, size {} bytes, bytes left: {}'.
format(chunk_counter, chunk_size,
file_size - byte_pointer))
chunk_size = unpack('<H', f.read(2))[0]
if chunk_size == 4119:
chunk_size = 4131
f.seek(byte_pointer)
chunk_body = f.read(chunk_size)
spectrum = Spectrum.parse_raw(chunk_body)
#spectrum.print_header()
if len(spectrum.body) > 500:
if len(vnir) == 0:
vnir = spectrum.body
else:
vnir = np.vstack([vnir, spectrum.body])
else:
if len(swir) == 0:
swir = spectrum.body
else:
swir = np.vstack([swir, spectrum.body])
byte_pointer = f.tell()
chunk_counter += 1
self.context.logger.debug(
"vnir data shape in combined raw files: %s \n "
"swir data shape in combined raw files: %s" %
(vnir.shape, swir.shape))
scanDim = vnir.shape[0]
wvl = self.read_wavelength(vnir.shape[1], cal_data)
ds = self.templ.l0_template_dataset(wvl, scanDim, fileformat)
scanDim = swir.shape[0]
wvl = self.read_wavelength(swir.shape[1], cal_data_swir)
ds_swir = self.templ.l0_template_dataset(wvl,
scanDim,
fileformat,
swir=True)
scan_number = 0
scan_number_swir = 0
for spectra in series:
model = dict(zip(model_name, spectra.split('_')[:-1]))
specBlock = model['series_rep'] + '_' + model[
'series_id'] + '_' + model['vaa'] + '_' + model[
'azimuth_ref'] + '_' + model['vza']
# spectra attributes from metadata file
specattr = dict(metadata[specBlock])
# name of spectra file
acquisitionTime = specattr[spectra]
acquisitionTime = datetime.datetime.strptime(
acquisitionTime + "UTC", '%Y%m%dT%H%M%S%Z')
acquisitionTime = acquisitionTime.replace(tzinfo=timezone.utc)
# -----------------------
# read the file
# -----------------------
with open(FOLDER_NAME + spectra, "rb") as f:
f.seek(0, 2)
file_size = f.tell()
f.seek(0)
byte_pointer = 0
chunk_size = 1
chunk_counter = 1
while file_size - byte_pointer:
chunk_size = unpack('<H', f.read(2))[0]
if chunk_size == 4119:
chunk_size = 4131
f.seek(byte_pointer)
chunk_body = f.read(chunk_size)
spectrum = Spectrum.parse_raw(chunk_body)
#spectrum.print_header()
if len(spectrum.body) > 500:
scan = spectrum.body # should include this back again when crc32 is in the headers! #crc32 = self.read_footer(f, 4)
# HypernetsReader(self.context).plot_spectra(spectra, scan)
# fill in dataset # maybe xarray has a better way to do - check merge, concat, ...
series_id = model['series_id']
ds["series_id"][scan_number] = series_id
ds["viewing_azimuth_angle"][scan_number] = model['vaa']
ds["viewing_zenith_angle"][scan_number] = model['vza']
# estimate time based on timestamp
ds["acquisition_time"][
scan_number] = datetime.datetime.timestamp(
acquisitionTime)
# #print(datetime.fromtimestamp(acquisitionTime))
# # didn't use acquisition time from instrument
# # possibility that acquisition time is time since reboot, but how to now reboot time?
# # if we use the metadata time header
# timestamp=header['acquisition_time']
# ts = int(timestamp)/1000
# date_time_str = timereboot+'UTC'
# print(date_time_str)
# date_time_obj = datetime.strptime(date_time_str, '%Y%m%dT%H%M%S%Z')
# print(date_time_obj)
# timereboot = datetime.timestamp(date_time_obj)
# print("timereboot =", timereboot)
# print(datetime.fromtimestamp(timereboot))
# print(datetime.fromtimestamp(int(ts+timereboot)))
# print(datetime.fromtimestamp(int(ts+timereboot))-date_time_obj)
if lat is not None:
ds.attrs["site_latitude"] = lat
ds.attrs["site_longitude"] = lon
ds["solar_zenith_angle"][
scan_number] = get_altitude(
float(lat), float(lon), acquisitionTime)
ds["solar_azimuth_angle"][
scan_number] = get_azimuth(
float(lat), float(lon), acquisitionTime)
else:
self.context.logger.error(
"Lattitude is not found, using default values instead for lat, lon, sza and saa."
)
ds['quality_flag'][scan_number] = flag
ds['integration_time'][
scan_number] = spectrum.header.exposure_time
ds['temperature'][
scan_number] = spectrum.header.temperature
# accelaration:
# Reference acceleration data contains 3x 16 bit signed integers with X, Y and Z
# acceleration measurements respectively. These are factory-calibrated steady-state
# reference acceleration measurements of the gravity vector when instrument is in
# horizontal position. Due to device manufacturing tolerances, these are
# device-specific and should be applied, when estimating tilt from the measured
# acceleration data. Each measurement is bit count of full range ±19.6 m s−2 .
# Acceleration for each axis can be calculated per Eq. (4).
a = 19.6
b = 2**15
ds['acceleration_x_mean'][
scan_number] = spectrum.header.accel_stats.mean_x * a / b
ds['acceleration_x_std'][
scan_number] = spectrum.header.accel_stats.std_x * a / b
ds['acceleration_y_mean'][
scan_number] = spectrum.header.accel_stats.mean_y * a / b
ds['acceleration_y_std'][
scan_number] = spectrum.header.accel_stats.std_y * a / b
ds['acceleration_z_mean'][
scan_number] = spectrum.header.accel_stats.mean_z * a / b
ds['acceleration_z_std'][
scan_number] = spectrum.header.accel_stats.std_z * a / b
ds['digital_number'][:, scan_number] = scan
scan_number += 1
else:
scan = spectrum.body # should include this back again when crc32 is in the headers! #crc32 = self.read_footer(f, 4)
# HypernetsReader(self.context).plot_spectra(spectra, scan)
# fill in dataset # maybe xarray has a better way to do - check merge, concat, ...
series_id = model['series_id']
ds_swir["series_id"][scan_number_swir] = series_id
ds_swir["viewing_azimuth_angle"][
scan_number_swir] = model['vaa']
ds_swir["viewing_zenith_angle"][
scan_number_swir] = model['vza']
# estimate time based on timestamp
ds_swir["acquisition_time"][
scan_number_swir] = datetime.datetime.timestamp(
acquisitionTime)
# #print(datetime.fromtimestamp(acquisitionTime))
# # didn't use acquisition time from instrument
# # possibility that acquisition time is time since reboot, but how to now reboot time?
# # if we use the metadata time header
# timestamp=header['acquisition_time']
# ts = int(timestamp)/1000
# date_time_str = timereboot+'UTC'
# print(date_time_str)
# date_time_obj = datetime.strptime(date_time_str, '%Y%m%dT%H%M%S%Z')
# print(date_time_obj)
# timereboot = datetime.timestamp(date_time_obj)
# print("timereboot =", timereboot)
# print(datetime.fromtimestamp(timereboot))
# print(datetime.fromtimestamp(int(ts+timereboot)))
# print(datetime.fromtimestamp(int(ts+timereboot))-date_time_obj)
if lat is not None:
ds_swir.attrs["site_latitude"] = lat
ds_swir.attrs["site_longitude"] = lon
ds_swir["solar_zenith_angle"][
scan_number_swir] = get_altitude(
float(lat), float(lon), acquisitionTime)
ds_swir["solar_azimuth_angle"][
scan_number_swir] = get_azimuth(
float(lat), float(lon), acquisitionTime)
else:
self.context.logger.error(
"Lattitude is not found, using default values instead for lat, lon, sza and saa."
)
ds_swir['quality_flag'][scan_number_swir] = flag
ds_swir['integration_time'][
scan_number_swir] = spectrum.header.exposure_time
ds_swir['temperature'][
scan_number_swir] = spectrum.header.temperature
# accelaration:
# Reference acceleration data contains 3x 16 bit signed integers with X, Y and Z
# acceleration measurements respectively. These are factory-calibrated steady-state
# reference acceleration measurements of the gravity vector when instrument is in
# horizontal position. Due to device manufacturing tolerances, these are
# device-specific and should be applied, when estimating tilt from the measured
# acceleration data. Each measurement is bit count of full range ±19.6 m s−2 .
# Acceleration for each axis can be calculated per Eq. (4).
a = 19.6
b = 2**15
ds_swir['acceleration_x_mean'][
scan_number_swir] = spectrum.header.accel_stats.mean_x * a / b
ds_swir['acceleration_x_std'][
scan_number_swir] = spectrum.header.accel_stats.std_x * a / b
ds_swir['acceleration_y_mean'][
scan_number_swir] = spectrum.header.accel_stats.mean_y * a / b
ds_swir['acceleration_y_std'][
scan_number_swir] = spectrum.header.accel_stats.std_y * a / b
ds_swir['acceleration_z_mean'][
scan_number_swir] = spectrum.header.accel_stats.mean_z * a / b
ds_swir['acceleration_z_std'][
scan_number_swir] = spectrum.header.accel_stats.std_z * a / b
ds_swir['digital_number'][:, scan_number_swir] = scan
scan_number_swir += 1
byte_pointer = f.tell()
chunk_counter += 1
return ds, ds_swir
def read_metadata(self, seq_dir):
model_name = self.model
flag = 0
# Spectra name : AA_BBB_CCCC_D_EEEE_FFF_GG_HHHH_II_JJJJ.spe
# A : iterator over "the sequence repeat time"
# B : Number of the line in the sequence file (csv file)
# C : azimuth pointing angle
# D : reference for the azimuth angle
# E : zenith pointing angle
# F : mode
# G : action
# H : integration time
# I : number of scan in the serie
# J : serie time
# .spe : extension
# D (reference) :
# 0 = abs
# 1 = nor
# 2 = sun
# F (mode) :
# MODE_NONE : 0x00 (000)
# MODE_SWIR : 0X40 (064)
# MODE_VIS : 0x80 (128)
# MODE_BOTH : 0xC0 (192)
# G (action) :
# ACTION_BLACK : 0x00 (00)
# ACTION_RAD : 0x10 (16)
# ACTION_IRR : 0x08 (08)
# ACTION_CAL : 0x01 (01)
# ACTION_PIC : 0x02 (02)
# ACTION_NONE : 0x03 (03)
metadata = ConfigParser()
print("seq", os.path.join(seq_dir, "metadata.txt"))
if os.path.exists(os.path.join(seq_dir, "metadata.txt")):
metadata.read(os.path.join(seq_dir, "metadata.txt"))
# ------------------------------
# global attributes + wavelengths -> need to check for swir
# ----------------------------------
globalattr = dict(metadata['Metadata'])
seq = globalattr['datetime']
# reboot time if we want to use acquisition time
# timereboot=globalattr['datetime']
# look for latitude and longitude or lat and lon , more elegant way??
if 'latitude' in (globalattr.keys()):
lat = float(globalattr['latitude'])
elif 'lat' in (globalattr.keys()):
lat = float(globalattr['lat'])
else:
# self.context.logger.error("Latitude is not given, use default")
lat = self.context.get_config_value("lat")
flag = flag + 2**self.context.get_config_value(
"lat_default") # du.set_flag(flag, "lat_default") #
if 'longitude' in (globalattr.keys()):
lon = float(globalattr['longitude'])
elif 'lon' in (globalattr.keys()):
lon = float(globalattr['lon'])
else:
# self.context.logger.error("Longitude is not given, use default")
lon = self.context.get_config_value("lon")
flag = flag + 2**self.context.get_config_value(
"lon_default") # du.set_flag(flag, "lon_default") #
# 2. Estimate wavelengths - NEED TO CHANGE HERE!!!!!!
# ----------------------
# from 1 to 14 cause only able to read the visible wavelengths.... how to read the swir once?
# to change!!!!
if 'cc' not in globalattr:
cc = self.cc_vis
else:
cc = list(str.split(globalattr['cc'], "\n"))
cc = {
k.strip(): float(v.strip())
for k, v in (i.split(":") for i in cc[1:14])
}
# 3. Read series
# ---------------------------
# check for radiance and irradiance series within the metadata
series_all = metadata.sections()[1:len(metadata)]
seriesName = []
seriesPict = []
for i in series_all:
seriesattr = dict(metadata[i])
seriesName.extend(
list(name for name in seriesattr if '.spe' in name))
seriesPict.extend(
list(name for name in seriesattr if '.jpg' in name))
# ----------------
# Make list per action
# ----------------
# ACTION_BLACK : 0x00 (00)
# ACTION_RAD : 0x10 (16)
# ACTION_IRR : 0x08 (08)
# ACTION_CAL : 0x01 (01)
# ACTION_PIC : 0x02 (02) - NOT IN THE FILENAME!
# ACTION_NONE : 0x03 (03)
index_action = model_name.index("action")
action = [re.split('_|\.', i)[index_action] for i in seriesName]
# self.context.logger.info(action)
# this is slow????
seriesIrr = [x for x, y in zip(seriesName, action) if int(y) == 8]
seriesBlack = [
x for x, y in zip(seriesName, action) if int(y) == 0
]
seriesRad = [x for x, y in zip(seriesName, action) if int(y) == 16]
else:
self.context.logger.error(
"Missing metadata file in sequence directory - check sequence directory"
)
self.context.anomaly_db.add_anomaly("s")
return seq, lat, lon, cc, metadata, seriesIrr, seriesRad, seriesBlack, seriesPict, flag
def read_sequence(self,
seq_dir,
calibration_data_rad,
calibration_data_irr,
calibration_data_swir_rad=None,
calibration_data_swir_irr=None):
# define data to return none at end of method if does not exist
l0_irr = None
l0_rad = None
l0_bla = None
seq, lat, lon, cc, metadata, seriesIrr, seriesRad, seriesBlack, seriesPict, flag = self.read_metadata(
seq_dir)
if seriesIrr:
if self.context.get_config_value("network") == "w":
l0_irr = self.read_series(seq_dir, seriesIrr, lat, lon,
metadata, flag, "L0_IRR",
calibration_data_irr)
if self.context.get_config_value("write_l0"):
self.writer.write(l0_irr, overwrite=True)
else:
l0_irr, l0_swir_irr = self.read_series_L(
seq_dir, seriesIrr, lat, lon, metadata, flag, "L0_IRR",
calibration_data_irr, calibration_data_swir_irr)
if self.context.get_config_value("write_l0"):
self.writer.write(l0_irr, overwrite=True)
self.writer.write(l0_swir_irr, overwrite=True)
else:
self.context.logger.error("No irradiance data for this sequence")
if seriesRad:
if self.context.get_config_value("network") == "w":
l0_rad = self.read_series(seq_dir, seriesRad, lat, lon,
metadata, flag, "L0_RAD",
calibration_data_rad)
if self.context.get_config_value("write_l0"):
self.writer.write(l0_rad, overwrite=True)
else:
l0_rad, l0_swir_rad = self.read_series_L(
seq_dir, seriesRad, lat, lon, metadata, flag, "L0_RAD",
calibration_data_rad, calibration_data_swir_rad)
if self.context.get_config_value("write_l0"):
self.writer.write(l0_rad, overwrite=True)
self.writer.write(l0_swir_rad, overwrite=True)
else:
self.context.logger.error("No radiance data for this sequence")
if seriesBlack:
if self.context.get_config_value("network") == "w":
l0_bla = self.read_series(seq_dir, seriesBlack, lat, lon,
metadata, flag, "L0_BLA",
calibration_data_rad)
if self.context.get_config_value("write_l0"):
self.writer.write(l0_bla, overwrite=True)
else:
l0_bla, l0_swir_bla = self.read_series_L(
seq_dir, seriesBlack, lat, lon, metadata, flag, "L0_BLA",
calibration_data_rad, calibration_data_swir_rad)
if self.context.get_config_value("write_l0"):
self.writer.write(l0_bla, overwrite=True)
self.writer.write(l0_swir_bla, overwrite=True)
else:
self.context.logger.error("No black data for this sequence")
if seriesPict:
print("Here we should move the pictures to some place???")
else:
self.context.logger.error("No pictures for this sequence")
if self.context.get_config_value("network") == "w":
return l0_irr, l0_rad, l0_bla
else:
return l0_irr, l0_rad, l0_bla, l0_swir_irr, l0_swir_rad, l0_swir_bla
def __init__(self, url, context):
self.context = context
self.writer = HypernetsWriter(context)
super().__init__(url)
class CalibrationConverter:
def __init__(self, context):
dir_path = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
self.path_ascii = os.path.join(dir_path, 'calibration_files_ascii',
'HYPSTAR_cal')
self.path_netcdf = os.path.join(
dir_path, 'hypernets_processor/calibration/calibration_files',
'HYPSTAR_cal')
context.set_config_value("product_format", "netcdf")
self.templ = DataTemplates(context)
self.writer = HypernetsWriter(context)
self.context = context
def read_calib_files(self):
hypstar = "hypstar_" + str(
self.context.get_config_value("hypstar_cal_number"))
hypstar_path = os.path.join(self.path_netcdf, hypstar)
name = "HYPERNETS_CAL_" + hypstar.upper() + "_RAD_v" + str(
version) + ".nc"
calibration_data_rad = xarray.open_dataset(
os.path.join(hypstar_path, name))
name = "HYPERNETS_CAL_" + hypstar.upper() + "_IRR_v" + str(
version) + ".nc"
calibration_data_irr = xarray.open_dataset(
os.path.join(hypstar_path, name))
calibration_data_times = calibration_data_rad[
"calibrationdates"].values
nonlin_times = calibration_data_rad["nonlineardates"].values
wav_times = calibration_data_rad["wavdates"].values
calibration_data_rad = calibration_data_rad.sel(
calibrationdates=calibration_data_times[-1])
calibration_data_rad = calibration_data_rad.sel(
nonlineardates=nonlin_times[-1])
calibration_data_rad = calibration_data_rad.sel(wavdates=wav_times[-1])
calibration_data_irr = calibration_data_irr.sel(
calibrationdates=calibration_data_times[-1])
calibration_data_irr = calibration_data_irr.sel(
nonlineardates=nonlin_times[-1])
calibration_data_irr = calibration_data_irr.sel(wavdates=wav_times[-1])
if self.context.get_config_value("network") == "l":
name = "HYPERNETS_CAL_" + hypstar.upper() + "_RAD_SWIR_v" + str(
version) + ".nc"
calibration_data_rad_swir = xarray.open_dataset(
os.path.join(hypstar_path, name))
name = "HYPERNETS_CAL_" + hypstar.upper() + "_IRR_SWIR_v" + str(
version) + ".nc"
calibration_data_irr_swir = xarray.open_dataset(
os.path.join(hypstar_path, name))
calibration_data_times = calibration_data_rad_swir[
"calibrationdates"].values
nonlin_times = calibration_data_rad_swir["nonlineardates"].values
calibration_data_rad_swir = calibration_data_rad_swir.sel(
calibrationdates=calibration_data_times[-1])
calibration_data_rad_swir = calibration_data_rad_swir.sel(
nonlineardates=nonlin_times[-1])
calibration_data_rad_swir = calibration_data_rad_swir.sel(
wavdates=wav_times[-1])
calibration_data_irr_swir = calibration_data_irr_swir.sel(
calibrationdates=calibration_data_times[-1])
calibration_data_irr_swir = calibration_data_irr_swir.sel(
nonlineardates=nonlin_times[-1])
calibration_data_irr_swir = calibration_data_irr_swir.sel(
wavdates=wav_times[-1])
return (calibration_data_rad, calibration_data_irr,
calibration_data_rad_swir, calibration_data_irr_swir)
else:
return calibration_data_rad, calibration_data_irr
def convert_all_calibration_data(self):
measurandstrings = ["radiance", "irradiance"]
hypstars = [
os.path.basename(path)
for path in glob.glob(os.path.join(self.path_ascii, "hypstar_*"))
]
for hypstar in hypstars:
print("processing " + hypstar)
hypstar_path = os.path.join(self.path_netcdf, hypstar)
if not os.path.exists(hypstar_path):
os.makedirs(hypstar_path)
for measurandstring in measurandstrings:
if measurandstring == "radiance":
tag = "_RAD_"
else:
tag = "_IRR_"
calib_data = self.prepare_calibration_data(
measurandstring, hypstar=hypstar[8::])
calib_data.attrs["product_name"] = "HYPERNETS_CAL_"+hypstar.upper()\
+tag+"v"+str(version)
self.writer.write(calib_data,
directory=hypstar_path,
overwrite=True)
if hypstar[8] == "2":
tag = tag + "SWIR_"
calib_data = self.prepare_calibration_data(
measurandstring, hypstar=hypstar[8::], swir=True)
calib_data.attrs["product_name"] = "HYPERNETS_CAL_"+\
hypstar.upper()+tag+"v"+str(version)
self.writer.write(calib_data,
directory=hypstar_path,
overwrite=True)
def prepare_calibration_data(self, measurandstring, hypstar, swir=False):
if swir:
sensortag = "swir"
else:
sensortag = "vnir"
directory = self.path_ascii
caldatepaths = [
os.path.basename(path) for path in glob.glob(
os.path.join(directory, "hypstar_" + str(hypstar) +
"/radiometric/*"))
]
nonlindates = []
caldates = []
for caldatepath in caldatepaths:
caldate = caldatepath
nonlinpath = glob.glob(
os.path.join(
directory, "hypstar_" + str(hypstar) + "\\radiometric\\" +
str(caldatepath) + "\\hypstar_" + str(hypstar) +
"_nonlin_corr_coefs_*.dat"))[0]
if os.path.exists(nonlinpath):
nonlindates = np.append(nonlindates, caldate)
non_linear_cals = np.genfromtxt(nonlinpath)[:, 0]
if measurandstring == "radiance":
calpath = glob.glob(
os.path.join(
directory, "hypstar_" + str(hypstar) +
"\\radiometric\\" + str(caldatepath) + "\\hypstar_" +
str(hypstar) + "_radcal_L_*_%s.dat" % (sensortag)))[0]
else:
calpath = glob.glob(
os.path.join(
directory, "hypstar_" + str(hypstar) +
"\\radiometric\\" + str(caldatepath) + "\\hypstar_" +
str(hypstar) + "_radcal_E_*_%s.dat" % (sensortag)))[0]
if os.path.exists(calpath):
caldates = np.append(caldates, caldate)
gains = np.genfromtxt(calpath)
wavs = gains[:, 1]
wavcaldatepaths = [
os.path.basename(path) for path in glob.glob(
os.path.join(directory, "hypstar_" + str(hypstar) +
"/wavelength/*"))
]
wavcaldates = []
for wavcaldatepath in wavcaldatepaths:
wavcaldate = wavcaldatepath
wavcalpath = glob.glob(
os.path.join(
directory, "hypstar_" + str(hypstar) + "\\wavelength\\" +
str(wavcaldatepath) + "\\hypstar_" + str(hypstar) +
"_wl_coefs_*.dat"))[0]
if os.path.exists(wavcalpath):
wavcaldates = np.append(wavcaldates, wavcaldate)
wav_cals = np.genfromtxt(wavcalpath)[:, 0]
calibration_data = self.templ.calibration_dataset(
wavs, non_linear_cals, wav_cals, caldates, nonlindates,
wavcaldates)
i_nonlin = 0
for caldatepath in caldatepaths:
nonlinpath = glob.glob(
os.path.join(
directory, "hypstar_" + str(hypstar) + "\\radiometric\\" +
str(caldatepath) + "\\hypstar_" + str(hypstar) +
"_nonlin_corr_coefs_*.dat"))[0]
if os.path.exists(nonlinpath):
non_linear_cals = np.genfromtxt(nonlinpath)[:, 0]
calibration_data["non_linearity_coefficients"].values[
i_nonlin] = non_linear_cals
i_nonlin += 1
i_wavcoef = 0
for wavcaldatepath in wavcaldatepaths:
wavcaldate = wavcaldatepath
wavcalpath = glob.glob(
os.path.join(
directory, "hypstar_" + str(hypstar) + "\\wavelength\\" +
str(wavcaldatepath) + "\\hypstar_" + str(hypstar) +
"_wl_coefs_*.dat"))[0]
if os.path.exists(wavcalpath):
wav_cals = np.genfromtxt(wavcalpath)
if measurandstring == "radiance" and not swir:
wav_cals = wav_cals[:, 0]
if measurandstring == "irradiance" and not swir:
wav_cals = wav_cals[:, 1]
if measurandstring == "radiance" and swir:
wav_cals = wav_cals[:, 2]
if measurandstring == "irradiance" and swir:
wav_cals = wav_cals[:, 3]
calibration_data["wavelength_coefficients"].values[
i_wavcoef] = wav_cals
i_wavcoef += 1
i_cal = 0
for caldatepath in caldatepaths:
if measurandstring == "radiance":
calpath = glob.glob(
os.path.join(
directory, "hypstar_" + str(hypstar) +
"\\radiometric\\" + str(caldatepath) + "\\hypstar_" +
str(hypstar) + "_radcal_L_*_%s.dat" % (sensortag)))[0]
else:
calpath = glob.glob(
os.path.join(
directory, "hypstar_" + str(hypstar) +
"\\radiometric\\" + str(caldatepath) + "\\hypstar_" +
str(hypstar) + "_radcal_E_*_%s.dat" % (sensortag)))[0]
if os.path.exists(calpath):
caldates = np.append(caldates, caldate)
gains = np.genfromtxt(calpath)
calibration_data["wavelengths"].values[i_cal] = gains[:, 1]
calibration_data["wavpix"].values[i_cal] = gains[:, 0]
calibration_data["gains"].values[i_cal] = gains[:, 2]
#calibration_data["u_random_gains"].values = None
#calibration_data["corr_random_gains"].values = None
calibration_data["u_systematic_indep_gains"].values[
i_cal] = gains[:, 2] * (
gains[:, 6]**2 + gains[:, 7]**2 + gains[:, 8]**2 +
gains[:, 9]**2 + gains[:, 10]**2 + gains[:, 11]**2 +
gains[:, 12]**2 + gains[:, 13]**2 + gains[:, 14]**2 +
gains[:, 15]**2 + gains[:, 16]**2 + gains[:, 17]**2 +
gains[:, 19]**2)**0.5 / 100
cov_diag = punpy.convert_corr_to_cov(
np.eye(len(gains[:, 2])),
gains[:, 2] * (gains[:, 19]) / 100)
cov_other = punpy.convert_corr_to_cov(
np.eye(len(gains[:, 2])), gains[:, 2] *
(gains[:, 8]**2 + gains[:, 9]**2 + gains[:, 11]**2 +
gains[:, 16]**2 + gains[:, 17]**2)**0.5 / 100)
cov_full = punpy.convert_corr_to_cov(
np.ones((len(gains[:, 2]), len(gains[:,
2]))), gains[:, 2] *
(gains[:, 7]**2 + gains[:, 10]**2 + gains[:, 12]**2 +
gains[:, 13]**2 + gains[:, 14]**2 + gains[:, 15]**2)**0.5
/ 100)
cov_filament = punpy.convert_corr_to_cov(
np.ones((len(gains[:, 2]), len(gains[:, 2]))),
gains[:, 2] * (gains[:, 6]**2)**0.5 / 100)
calibration_data["corr_systematic_indep_gains"].values[i_cal] = \
punpy.correlation_from_covariance(cov_diag+cov_other+cov_full+cov_filament)
calibration_data["u_systematic_corr_rad_irr_gains"].values[
i_cal] = gains[:, 2] * (gains[:, 4]**2 + gains[:, 5]**2 +
gains[:, 18]**2)**0.5 / 100
cov_other = punpy.convert_corr_to_cov(
np.eye(len(gains[:, 2])), gains[:, 2] *
(gains[:, 4]**2 + gains[:, 18]**2)**0.5 / 100)
cov_filament = punpy.convert_corr_to_cov(
np.ones((len(gains[:, 2]), len(gains[:, 2]))),
gains[:, 2] * (gains[:, 5]**2)**0.5 / 100)
calibration_data["corr_systematic_corr_rad_irr_gains"].values[i_cal] = \
punpy.correlation_from_covariance(cov_other+cov_filament)
i_cal += 1
return calibration_data
class Calibrate:
def __init__(self, context, MCsteps=1000, parallel_cores=0):
self._measurement_function_factory = MeasurementFunctionFactory()
self.prop = PropagateUnc(context,
MCsteps,
parallel_cores=parallel_cores)
self.templ = DataTemplates(context)
self.writer = HypernetsWriter(context)
self.plot = Plotting(context)
self.context = context
def calibrate_l1a(self,
measurandstring,
dataset_l0,
dataset_l0_bla,
calibration_data,
swir=False):
if measurandstring != "radiance" and measurandstring != "irradiance":
self.context.logger.error(
"the measurandstring needs to be either 'radiance' or 'irradiance"
)
exit()
if self.context.get_config_value("plot_l0"):
self.plot.plot_scans_in_series("digital_number", dataset_l0)
calibrate_function = self._measurement_function_factory.get_measurement_function(
self.context.get_config_value("measurement_function_calibrate"))
input_vars = calibrate_function.get_argument_names()
dataset_l0 = self.preprocess_l0(dataset_l0, dataset_l0_bla,
calibration_data)
dataset_l1a = self.templ.l1a_template_from_l0_dataset(
measurandstring, dataset_l0, swir)
input_qty = self.prop.find_input_l1a(input_vars, dataset_l0,
calibration_data)
u_random_input_qty = self.prop.find_u_random_input_l1a(
input_vars, dataset_l0, calibration_data)
u_systematic_input_qty_indep,u_systematic_input_qty_corr,\
corr_systematic_input_qty_indep,corr_systematic_input_qty_corr = self.prop.find_u_systematic_input_l1a(input_vars, dataset_l0, calibration_data)
dataset_l1a = self.prop.process_measurement_function_l1a(
measurandstring, dataset_l1a, calibrate_function.function,
input_qty, u_random_input_qty, u_systematic_input_qty_indep,
u_systematic_input_qty_corr, corr_systematic_input_qty_indep,
corr_systematic_input_qty_corr)
if self.context.get_config_value("write_l1a"):
self.writer.write(dataset_l1a, overwrite=True)
if self.context.get_config_value("plot_l1a"):
self.plot.plot_scans_in_series(measurandstring, dataset_l1a)
if self.context.get_config_value("plot_l1a_diff"):
self.plot.plot_diff_scans(measurandstring, dataset_l1a)
if self.context.get_config_value("plot_uncertainty"):
self.plot.plot_relative_uncertainty(measurandstring, dataset_l1a)
if self.context.get_config_value("plot_correlation"):
self.plot.plot_correlation(measurandstring, dataset_l1a)
return dataset_l1a
def find_nearest_black(self, dataset, acq_time, int_time):
ids = np.where((abs(dataset['acquisition_time'] - acq_time) == min(
abs(dataset['acquisition_time'] - acq_time)))
& (dataset['integration_time'] == int_time))
#todo check if interation time alwasy has to be same
return np.mean(dataset["digital_number"].values[:, ids], axis=2)[:, 0]
def preprocess_l0(self, datasetl0, datasetl0_bla, dataset_calib):
"""
Identifies and removes faulty measurements (e.g. due to cloud cover).
:param dataset_l0:
:type dataset_l0:
:return:
:rtype:
"""
wavs = dataset_calib["wavelength"].values
wavpix = dataset_calib["wavpix"].values
datasetl0 = datasetl0.isel(wavelength=slice(int(wavpix[0]),
int(wavpix[-1]) + 1))
datasetl0_bla = datasetl0_bla.isel(
wavelength=slice(int(wavpix[0]),
int(wavpix[-1]) + 1))
mask = self.clip_and_mask(datasetl0, datasetl0_bla)
datasetl0 = datasetl0.assign_coords(wavelength=wavs)
datasetl0_bla = datasetl0_bla.assign_coords(wavelength=wavs)
datasetl0["quality_flag"][np.where(mask == 1)] = DatasetUtil.set_flag(
datasetl0["quality_flag"][np.where(mask == 1)],
"outliers") #for i in range(len(mask))]
DN_rand = DatasetUtil.create_variable(
[len(datasetl0["wavelength"]),
len(datasetl0["scan"])],
dim_names=["wavelength", "scan"],
dtype=np.uint32,
fill_value=0)
datasetl0["u_random_digital_number"] = DN_rand
rand = np.zeros_like(DN_rand.values)
series_ids = np.unique(datasetl0['series_id'])
for i in range(len(series_ids)):
ids = np.where(datasetl0['series_id'] == series_ids[i])[0]
ids_masked = np.where((datasetl0['series_id'] == series_ids[i])
& (mask == 0))[0]
dark_signals = np.zeros_like(
datasetl0['digital_number'].values[:, ids_masked])
for ii, id in enumerate(ids_masked):
dark_signals[:, ii] = self.find_nearest_black(
datasetl0_bla, datasetl0['acquisition_time'].values[id],
datasetl0['integration_time'].values[id])
std = np.std((datasetl0['digital_number'].values[:, ids_masked] -
dark_signals),
axis=1)
for ii, id in enumerate(ids):
rand[:, id] = std
datasetl0["u_random_digital_number"].values = rand
DN_dark = DatasetUtil.create_variable(
[len(datasetl0["wavelength"]),
len(datasetl0["scan"])],
dim_names=["wavelength", "scan"],
dtype=np.uint32,
fill_value=0)
datasetl0["dark_signal"] = DN_dark
dark_signals = []
acqui = datasetl0['acquisition_time'].values
inttimes = datasetl0['integration_time'].values
for i in range(len(acqui)):
dark_signals.append(
self.find_nearest_black(datasetl0_bla, acqui[i], inttimes[i]))
datasetl0["dark_signal"].values = np.array(dark_signals).T
return datasetl0
def clip_and_mask(self, dataset, dataset_bla, k_unc=3):
mask = []
# check if zeros, max, fillvalue:
# check if integrated signal is outlier
series_ids = np.unique(dataset['series_id'])
for i in range(len(series_ids)):
ids = np.where(dataset['series_id'] == series_ids[i])
dark_signals = self.find_nearest_black(
dataset_bla, np.mean(dataset['acquisition_time'].values[ids]),
np.mean(dataset['integration_time'].values[ids]))
intsig = np.nanmean((dataset["digital_number"].values[:, ids] -
dark_signals[:, None, None]),
axis=0)[0]
noisestd, noiseavg = self.sigma_clip(
intsig) # calculate std and avg for non NaN columns
maski = np.zeros_like(intsig) # mask the columns that have NaN
maski[np.where(np.abs(intsig - noiseavg) >= k_unc * noisestd)] = 1
mask = np.append(mask, maski)
# check if 10% of pixels are outiers
# mask_wvl = np.zeros((len(datasetl0["wavelength"]),len(datasetl0["scan"])))
# for i in range(len(dataset["wavelength"])):
return mask
def sigma_clip(self,
values,
tolerance=0.01,
median=True,
sigma_thresh=3.0):
# Remove NaNs from input values
values = np.array(values)
values = values[np.where(np.isnan(values) == False)]
values_original = np.copy(values)
# Continue loop until result converges
diff = 10E10
while diff > tolerance:
# Assess current input iteration
if median == False:
average = np.mean(values)
elif median == True:
average = np.median(values)
sigma_old = np.std(values)
# Mask those pixels that lie more than 3 stdev away from mean
check = np.zeros([len(values)])
check[np.where(values > (average +
(sigma_thresh * sigma_old)))] = 1
# check[ np.where( values<(average-(sigma_thresh*sigma_old)) ) ] = 1
values = values[np.where(check < 1)]
# Re-measure sigma and test for convergence
sigma_new = np.std(values)
diff = abs(sigma_old - sigma_new) / sigma_old
# Perform final mask
check = np.zeros([len(values)])
check[np.where(values > (average + (sigma_thresh * sigma_old)))] = 1
check[np.where(values < (average - (sigma_thresh * sigma_old)))] = 1
values = values[np.where(check < 1)]
# Return results
return sigma_new, average
def test_return_fmt_netcdf4(self):
hw = HypernetsWriter()
self.assertEqual("nc", hw.return_fmt(fmt="netcdf4"))
class CombineSWIR:
def __init__(self, context, MCsteps=1000, parallel_cores=1):
self._measurement_function_factory = CombineFactory()
self.prop = PropagateUnc(context,
MCsteps,
parallel_cores=parallel_cores)
self.avg = Average(context=context)
self.templ = DataTemplates(context)
self.writer = HypernetsWriter(context)
self.plot = Plotting(context)
self.context = context
def combine(self, measurandstring, dataset_l1a, dataset_l1a_swir):
dataset_l1a = self.perform_checks(dataset_l1a)
dataset_l1b = self.avg.average_l1b(measurandstring, dataset_l1a)
dataset_l1b_swir = self.avg.average_l1b(measurandstring,
dataset_l1a_swir)
combine_function = self._measurement_function_factory.get_measurement_function(
self.context.get_config_value("measurement_function_combine"))
input_vars = combine_function.get_argument_names()
input_qty = [
dataset_l1b["wavelength"].values,
dataset_l1b[measurandstring].values,
dataset_l1b_swir["wavelength"].values,
dataset_l1b_swir[measurandstring].values,
self.context.get_config_value("combine_lim_wav")
]
u_random_input_qty = [
None, dataset_l1b["u_random_" + measurandstring].values, None,
dataset_l1b_swir["u_random_" + measurandstring].values, None
]
u_systematic_input_qty_indep = [
None, dataset_l1b["u_systematic_indep_" + measurandstring].values,
None,
dataset_l1b_swir["u_systematic_indep_" + measurandstring].values,
None
]
u_systematic_input_qty_corr = [
None,
dataset_l1b["u_systematic_corr_rad_irr_" + measurandstring].values,
None, dataset_l1b_swir["u_systematic_corr_rad_irr_" +
measurandstring].values, None
]
corr_systematic_input_qty_indep = [
None,
dataset_l1b["corr_systematic_indep_" + measurandstring].values,
None, dataset_l1b_swir["corr_systematic_indep_" +
measurandstring].values, None
]
corr_systematic_input_qty_corr = [
None, dataset_l1b["corr_systematic_corr_rad_irr_" +
measurandstring].values, None,
dataset_l1b_swir["corr_systematic_corr_rad_irr_" +
measurandstring].values, None
]
#todo do this more consistently with other modules, and do a direct copy for ranges that don't overlap
dataset_l1b_comb = self.templ.l1b_template_from_combine(
measurandstring, dataset_l1b, dataset_l1b_swir)
self.prop.process_measurement_function_l1(
measurandstring,
dataset_l1b_comb,
combine_function.function,
input_qty,
u_random_input_qty,
u_systematic_input_qty_indep,
u_systematic_input_qty_corr,
corr_systematic_input_qty_indep,
corr_systematic_input_qty_corr,
param_fixed=[True, False, True, False, True])
if self.context.get_config_value("write_l1b"):
self.writer.write(dataset_l1b_comb, overwrite=True)
if self.context.get_config_value("plot_l1b"):
self.plot.plot_series_in_sequence(measurandstring,
dataset_l1b_comb)
if self.context.get_config_value("plot_uncertainty"):
self.plot.plot_relative_uncertainty(measurandstring,
dataset_l1b_comb)
if self.context.get_config_value("plot_correlation"):
self.plot.plot_correlation(measurandstring, dataset_l1b_comb)
# if self.context.get_config_value("plot_diff"):
# self.plot.plot_diff_scans(measurandstring,dataset_l1a,dataset_l1b)
return dataset_l1b_comb
def perform_checks(self, dataset_l1):
"""
Identifies and removes faulty measurements (e.g. due to cloud cover).
:param dataset_l0:
:type dataset_l0:
:return:
:rtype:
"""
return dataset_l1
def test_return_fmt_csv(self):
hw = HypernetsWriter()
self.assertEqual("csv", hw.return_fmt(fmt="csv"))
class SurfaceReflectance:
def __init__(self, context, MCsteps=1000, parallel_cores=1):
self._measurement_function_factory = ProtocolFactory(context=context)
self.prop = PropagateUnc(context,
MCsteps,
parallel_cores=parallel_cores)
self.templ = DataTemplates(context=context)
self.writer = HypernetsWriter(context)
self.avg = Average(context)
self.calibrate = Calibrate(context)
self.plot = Plotting(context)
self.context = context
self.rh = RhymerHypstar(context)
self.rhp = RhymerProcessing(context)
self.rhs = RhymerShared(context)
def process_l1c(self, dataset):
dataset_l1c = self.templ.l1c_from_l1b_dataset(dataset)
dataset_l1c = self.rh.get_wind(dataset_l1c)
dataset_l1c = self.rh.get_fresnelrefl(dataset_l1c)
l1ctol1b_function = self._measurement_function_factory.get_measurement_function(
self.context.get_config_value(
"measurement_function_surface_reflectance"))
input_vars = l1ctol1b_function.get_argument_names()
input_qty = self.prop.find_input(input_vars, dataset_l1c)
u_random_input_qty = self.prop.find_u_random_input(
input_vars, dataset_l1c)
u_systematic_input_qty, corr_systematic_input_qty = \
self.prop.find_u_systematic_input(input_vars, dataset_l1c)
L1c = self.prop.process_measurement_function_l2(
[
"water_leaving_radiance", "reflectance_nosc", "reflectance",
"epsilon"
],
dataset_l1c,
l1ctol1b_function.function,
input_qty,
u_random_input_qty,
u_systematic_input_qty,
corr_systematic_input_qty,
param_fixed=[False, False, False, False, True])
failSimil = self.rh.qc_similarity(L1c)
L1c["quality_flag"][np.where(failSimil == 1)] = DatasetUtil.set_flag(
L1c["quality_flag"][np.where(failSimil == 1)],
"simil_fail") # for i in range(len(mask))]
if self.context.get_config_value("write_l1c"):
self.writer.write(L1c, overwrite=True)
for measurandstring in [
"water_leaving_radiance", "reflectance_nosc", "reflectance",
"epsilon"
]:
try:
if self.context.get_config_value("plot_l1c"):
self.plot.plot_series_in_sequence(measurandstring, L1c)
if self.context.get_config_value("plot_uncertainty"):
self.plot.plot_relative_uncertainty(measurandstring,
L1c,
L2=True)
except:
print("not plotting ", measurandstring)
return L1c
def process_l2(self, dataset):
dataset = self.perform_checks(dataset)
l1tol2_function = self._measurement_function_factory.get_measurement_function(
self.context.get_config_value(
"measurement_function_surface_reflectance"))
input_vars = l1tol2_function.get_argument_names()
input_qty = self.prop.find_input(input_vars, dataset)
u_random_input_qty = self.prop.find_u_random_input(input_vars, dataset)
u_systematic_input_qty, cov_systematic_input_qty = \
self.prop.find_u_systematic_input(input_vars, dataset)
if self.context.get_config_value("network").lower() == "w":
dataset_l2a = self.avg.average_L2(dataset)
for measurandstring in [
"water_leaving_radiance", "reflectance_nosc",
"reflectance", "epsilon"
]:
try:
if self.context.get_config_value("plot_l2a"):
self.plot.plot_series_in_sequence(
measurandstring, dataset_l2a)
if self.context.get_config_value("plot_uncertainty"):
self.plot.plot_relative_uncertainty(measurandstring,
dataset_l2a,
L2=True)
if self.context.get_config_value("plot_correlation"):
self.plot.plot_correlation(measurandstring,
dataset_l2a,
L2=True)
except:
print("not plotting ", measurandstring)
elif self.context.get_config_value("network").lower() == "l":
dataset_l2a = self.templ.l2_from_l1c_dataset(dataset)
dataset_l2a = self.prop.process_measurement_function_l2(
["reflectance"], dataset_l2a, l1tol2_function.function,
input_qty, u_random_input_qty, u_systematic_input_qty,
cov_systematic_input_qty)
if self.context.get_config_value("plot_l2a"):
self.plot.plot_series_in_sequence("reflectance", dataset_l2a)
if self.context.get_config_value("plot_uncertainty"):
self.plot.plot_relative_uncertainty("reflectance",
dataset_l2a,
L2=True)
if self.context.get_config_value("plot_correlation"):
self.plot.plot_correlation("reflectance", dataset_l2a, L2=True)
else:
self.context.logger.error("network is not correctly defined")
if self.context.get_config_value("write_l2a"):
self.writer.write(dataset_l2a, overwrite=True)
return dataset_l2a
def perform_checks(self, dataset_l1):
"""
Identifies and removes faulty measurements (e.g. due to cloud cover).
:param dataset_l0:
:type dataset_l0:
:return:
:rtype:
"""
return dataset_l1
def test_return_fmt_invalid(self):
hw = HypernetsWriter()
self.assertRaises(NameError, hw.return_fmt, "invalid")
class Interpolate:
def __init__(self,context,MCsteps=1000,parallel_cores=1):
self._measurement_function_factory = InterpolationFactory()
self.prop = PropagateUnc(context, MCsteps, parallel_cores=parallel_cores)
self.templ = DataTemplates(context=context)
self.writer=HypernetsWriter(context)
self.plot=Plotting(context)
self.context=context
def interpolate_l1b_w(self, dataset_l1b, dataset_l1a_uprad,dataset_l1b_downrad, dataset_l1b_irr):
# chek for upwelling radiance
upscan = [i for i, e in enumerate(dataset_l1a_uprad['viewing_zenith_angle'].values) if e < 90]
dataset_l1b=self.templ.l1c_int_template_from_l1a_dataset_water(dataset_l1a_uprad)
dataset_l1b["wavelength"] = dataset_l1a_uprad["wavelength"]
dataset_l1b["upwelling_radiance"] = dataset_l1a_uprad["radiance"].sel(scan=upscan)
dataset_l1b["acquisition_time"] = dataset_l1a_uprad["acquisition_time"].sel(scan=upscan)
# is this correct????
dataset_l1b["u_random_upwelling_radiance"] = dataset_l1a_uprad["u_random_radiance"].sel(scan=upscan)
dataset_l1b["u_systematic_indep_upwelling_radiance"] = dataset_l1a_uprad["u_systematic_indep_radiance"].sel(scan=upscan)
dataset_l1b["u_systematic_corr_rad_irr_upwelling_radiance"] = dataset_l1a_uprad["u_systematic_corr_rad_irr_radiance"].sel(scan=upscan)
dataset_l1b["corr_random_upwelling_radiance"] = dataset_l1a_uprad["corr_random_radiance"]
dataset_l1b["corr_systematic_indep_upwelling_radiance"] = dataset_l1a_uprad["corr_systematic_indep_radiance"]
dataset_l1b["corr_systematic_corr_rad_irr_upwelling_radiance"] = dataset_l1a_uprad["corr_systematic_corr_rad_irr_radiance"]
self.context.logger.info("interpolate sky radiance")
dataset_l1b=self.interpolate_skyradiance(dataset_l1b, dataset_l1b_downrad)
self.context.logger.info("interpolate irradiances")
dataset_l1b=self.interpolate_irradiance(dataset_l1b, dataset_l1b_irr)
return dataset_l1b
def interpolate_l1c(self,dataset_l1b_rad,dataset_l1b_irr):
dataset_l1c=self.templ.l1c_from_l1b_dataset(dataset_l1b_rad)
dataset_l1c["acquisition_time"].values = dataset_l1b_rad["acquisition_time"].values
dataset_l1c=self.interpolate_irradiance(dataset_l1c,dataset_l1b_irr)
if self.context.get_config_value("write_l1c"):
self.writer.write(dataset_l1c,overwrite=True)
if self.context.get_config_value("plot_l1c"):
self.plot.plot_series_in_sequence("irradiance",dataset_l1c)
if self.context.get_config_value("plot_uncertainty"):
self.plot.plot_relative_uncertainty("irradiance",dataset_l1c)
if self.context.get_config_value("plot_correlation"):
self.plot.plot_correlation("irradiance",dataset_l1c)
return dataset_l1c
def interpolate_irradiance(self,dataset_l1c,dataset_l1b_irr):
measurement_function_interpolate_wav = self.context.get_config_value(
'measurement_function_interpolate_wav')
interpolation_function_wav = self._measurement_function_factory\
.get_measurement_function(measurement_function_interpolate_wav)
measurement_function_interpolate_time = self.context.get_config_value(
'measurement_function_interpolate_time')
interpolation_function_time = self._measurement_function_factory\
.get_measurement_function(measurement_function_interpolate_time)
# Interpolate in wavelength to radiance wavelengths
wavs_rad=dataset_l1c["wavelength"].values
wavs_irr=dataset_l1b_irr["wavelength"].values
dataset_l1c_temp = self.templ.l1ctemp_dataset(dataset_l1c,dataset_l1b_irr)
dataset_l1c_temp = self.prop.process_measurement_function_l1("irradiance",
dataset_l1c_temp,interpolation_function_wav.function,
[wavs_rad,wavs_irr,dataset_l1b_irr['irradiance'].values],
[None,None,dataset_l1b_irr['u_random_irradiance'].values],
[None,None,dataset_l1b_irr['u_systematic_indep_irradiance'].values],
[None,None,dataset_l1b_irr['u_systematic_corr_rad_irr_irradiance'].values],
[None,None,dataset_l1b_irr["corr_systematic_indep_irradiance"].values],
[None,None,dataset_l1b_irr["corr_systematic_corr_rad_irr_irradiance"].values],
)
# Interpolate in time to radiance times
acqui_irr = dataset_l1b_irr['acquisition_time'].values
acqui_rad = dataset_l1c['acquisition_time'].values
dataset_l1c = self.prop.process_measurement_function_l1("irradiance",
dataset_l1c,interpolation_function_time.function,
[acqui_rad,acqui_irr,dataset_l1c_temp['irradiance'].values],
[None,None,dataset_l1c_temp['u_random_irradiance'].values],
[None,None,dataset_l1c_temp['u_systematic_indep_irradiance'].values],
[None,None,dataset_l1c_temp['u_systematic_corr_rad_irr_irradiance'].values],
[None,None,dataset_l1c_temp["corr_systematic_indep_irradiance"].values],
[None,None,dataset_l1c_temp["corr_systematic_corr_rad_irr_irradiance"].values],
param_fixed=[False,True,True])
return dataset_l1c
def interpolate_skyradiance(self,dataset_l1c,dataset_l1a_skyrad):
measurement_function_interpolate_time = self.context.get_config_value(
'measurement_function_interpolate_time')
interpolation_function_time = self._measurement_function_factory\
.get_measurement_function(measurement_function_interpolate_time)
acqui_irr = dataset_l1a_skyrad['acquisition_time'].values
acqui_rad = dataset_l1c['acquisition_time'].values
dataset_l1c = self.prop.process_measurement_function_l1("downwelling_radiance",dataset_l1c,
interpolation_function_time.function,
[acqui_rad,acqui_irr,
dataset_l1a_skyrad[
'radiance'].values],
[None,None,dataset_l1a_skyrad[
'u_random_radiance'].values],
[None,None,dataset_l1a_skyrad[
'u_systematic_indep_radiance'].values],
[None,None,dataset_l1a_skyrad[
'u_systematic_corr_rad_irr_radiance'].values],
[None,None,dataset_l1a_skyrad["corr_systematic_indep_radiance"].values],
[None,None,dataset_l1a_skyrad["corr_systematic_corr_rad_irr_radiance"].values],
param_fixed=[False,True,True])
return dataset_l1c
def __init__(self, context):
self.templ = DataTemplates(context=context)
self.context = context
self.writer = HypernetsWriter(context)
def test_return_fmt_context(self):
context = Context()
context.set_config_value("product_format", "netcdf")
hw = HypernetsWriter(context)
self.assertEqual("nc", hw.return_fmt())
def test_return_directory_noinput(self):
hw = HypernetsWriter()
self.assertRaises(ValueError, hw.return_directory)
def process_sequence(self, sequence_path):
"""
Processes sequence file
"""
# update context
self.context.set_config_value(
"time",
parse_sequence_path(sequence_path)["datetime"])
self.context.set_config_value("sequence_path", sequence_path)
self.context.set_config_value("sequence_name",
os.path.basename(sequence_path))
reader = HypernetsReader(self.context)
calcon = CalibrationConverter(self.context)
cal = Calibrate(self.context, MCsteps=100)
surf = SurfaceReflectance(self.context, MCsteps=1000)
avg = Average(self.context, )
rhymer = RhymerHypstar(self.context)
writer = HypernetsWriter(self.context)
if self.context.get_config_value("network") == "w":
calibration_data_rad, calibration_data_irr = calcon.read_calib_files(
)
# Read L0
self.context.logger.info("Reading raw data...")
l0_irr, l0_rad, l0_bla = reader.read_sequence(
sequence_path, calibration_data_rad, calibration_data_irr)
self.context.logger.info("Done")
# Calibrate to L1a
self.context.logger.info("Processing to L1a...")
L1a_rad = cal.calibrate_l1a("radiance", l0_rad, l0_bla,
calibration_data_rad)
L1a_irr = cal.calibrate_l1a("irradiance", l0_irr, l0_bla,
calibration_data_irr)
self.context.logger.info("Done")
self.context.logger.info("Processing to L1b radiance...")
L1b_rad = avg.average_l1b("radiance", L1a_rad)
print(L1b_rad)
if self.context.get_config_value("write_l1b"):
writer.write(L1b_rad, overwrite=True)
self.context.logger.info("Done")
self.context.logger.info("Processing to L1b irradiance...")
L1b_irr = avg.average_l1b("irradiance", L1a_irr)
if self.context.get_config_value("write_l1b"):
writer.write(L1b_irr, overwrite=True)
self.context.logger.info("Done")
self.context.logger.info("Processing to L1c...")
L1c_int = rhymer.process_l1c_int(L1a_rad, L1a_irr)
L1c = surf.process_l1c(L1c_int)
self.context.logger.info("Done")
self.context.logger.info("Processing to L2a...")
L2a = surf.process_l2(L1c)
self.context.logger.info("Done")
elif self.context.get_config_value("network") == "l":
comb = CombineSWIR(self.context, MCsteps=100)
intp = Interpolate(self.context, MCsteps=1000)
# Read L0
self.context.logger.info("Reading raw data...")
(calibration_data_rad, calibration_data_irr,
calibration_data_swir_rad,
calibration_data_swir_irr) = calcon.read_calib_files()
l0_irr, l0_rad, l0_bla, l0_swir_irr, l0_swir_rad, l0_swir_bla = reader.read_sequence(
sequence_path, calibration_data_rad, calibration_data_irr,
calibration_data_swir_rad, calibration_data_swir_irr)
self.context.logger.info("Done")
# Calibrate to L1a
self.context.logger.info("Processing to L1a...")
L1a_rad = cal.calibrate_l1a("radiance", l0_rad, l0_bla,
calibration_data_rad)
L1a_irr = cal.calibrate_l1a("irradiance", l0_irr, l0_bla,
calibration_data_irr)
L1a_swir_rad = cal.calibrate_l1a("radiance",
l0_swir_rad,
l0_swir_bla,
calibration_data_swir_rad,
swir=True)
L1a_swir_irr = cal.calibrate_l1a("irradiance",
l0_swir_irr,
l0_swir_bla,
calibration_data_swir_irr,
swir=True)
self.context.logger.info("Done")
self.context.logger.info("Processing to L1b radiance...")
L1b_rad = comb.combine("radiance", L1a_rad, L1a_swir_rad)
self.context.logger.info("Done")
self.context.logger.info("Processing to L1b irradiance...")
L1b_irr = comb.combine("irradiance", L1a_irr, L1a_swir_irr)
self.context.logger.info("Done")
self.context.logger.info("Processing to L1c...")
L1c = intp.interpolate_l1c(L1b_rad, L1b_irr)
self.context.logger.info("Done")
self.context.logger.info("Processing to L2a...")
L2a = surf.process_l2(L1c)
self.context.logger.info("Done")
else:
raise NameError("Invalid network: " +
self.context.get_config_value("network"))
return None
def test_return_directory(self):
hw = HypernetsWriter()
self.assertEqual("directory",
hw.return_directory(directory="directory"))