def create_secondary_swath_object(self, product_name, swath_definition,
filename, data_type, products_created):
product_def = PRODUCTS[product_name]
dep_objects = [
products_created[dep_name] for dep_name in product_def.dependencies
]
filepaths = sorted(
set([
filepath for swath in dep_objects
for filepath in swath["source_filenames"]
]))
s = dep_objects[0]
one_swath = containers.SwathProduct(
product_name=product_name,
description=product_def.description,
units=product_def.units,
satellite=s["satellite"],
instrument=s["instrument"],
begin_time=s["begin_time"],
end_time=s["end_time"],
swath_definition=swath_definition,
fill_value=numpy.nan,
swath_rows=s["swath_rows"],
swath_columns=s["swath_columns"],
data_type=data_type,
swath_data=filename,
source_filenames=filepaths,
data_kind=product_def.data_kind,
rows_per_scan=s["rows_per_scan"])
return one_swath
def create_raw_swath_object(self, product_name, swath_definition):
product_def = PRODUCTS[product_name]
file_reader = self.file_readers[product_def.file_type]
filename = product_name + ".dat"
if os.path.isfile(filename):
if not self.overwrite_existing:
LOG.error("Binary file already exists: %s" % (filename,))
raise RuntimeError("Binary file already exists: %s" % (filename,))
else:
LOG.warning("Binary file already exists, will overwrite: %s", filename)
# TODO: Get the data type from the data or allow the user to specify
try:
shape = file_reader.write_var_to_flat_binary(product_def.file_key, filename)
except StandardError:
LOG.error("Could not extract data from file")
LOG.debug("Extraction exception: ", exc_info=True)
raise
one_swath = containers.SwathProduct(
product_name=product_name, description=product_def.description, units=product_def.units,
satellite=file_reader.satellite, instrument=file_reader.instrument,
begin_time=file_reader.begin_time, end_time=file_reader.end_time,
swath_definition=swath_definition, fill_value=numpy.nan,
swath_rows=shape[0], swath_columns=shape[1], data_type=numpy.float32, swath_data=filename,
source_filenames=file_reader.filepaths, data_kind=product_def.data_kind, rows_per_scan=0
)
return one_swath
def create_raw_swath_object(self, product_name, swath_definition):
product_def = PRODUCTS[product_name]
try:
file_type = product_def.get_file_type(self.available_file_types)
file_key = product_def.get_file_key(self.available_file_types)
except StandardError:
LOG.error(
"Could not create product '%s' because some data files are missing"
% (product_name, ))
raise RuntimeError(
"Could not create product '%s' because some data files are missing"
% (product_name, ))
file_reader = self.file_readers[file_type]
LOG.debug(
"Using file type '%s' and getting file key '%s' for product '%s'",
file_type, file_key, product_name)
LOG.debug("Writing product '%s' data to binary file", product_name)
filename = product_name + ".dat"
if os.path.isfile(filename):
if not self.overwrite_existing:
LOG.error("Binary file already exists: %s" % (filename, ))
raise RuntimeError("Binary file already exists: %s" %
(filename, ))
else:
LOG.warning("Binary file already exists, will overwrite: %s",
filename)
try:
data_type = file_reader.get_data_type(file_key)
fill_value = file_reader.get_fill_value(file_key)
shape = file_reader.write_var_to_flat_binary(file_key,
filename,
dtype=data_type)
rows_per_scan = GEO_PAIRS[product_def.get_geo_pair_name(
self.available_file_types)].rows_per_scan
except StandardError:
LOG.error("Could not extract data from file")
LOG.debug("Extraction exception: ", exc_info=True)
raise
one_swath = containers.SwathProduct(
product_name=product_name,
description=product_def.description,
units=product_def.units,
satellite=file_reader.satellite,
instrument=file_reader.instrument,
begin_time=file_reader.begin_time,
end_time=file_reader.end_time,
swath_definition=swath_definition,
fill_value=fill_value,
swath_rows=shape[0],
swath_columns=shape[1],
data_type=data_type,
swath_data=filename,
source_filenames=file_reader.filepaths,
data_kind=product_def.data_kind,
rows_per_scan=rows_per_scan or shape[0],
)
return one_swath
def create_raw_swath_object(self, product_name, swath_definition):
product_def = self.PRODUCTS[product_name]
file_reader = self.file_readers[product_def.file_type]
filename = product_name + ".dat"
if os.path.isfile(filename):
if not self.overwrite_existing:
LOG.error("Binary file already exists: %s" % (filename, ))
raise RuntimeError("Binary file already exists: %s" %
(filename, ))
else:
LOG.warning("Binary file already exists, will overwrite: %s",
filename)
# TODO: Get the data type from the data or allow the user to specify
try:
shape = file_reader.write_var_to_flat_binary(
product_def.file_key, filename)
except (OSError, ValueError):
LOG.error("Could not extract data from file")
LOG.debug("Extraction exception: ", exc_info=True)
raise
kwargs = {}
if hasattr(product_def, "channel_index"):
kwargs["channel_index"] = product_def.channel_index
# frequencies should be the same for each file (if multiple loaded) so just take the first (0) one
kwargs["frequency"] = file_reader[FREQ_VAR][0][
product_def.channel_index]
elif hasattr(product_def, "frequency"):
channel_index = file_reader.get_channel_index(
product_def.frequency)
kwargs["channel_index"] = channel_index
kwargs["frequency"] = product_def.frequency
one_swath = containers.SwathProduct(
product_name=product_name,
description=product_def.description,
units=product_def.units,
reader='mirs',
satellite=file_reader.satellite,
instrument=file_reader.instrument,
begin_time=file_reader.begin_time,
end_time=file_reader.end_time,
swath_definition=swath_definition,
fill_value=np.nan,
swath_rows=shape[0],
swath_columns=shape[1],
data_type=np.float32,
swath_data=filename,
source_filenames=file_reader.filepaths,
data_kind=product_def.data_kind,
rows_per_scan=shape[0],
**kwargs)
return one_swath
def create_raw_swath_object(self, product_name, swath_definition):
product_def = PRODUCTS[product_name]
# file_type = PRODUCTS.file_type_for_product(product_name, use_terrain_corrected=self.use_terrain_corrected)
# file_key = PRODUCTS.file_key_for_product(product_name, use_terrain_corrected=self.use_terrain_corrected)
# simple version
file_key = product_def.file_key
pressure = getattr(product_def, "pressure", None)
LOG.debug("Getting file key '%s' for product '%s'", file_key,
product_name)
LOG.debug("Writing product '%s' data to binary file", product_name)
filename = product_name + ".dat"
if os.path.isfile(filename):
if not self.overwrite_existing:
LOG.error("Binary file already exists: %s" % (filename, ))
raise RuntimeError("Binary file already exists: %s" %
(filename, ))
else:
LOG.warning("Binary file already exists, will overwrite: %s",
filename)
try:
filename = product_name + ".dat"
shape = _write_var_to_binary_file(filename,
self.file_objects,
file_key,
pressure=pressure)
rows_per_scan = self.rows_per_scan
except OSError:
LOG.error("Could not extract data from file")
LOG.debug("Extraction exception: ", exc_info=True)
raise
one_swath = containers.SwathProduct(
product_name=product_name,
description=product_def.description,
units=product_def.units,
satellite=self.satellite,
instrument=self.instrument,
begin_time=self.begin_time,
end_time=self.end_time,
swath_definition=swath_definition,
fill_value=np.nan,
swath_rows=shape[0],
swath_columns=shape[1],
data_type=np.float32,
swath_data=filename,
source_filenames=self.filepaths,
data_kind=product_def.data_kind,
rows_per_scan=rows_per_scan)
return one_swath
def dataset_to_swath_product(ds, swath_def, overwrite_existing=False):
info = ds.info.copy()
info.pop("area")
if ds.ndim == 3:
# RGB composite
if ds.shape[0] in [3, 4]:
channels = ds.shape[0]
else:
# unpreferred array orientation
channels = ds.shape[-1]
ds = np.rollaxis(ds, 2)
else:
channels = 1
if ds.ndim == 1:
rows, cols = ds.shape[0], 1
else:
rows, cols = ds.shape[-2:]
p2g_metadata = {
"product_name":
info["name"],
"satellite":
info["platform"].lower(),
"instrument":
info["sensor"].lower() if isinstance(info["sensor"], str) else list(
info["sensor"])[0].lower(),
"data_kind":
info["standard_name"],
"begin_time":
info["start_time"],
"end_time":
info["end_time"],
"fill_value":
np.nan,
"swath_columns":
cols,
"swath_rows":
rows,
"rows_per_scan":
info.get("rows_per_scan", rows),
"data_type":
ds.dtype,
"swath_definition":
swath_def,
"channels":
channels,
}
info.update(p2g_metadata)
if channels == 1:
filename = info["name"] + ".dat"
info["swath_data"] = filename
if os.path.isfile(filename):
if not overwrite_existing:
LOG.error("Binary file already exists: %s" % (filename, ))
raise RuntimeError("Binary file already exists: %s" %
(filename, ))
else:
LOG.warning("Binary file already exists, will overwrite: %s",
filename)
p2g_arr = np.memmap(filename,
mode="w+",
dtype=ds.dtype,
shape=ds.shape)
p2g_arr[:] = ds.data
p2g_arr[ds.mask] = np.nan
yield containers.SwathProduct(**info)
else:
for chn_idx in range(channels):
tmp_info = info.copy()
tmp_info["product_name"] = info[
"product_name"] + "_rgb_{:d}".format(chn_idx)
filename = tmp_info["product_name"] + ".dat"
tmp_info["swath_data"] = filename
if os.path.isfile(filename):
if not overwrite_existing:
LOG.error("Binary file already exists: %s" % (filename, ))
raise RuntimeError("Binary file already exists: %s" %
(filename, ))
else:
LOG.warning(
"Binary file already exists, will overwrite: %s",
filename)
p2g_arr = np.memmap(filename,
mode="w+",
dtype=ds.dtype,
shape=ds.shape[-2:])
p2g_arr[:] = ds.data[chn_idx]
p2g_arr[ds.mask[chn_idx]] = np.nan
yield containers.SwathProduct(**tmp_info)
def dataarray_to_swath_product(ds, swath_def, overwrite_existing=False):
info = ds.attrs.copy()
info.pop("area")
if ds.ndim == 3:
# RGB composite
if ds.shape[0] in [3, 4]:
channels = ds.shape[0]
else:
# unpreferred array orientation
channels = ds.shape[-1]
ds = np.rollaxis(ds, 2)
else:
channels = 1
if ds.ndim == 1:
rows, cols = ds.shape[0], 1
else:
rows, cols = ds.shape[-2:]
if np.issubdtype(np.dtype(ds.dtype), np.floating):
dtype = np.float32
default_fill = np.nan
else:
dtype = ds.dtype
# Invalid data is 0
default_fill = info.get('_FillValue', 0)
if isinstance(info["sensor"], bytes):
info["sensor"] = info["sensor"].decode("utf-8")
p2g_metadata = {
"product_name":
info["name"],
"satellite":
normalize_satellite_name(info["platform_name"]),
"instrument":
info["sensor"].lower() if isinstance(info["sensor"], str) else list(
info["sensor"])[0].lower(),
"data_kind":
info.get("standard_name", info['name']),
"begin_time":
info["start_time"],
"end_time":
info["end_time"],
"fill_value":
default_fill,
"swath_columns":
cols,
"swath_rows":
rows,
"rows_per_scan":
info.get("rows_per_scan", rows),
"data_type":
dtype,
"swath_definition":
swath_def,
"channels":
channels,
}
info.update(p2g_metadata)
if channels == 1:
filename = info["name"] + ".dat"
info["swath_data"] = filename
if os.path.isfile(filename):
if not overwrite_existing:
LOG.error("Binary file already exists: %s" % (filename, ))
raise RuntimeError("Binary file already exists: %s" %
(filename, ))
else:
LOG.warning("Binary file already exists, will overwrite: %s",
filename)
LOG.info("Writing band data to disk cache...")
p2g_arr = np.memmap(filename, mode="w+", dtype=dtype, shape=ds.shape)
ds = ds.where(ds.notnull(), np.nan)
da.store(ds.data.astype(dtype), p2g_arr)
yield containers.SwathProduct(**info)
else:
for chn_idx in range(channels):
tmp_info = info.copy()
tmp_info["product_name"] = info[
"product_name"] + "_rgb_{:d}".format(chn_idx)
filename = tmp_info["product_name"] + ".dat"
tmp_info["swath_data"] = filename
if os.path.isfile(filename):
if not overwrite_existing:
LOG.error("Binary file already exists: %s" % (filename, ))
raise RuntimeError("Binary file already exists: %s" %
(filename, ))
else:
LOG.warning(
"Binary file already exists, will overwrite: %s",
filename)
LOG.info("Writing band data to disk cache...")
p2g_arr = np.memmap(filename,
mode="w+",
dtype=dtype,
shape=ds.shape[-2:])
da.store(ds.data[chn_idx].astype(dtype), p2g_arr)
yield containers.SwathProduct(**tmp_info)
