def convert_satpy_to_p2g_swath(frontend, scene):
p2g_scene = containers.SwathScene()
overwrite_existing = frontend.overwrite_existing
areas = {}
for ds in scene:
a = ds.info['area']
area_name = getattr(a, 'name', None)
if area_name is None:
# generate an identifying name
# s = '_'.join(map(str, ds.info['area'].shape))
a.name = area_name = "{}_{}".format(a.lons.info['name'],
a.lats.info['name'])
if area_name in areas:
swath_def = areas[area_name]
else:
areas[area_name] = swath_def = area_to_swath_def(
ds.info["area"], overwrite_existing=overwrite_existing)
def_rps = ds.shape[0] if ds.ndim <= 2 else ds.shape[-2]
swath_def.setdefault("rows_per_scan",
ds.info.get("rows_per_scan", def_rps))
for swath_product in dataset_to_swath_product(
ds, swath_def, overwrite_existing=overwrite_existing):
p2g_scene[swath_product["product_name"]] = swath_product
return p2g_scene
def convert_satpy_to_p2g_swath(frontend, scene, convert_area_defs=True):
"""Convert a Satpy Scene in to a Polar2Grid SwathScene.
If ``convert_area_defs`` is ``True`` (default) then `AreaDefinition`
objects will be converted to `SwathDefinition` objects by accessing
their longitude and latitude arrays. If ``False`` then an exception
is raised when an `AreaDefinition` is encountered.
"""
p2g_scene = containers.SwathScene()
overwrite_existing = frontend.overwrite_existing
areas = {}
for ds in scene:
a = ds.attrs['area']
area_name = getattr(a, 'name', getattr(a, 'description', None))
if area_name is None:
# generate an identifying name
a.name = area_name = "{}_{}".format(a.lons.attrs['name'],
a.lats.attrs['name'])
if area_name in areas:
swath_def = areas[area_name]
elif isinstance(a, AreaDefinition) and not convert_area_defs:
raise ValueError(
"AreaDefinition found in SwathScene, will not convert to swath"
)
else:
areas[area_name] = swath_def = area_to_swath_def(
ds.attrs["area"],
chunks=ds.data.chunks,
overwrite_existing=overwrite_existing)
def_rps = ds.shape[0] if ds.ndim <= 2 else ds.shape[-2]
swath_def.setdefault("rows_per_scan",
ds.attrs.get("rows_per_scan", def_rps))
for swath_product in dataarray_to_swath_product(
ds, swath_def, overwrite_existing=overwrite_existing):
swath_product.setdefault('reader', frontend.reader)
p2g_scene[swath_product["product_name"]] = swath_product
return p2g_scene
def create_scene(self, products=None, nprocs=1, **kwargs):
if nprocs != 1:
raise NotImplementedError("The MIRS frontend does not support multiple processes yet")
if products is None:
LOG.debug("No products specified to frontend, will try to load logical defaults")
products = self.default_products
# Do we actually have all of the files needed to create the requested products?
products = self.loadable_products(products)
# Needs to be ordered (least-depended product -> most-depended product)
products_needed = PRODUCTS.dependency_ordered_products(products)
geo_pairs_needed = PRODUCTS.geo_pairs_for_products(products_needed, self.available_file_types)
# both lists below include raw products that need extra processing/masking
raw_products_needed = (p for p in products_needed if PRODUCTS.is_raw(p, geo_is_raw=True))
secondary_products_needed = [p for p in products_needed if PRODUCTS.needs_processing(p)]
for p in secondary_products_needed:
if p not in self.secondary_product_functions:
msg = "Product (secondary or extra processing) required, but not sure how to make it: '%s'" % (p,)
LOG.error(msg)
raise ValueError(msg)
LOG.debug("Extracting data to create the following products:\n\t%s", "\n\t".join(products))
# final scene object we'll be providing to the caller
scene = containers.SwathScene()
# Dictionary of all products created so far (local variable so we don't hold on to any product objects)
products_created = {}
swath_definitions = {}
# Load geographic products - every product needs a geo-product
for geo_pair_name in geo_pairs_needed:
lon_product_name = GEO_PAIRS[geo_pair_name].lon_product
lat_product_name = GEO_PAIRS[geo_pair_name].lat_product
# longitude
if lon_product_name not in products_created:
one_lon_swath = self.create_raw_swath_object(lon_product_name, None)
products_created[lon_product_name] = one_lon_swath
if lon_product_name in products:
# only process the geolocation product if the user requested it that way
scene[lon_product_name] = one_lon_swath
else:
one_lon_swath = products_created[lon_product_name]
# latitude
if lat_product_name not in products_created:
one_lat_swath = self.create_raw_swath_object(lat_product_name, None)
products_created[lat_product_name] = one_lat_swath
if lat_product_name in products:
# only process the geolocation product if the user requested it that way
scene[lat_product_name] = one_lat_swath
else:
one_lat_swath = products_created[lat_product_name]
swath_definitions[geo_pair_name] = self.create_swath_definition(one_lon_swath, one_lat_swath)
# Create each raw products (products that are loaded directly from the file)
for product_name in raw_products_needed:
if product_name in products_created:
# already created
continue
try:
LOG.info("Creating data product '%s'", product_name)
swath_def = swath_definitions[PRODUCTS[product_name].get_geo_pair_name(self.available_file_types)]
one_swath = products_created[product_name] = self.create_raw_swath_object(product_name, swath_def)
except StandardError:
LOG.error("Could not create raw product '%s'", product_name)
if self.exit_on_error:
raise
continue
if product_name in products:
# the user wants this product
scene[product_name] = one_swath
return scene
def create_scene(self, products=None, **kwargs):
LOG.debug("Loading scene data...")
# If the user didn't provide the products they want, figure out which ones we can create
if products is None:
LOG.debug(
"No products specified to frontend, will try to load logical defaults products"
)
products = self.default_products
# Do we actually have all of the files needed to create the requested products?
products = self.loadable_products(products)
# Needs to be ordered (least-depended product -> most-depended product)
products_needed = PRODUCTS.dependency_ordered_products(products)
geo_pairs_needed = PRODUCTS.geo_pairs_for_products(products_needed)
# both lists below include raw products that need extra processing/masking
raw_products_needed = (p for p in products_needed
if PRODUCTS.is_raw(p, geo_is_raw=False))
secondary_products_needed = [
p for p in products_needed if PRODUCTS.needs_processing(p)
]
for p in secondary_products_needed:
if p not in self.secondary_product_functions:
LOG.error(
"Product (secondary or extra processing) required, but not sure how to make it: '%s'",
p)
raise ValueError(
"Product (secondary or extra processing) required, but not sure how to make it: '%s'"
% (p, ))
# final scene object we'll be providing to the caller
scene = containers.SwathScene()
# Dictionary of all products created so far (local variable so we don't hold on to any product objects)
products_created = {}
swath_definitions = {}
# Load geolocation files
for geo_pair_name in geo_pairs_needed:
### Lon Product ###
lon_product_name = GEO_PAIRS[geo_pair_name].lon_product
LOG.info("Creating navigation product '%s'", lon_product_name)
lon_swath = products_created[
lon_product_name] = self.create_raw_swath_object(
lon_product_name, None)
if lon_product_name in products:
scene[lon_product_name] = lon_swath
### Lat Product ###
lat_product_name = GEO_PAIRS[geo_pair_name].lat_product
LOG.info("Creating navigation product '%s'", lat_product_name)
lat_swath = products_created[
lat_product_name] = self.create_raw_swath_object(
lat_product_name, None)
if lat_product_name in products:
scene[lat_product_name] = lat_swath
# Create the SwathDefinition
swath_def = self.create_swath_definition(lon_swath, lat_swath)
swath_definitions[swath_def["swath_name"]] = swath_def
# Create each raw products (products that are loaded directly from the file)
for product_name in raw_products_needed:
if product_name in products_created:
# already created
continue
try:
LOG.info("Creating data product '%s'", product_name)
swath_def = swath_definitions[
PRODUCTS[product_name].geo_pair_name]
one_swath = products_created[
product_name] = self.create_raw_swath_object(
product_name, swath_def)
except StandardError:
LOG.error("Could not create raw product '%s'", product_name)
if self.exit_on_error:
raise
continue
if product_name in products:
# the user wants this product
scene[product_name] = one_swath
# Dependent products and Special cases (i.e. non-raw products that need further processing)
for product_name in reversed(secondary_products_needed):
product_func = self.secondary_product_functions[product_name]
swath_def = swath_definitions[PRODUCTS[product_name].geo_pair_name]
try:
LOG.info("Creating secondary product '%s'", product_name)
one_swath = product_func(self, product_name, swath_def,
products_created)
except StandardError:
LOG.error("Could not create product (unexpected error): '%s'",
product_name)
LOG.debug("Could not create product (unexpected error): '%s'",
product_name,
exc_info=True)
if self.exit_on_error:
raise
continue
products_created[product_name] = one_swath
if product_name in products:
# the user wants this product
scene[product_name] = one_swath
return scene
def create_scene(self,
products=None,
nprocs=1,
all_bt_channels=False,
**kwargs):
if nprocs != 1:
raise NotImplementedError(
"The MIRS frontend does not support multiple processes yet")
if products is None:
if not all_bt_channels:
LOG.debug(
"No products specified to frontend, will try to load logical defaults"
)
products = self.default_products
else:
products = []
if all_bt_channels:
products.extend([
x for x in self.PRODUCTS.keys()
if x.startswith('btemp_') and x != PRODUCT_BT_CHANS
])
products = list(set(products))
# Do we actually have all of the files needed to create the requested products?
products = self.loadable_products(products)
# Needs to be ordered (least-depended product -> most-depended product)
products_needed = self.PRODUCTS.dependency_ordered_products(products)
geo_pairs_needed = self.PRODUCTS.geo_pairs_for_products(
products_needed, self.available_file_types)
# both lists below include raw products that need extra processing/masking
raw_products_needed = (p for p in products_needed
if self.PRODUCTS.is_raw(p, geo_is_raw=True))
secondary_products_needed = [
p for p in products_needed if self.PRODUCTS.needs_processing(p)
]
for p in secondary_products_needed:
if p not in self.secondary_product_functions:
msg = "Product (secondary or extra processing) required, but not sure how to make it: '%s'" % (
p, )
LOG.error(msg)
raise ValueError(msg)
LOG.debug("Extracting data to create the following products:\n\t%s",
"\n\t".join(products))
# final scene object we'll be providing to the caller
scene = containers.SwathScene()
# Dictionary of all products created so far (local variable so we don't hold on to any product objects)
products_created = {}
swath_definitions = {}
# Load geographic products - every product needs a geo-product
for geo_pair_name in geo_pairs_needed:
lon_product_name = self.GEO_PAIRS[geo_pair_name].lon_product
lat_product_name = self.GEO_PAIRS[geo_pair_name].lat_product
# longitude
if lon_product_name not in products_created:
one_lon_swath = self.create_raw_swath_object(
lon_product_name, None)
products_created[lon_product_name] = one_lon_swath
if lon_product_name in products:
# only process the geolocation product if the user requested it that way
scene[lon_product_name] = one_lon_swath
else:
one_lon_swath = products_created[lon_product_name]
# latitude
if lat_product_name not in products_created:
one_lat_swath = self.create_raw_swath_object(
lat_product_name, None)
products_created[lat_product_name] = one_lat_swath
if lat_product_name in products:
# only process the geolocation product if the user requested it that way
scene[lat_product_name] = one_lat_swath
else:
one_lat_swath = products_created[lat_product_name]
swath_definitions[geo_pair_name] = self.create_swath_definition(
one_lon_swath, one_lat_swath)
# Create each raw products (products that are loaded directly from the file)
for product_name in raw_products_needed:
if product_name in products_created:
# already created
continue
try:
LOG.info("Creating data product '%s'", product_name)
swath_def = swath_definitions[
self.PRODUCTS[product_name].get_geo_pair_name(
self.available_file_types)]
one_swath = products_created[
product_name] = self.create_raw_swath_object(
product_name, swath_def)
except (ValueError, OSError):
LOG.error("Could not create raw product '%s'", product_name)
LOG.debug("Debug: ", exc_info=True)
if self.exit_on_error:
raise
continue
if product_name in products:
# the user wants this product
scene[product_name] = one_swath
# Dependent products and Special cases (i.e. non-raw products that need further processing)
for product_name in reversed(secondary_products_needed):
product_func = self.secondary_product_functions[product_name]
swath_def = swath_definitions[
self.PRODUCTS[product_name].geo_pair_name]
try:
LOG.info("Creating secondary product '%s'", product_name)
one_swath = product_func(product_name, swath_def,
products_created)
except (ValueError, OSError, KeyError):
LOG.error("Could not create product (unexpected error): '%s'",
product_name)
LOG.debug("Could not create product (unexpected error): '%s'",
product_name,
exc_info=True)
if self.exit_on_error:
raise
continue
if one_swath is None:
LOG.debug(
"Secondary product function did not produce a swath product"
)
if product_name in scene:
LOG.debug(
"Removing original swath that was created before")
del scene[product_name]
continue
products_created[product_name] = one_swath
if product_name in products:
# the user wants this product
scene[product_name] = one_swath
return scene
def create_scene(self, products=None, nprocs=1, **kwargs):
if nprocs != 1:
raise NotImplementedError(
"The ACSPO frontend does not support multiple processes yet")
if products is None:
products = self.available_product_names
orig_products = set(products)
available_products = self.available_product_names
doable_products = orig_products & set(available_products)
for p in (orig_products - doable_products):
LOG.warning("Missing proper data files to create product: %s", p)
products = list(doable_products)
if not products:
LOG.debug("Original Products:\n\t%r", orig_products)
LOG.debug("Available Products:\n\t%r", available_products)
LOG.debug("Doable (final) Products:\n\t%r", products)
LOG.error(
"Can not create any of the requested products (missing required data files)"
)
raise RuntimeError(
"Can not create any of the requested products (missing required data files)"
)
LOG.debug("Extracting data to create the following products:\n\t%s",
"\n\t".join(products))
scene = containers.SwathScene()
# Figure out any dependencies
raw_products = []
for product_name in products:
if product_name not in PRODUCTS:
LOG.error("Unknown product name: %s", product_name)
raise ValueError("Unknown product name: %s" % (product_name, ))
if PRODUCTS[product_name].dependencies:
raise NotImplementedError(
"Don't know how to handle products dependent on other products"
)
raw_products.append(product_name)
# Load geographic products - every product needs a geo-product
products_created = {}
swath_definitions = {}
for geo_product_pair in GEO_PAIRS:
lon_product_name, lat_product_name = geo_product_pair
# longitude
if lon_product_name not in products_created:
one_lon_swath = self.create_raw_swath_object(
lon_product_name, None)
products_created[lon_product_name] = one_lon_swath
if lon_product_name in raw_products:
# only process the geolocation product if the user requested it that way
scene[lon_product_name] = one_lon_swath
else:
one_lon_swath = products_created[lon_product_name]
# latitude
if lat_product_name not in products_created:
one_lat_swath = self.create_raw_swath_object(
lat_product_name, None)
products_created[lat_product_name] = one_lat_swath
if lat_product_name in raw_products:
# only process the geolocation product if the user requested it that way
scene[lat_product_name] = one_lat_swath
else:
one_lat_swath = products_created[lat_product_name]
swath_definitions[geo_product_pair] = self.create_swath_definition(
one_lon_swath, one_lat_swath)
# Load raw products
for raw_product in raw_products:
# FUTURE: Get this info from the product definition
geo_pair = GEO_PAIRS[0]
if raw_product not in products_created:
try:
one_lat_swath = self.create_raw_swath_object(
raw_product, swath_definitions[geo_pair])
products_created[raw_product] = one_lat_swath
scene[raw_product] = one_lat_swath
except StandardError:
LOG.error("Could not create raw product '%s'", raw_product)
if self.exit_on_error:
raise
continue
return scene
def create_scene(self, products=None, **kwargs):
LOG.debug("Loading scene data...")
# If the user didn't provide the products they want, figure out which ones we can create
if products is None:
LOG.debug(
"No products specified to frontend, will try to load logical defaults products"
)
products = self.default_products
# Do we actually have all of the files needed to create the requested products?
products = self.loadable_products(products)
# Needs to be ordered (least-depended product -> most-depended product)
products_needed = PRODUCTS.dependency_ordered_products(products)
# all of our current products are raw (other frontends have secondary products, but not us)
raw_products_needed = products_needed
# final scene object we'll be providing to the caller
scene = containers.SwathScene()
# Dictionary of all products created so far (local variable so we don't hold on to any product objects)
products_created = {}
# Load geolocation files
# normally there are multiple resolutions provided by a frontend, but we only need one
# swath_definitions = {}
geo_pair_name = BASE_PAIR
### Lon Product ###
lon_product_name = GEO_PAIRS[geo_pair_name].lon_product
LOG.info("Creating navigation product '%s'", lon_product_name)
lon_swath = products_created[
lon_product_name] = self.create_raw_swath_object(
lon_product_name, None)
if lon_product_name in products:
scene[lon_product_name] = lon_swath
### Lat Product ###
lat_product_name = GEO_PAIRS[geo_pair_name].lat_product
LOG.info("Creating navigation product '%s'", lat_product_name)
lat_swath = products_created[
lat_product_name] = self.create_raw_swath_object(
lat_product_name, None)
if lat_product_name in products:
scene[lat_product_name] = lat_swath
# Create the SwathDefinition
swath_def = self.create_swath_definition(lon_swath, lat_swath)
# swath_definitions[swath_def["swath_name"]] = swath_def
# Create each raw products (products that are loaded directly from the file)
for product_name in raw_products_needed:
if product_name in products_created:
# already created
continue
try:
LOG.info("Creating data product '%s'", product_name)
one_swath = products_created[
product_name] = self.create_raw_swath_object(
product_name, swath_def)
except (RuntimeError, ValueError, OSError):
LOG.error("Could not create raw product '%s'", product_name)
if self.exit_on_error:
raise
continue
if product_name in products:
# the user wants this product
scene[product_name] = one_swath
return scene
