def convert_satpy_to_p2g_swath(frontend, scene): p2g_scene = containers.SwathScene() overwrite_existing = frontend.overwrite_existing areas = {} for ds in scene: a = ds.info['area'] area_name = getattr(a, 'name', None) if area_name is None: # generate an identifying name # s = '_'.join(map(str, ds.info['area'].shape)) a.name = area_name = "{}_{}".format(a.lons.info['name'], a.lats.info['name']) if area_name in areas: swath_def = areas[area_name] else: areas[area_name] = swath_def = area_to_swath_def( ds.info["area"], overwrite_existing=overwrite_existing) def_rps = ds.shape[0] if ds.ndim <= 2 else ds.shape[-2] swath_def.setdefault("rows_per_scan", ds.info.get("rows_per_scan", def_rps)) for swath_product in dataset_to_swath_product( ds, swath_def, overwrite_existing=overwrite_existing): p2g_scene[swath_product["product_name"]] = swath_product return p2g_scene
def convert_satpy_to_p2g_swath(frontend, scene, convert_area_defs=True): """Convert a Satpy Scene in to a Polar2Grid SwathScene. If ``convert_area_defs`` is ``True`` (default) then `AreaDefinition` objects will be converted to `SwathDefinition` objects by accessing their longitude and latitude arrays. If ``False`` then an exception is raised when an `AreaDefinition` is encountered. """ p2g_scene = containers.SwathScene() overwrite_existing = frontend.overwrite_existing areas = {} for ds in scene: a = ds.attrs['area'] area_name = getattr(a, 'name', getattr(a, 'description', None)) if area_name is None: # generate an identifying name a.name = area_name = "{}_{}".format(a.lons.attrs['name'], a.lats.attrs['name']) if area_name in areas: swath_def = areas[area_name] elif isinstance(a, AreaDefinition) and not convert_area_defs: raise ValueError( "AreaDefinition found in SwathScene, will not convert to swath" ) else: areas[area_name] = swath_def = area_to_swath_def( ds.attrs["area"], chunks=ds.data.chunks, overwrite_existing=overwrite_existing) def_rps = ds.shape[0] if ds.ndim <= 2 else ds.shape[-2] swath_def.setdefault("rows_per_scan", ds.attrs.get("rows_per_scan", def_rps)) for swath_product in dataarray_to_swath_product( ds, swath_def, overwrite_existing=overwrite_existing): swath_product.setdefault('reader', frontend.reader) p2g_scene[swath_product["product_name"]] = swath_product return p2g_scene
def create_scene(self, products=None, nprocs=1, **kwargs): if nprocs != 1: raise NotImplementedError("The MIRS frontend does not support multiple processes yet") if products is None: LOG.debug("No products specified to frontend, will try to load logical defaults") products = self.default_products # Do we actually have all of the files needed to create the requested products? products = self.loadable_products(products) # Needs to be ordered (least-depended product -> most-depended product) products_needed = PRODUCTS.dependency_ordered_products(products) geo_pairs_needed = PRODUCTS.geo_pairs_for_products(products_needed, self.available_file_types) # both lists below include raw products that need extra processing/masking raw_products_needed = (p for p in products_needed if PRODUCTS.is_raw(p, geo_is_raw=True)) secondary_products_needed = [p for p in products_needed if PRODUCTS.needs_processing(p)] for p in secondary_products_needed: if p not in self.secondary_product_functions: msg = "Product (secondary or extra processing) required, but not sure how to make it: '%s'" % (p,) LOG.error(msg) raise ValueError(msg) LOG.debug("Extracting data to create the following products:\n\t%s", "\n\t".join(products)) # final scene object we'll be providing to the caller scene = containers.SwathScene() # Dictionary of all products created so far (local variable so we don't hold on to any product objects) products_created = {} swath_definitions = {} # Load geographic products - every product needs a geo-product for geo_pair_name in geo_pairs_needed: lon_product_name = GEO_PAIRS[geo_pair_name].lon_product lat_product_name = GEO_PAIRS[geo_pair_name].lat_product # longitude if lon_product_name not in products_created: one_lon_swath = self.create_raw_swath_object(lon_product_name, None) products_created[lon_product_name] = one_lon_swath if lon_product_name in products: # only process the geolocation product if the user requested it that way scene[lon_product_name] = one_lon_swath else: one_lon_swath = products_created[lon_product_name] # latitude if lat_product_name not in products_created: one_lat_swath = self.create_raw_swath_object(lat_product_name, None) products_created[lat_product_name] = one_lat_swath if lat_product_name in products: # only process the geolocation product if the user requested it that way scene[lat_product_name] = one_lat_swath else: one_lat_swath = products_created[lat_product_name] swath_definitions[geo_pair_name] = self.create_swath_definition(one_lon_swath, one_lat_swath) # Create each raw products (products that are loaded directly from the file) for product_name in raw_products_needed: if product_name in products_created: # already created continue try: LOG.info("Creating data product '%s'", product_name) swath_def = swath_definitions[PRODUCTS[product_name].get_geo_pair_name(self.available_file_types)] one_swath = products_created[product_name] = self.create_raw_swath_object(product_name, swath_def) except StandardError: LOG.error("Could not create raw product '%s'", product_name) if self.exit_on_error: raise continue if product_name in products: # the user wants this product scene[product_name] = one_swath return scene
def create_scene(self, products=None, **kwargs): LOG.debug("Loading scene data...") # If the user didn't provide the products they want, figure out which ones we can create if products is None: LOG.debug( "No products specified to frontend, will try to load logical defaults products" ) products = self.default_products # Do we actually have all of the files needed to create the requested products? products = self.loadable_products(products) # Needs to be ordered (least-depended product -> most-depended product) products_needed = PRODUCTS.dependency_ordered_products(products) geo_pairs_needed = PRODUCTS.geo_pairs_for_products(products_needed) # both lists below include raw products that need extra processing/masking raw_products_needed = (p for p in products_needed if PRODUCTS.is_raw(p, geo_is_raw=False)) secondary_products_needed = [ p for p in products_needed if PRODUCTS.needs_processing(p) ] for p in secondary_products_needed: if p not in self.secondary_product_functions: LOG.error( "Product (secondary or extra processing) required, but not sure how to make it: '%s'", p) raise ValueError( "Product (secondary or extra processing) required, but not sure how to make it: '%s'" % (p, )) # final scene object we'll be providing to the caller scene = containers.SwathScene() # Dictionary of all products created so far (local variable so we don't hold on to any product objects) products_created = {} swath_definitions = {} # Load geolocation files for geo_pair_name in geo_pairs_needed: ### Lon Product ### lon_product_name = GEO_PAIRS[geo_pair_name].lon_product LOG.info("Creating navigation product '%s'", lon_product_name) lon_swath = products_created[ lon_product_name] = self.create_raw_swath_object( lon_product_name, None) if lon_product_name in products: scene[lon_product_name] = lon_swath ### Lat Product ### lat_product_name = GEO_PAIRS[geo_pair_name].lat_product LOG.info("Creating navigation product '%s'", lat_product_name) lat_swath = products_created[ lat_product_name] = self.create_raw_swath_object( lat_product_name, None) if lat_product_name in products: scene[lat_product_name] = lat_swath # Create the SwathDefinition swath_def = self.create_swath_definition(lon_swath, lat_swath) swath_definitions[swath_def["swath_name"]] = swath_def # Create each raw products (products that are loaded directly from the file) for product_name in raw_products_needed: if product_name in products_created: # already created continue try: LOG.info("Creating data product '%s'", product_name) swath_def = swath_definitions[ PRODUCTS[product_name].geo_pair_name] one_swath = products_created[ product_name] = self.create_raw_swath_object( product_name, swath_def) except StandardError: LOG.error("Could not create raw product '%s'", product_name) if self.exit_on_error: raise continue if product_name in products: # the user wants this product scene[product_name] = one_swath # Dependent products and Special cases (i.e. non-raw products that need further processing) for product_name in reversed(secondary_products_needed): product_func = self.secondary_product_functions[product_name] swath_def = swath_definitions[PRODUCTS[product_name].geo_pair_name] try: LOG.info("Creating secondary product '%s'", product_name) one_swath = product_func(self, product_name, swath_def, products_created) except StandardError: LOG.error("Could not create product (unexpected error): '%s'", product_name) LOG.debug("Could not create product (unexpected error): '%s'", product_name, exc_info=True) if self.exit_on_error: raise continue products_created[product_name] = one_swath if product_name in products: # the user wants this product scene[product_name] = one_swath return scene
def create_scene(self, products=None, nprocs=1, all_bt_channels=False, **kwargs): if nprocs != 1: raise NotImplementedError( "The MIRS frontend does not support multiple processes yet") if products is None: if not all_bt_channels: LOG.debug( "No products specified to frontend, will try to load logical defaults" ) products = self.default_products else: products = [] if all_bt_channels: products.extend([ x for x in self.PRODUCTS.keys() if x.startswith('btemp_') and x != PRODUCT_BT_CHANS ]) products = list(set(products)) # Do we actually have all of the files needed to create the requested products? products = self.loadable_products(products) # Needs to be ordered (least-depended product -> most-depended product) products_needed = self.PRODUCTS.dependency_ordered_products(products) geo_pairs_needed = self.PRODUCTS.geo_pairs_for_products( products_needed, self.available_file_types) # both lists below include raw products that need extra processing/masking raw_products_needed = (p for p in products_needed if self.PRODUCTS.is_raw(p, geo_is_raw=True)) secondary_products_needed = [ p for p in products_needed if self.PRODUCTS.needs_processing(p) ] for p in secondary_products_needed: if p not in self.secondary_product_functions: msg = "Product (secondary or extra processing) required, but not sure how to make it: '%s'" % ( p, ) LOG.error(msg) raise ValueError(msg) LOG.debug("Extracting data to create the following products:\n\t%s", "\n\t".join(products)) # final scene object we'll be providing to the caller scene = containers.SwathScene() # Dictionary of all products created so far (local variable so we don't hold on to any product objects) products_created = {} swath_definitions = {} # Load geographic products - every product needs a geo-product for geo_pair_name in geo_pairs_needed: lon_product_name = self.GEO_PAIRS[geo_pair_name].lon_product lat_product_name = self.GEO_PAIRS[geo_pair_name].lat_product # longitude if lon_product_name not in products_created: one_lon_swath = self.create_raw_swath_object( lon_product_name, None) products_created[lon_product_name] = one_lon_swath if lon_product_name in products: # only process the geolocation product if the user requested it that way scene[lon_product_name] = one_lon_swath else: one_lon_swath = products_created[lon_product_name] # latitude if lat_product_name not in products_created: one_lat_swath = self.create_raw_swath_object( lat_product_name, None) products_created[lat_product_name] = one_lat_swath if lat_product_name in products: # only process the geolocation product if the user requested it that way scene[lat_product_name] = one_lat_swath else: one_lat_swath = products_created[lat_product_name] swath_definitions[geo_pair_name] = self.create_swath_definition( one_lon_swath, one_lat_swath) # Create each raw products (products that are loaded directly from the file) for product_name in raw_products_needed: if product_name in products_created: # already created continue try: LOG.info("Creating data product '%s'", product_name) swath_def = swath_definitions[ self.PRODUCTS[product_name].get_geo_pair_name( self.available_file_types)] one_swath = products_created[ product_name] = self.create_raw_swath_object( product_name, swath_def) except (ValueError, OSError): LOG.error("Could not create raw product '%s'", product_name) LOG.debug("Debug: ", exc_info=True) if self.exit_on_error: raise continue if product_name in products: # the user wants this product scene[product_name] = one_swath # Dependent products and Special cases (i.e. non-raw products that need further processing) for product_name in reversed(secondary_products_needed): product_func = self.secondary_product_functions[product_name] swath_def = swath_definitions[ self.PRODUCTS[product_name].geo_pair_name] try: LOG.info("Creating secondary product '%s'", product_name) one_swath = product_func(product_name, swath_def, products_created) except (ValueError, OSError, KeyError): LOG.error("Could not create product (unexpected error): '%s'", product_name) LOG.debug("Could not create product (unexpected error): '%s'", product_name, exc_info=True) if self.exit_on_error: raise continue if one_swath is None: LOG.debug( "Secondary product function did not produce a swath product" ) if product_name in scene: LOG.debug( "Removing original swath that was created before") del scene[product_name] continue products_created[product_name] = one_swath if product_name in products: # the user wants this product scene[product_name] = one_swath return scene
def create_scene(self, products=None, nprocs=1, **kwargs): if nprocs != 1: raise NotImplementedError( "The ACSPO frontend does not support multiple processes yet") if products is None: products = self.available_product_names orig_products = set(products) available_products = self.available_product_names doable_products = orig_products & set(available_products) for p in (orig_products - doable_products): LOG.warning("Missing proper data files to create product: %s", p) products = list(doable_products) if not products: LOG.debug("Original Products:\n\t%r", orig_products) LOG.debug("Available Products:\n\t%r", available_products) LOG.debug("Doable (final) Products:\n\t%r", products) LOG.error( "Can not create any of the requested products (missing required data files)" ) raise RuntimeError( "Can not create any of the requested products (missing required data files)" ) LOG.debug("Extracting data to create the following products:\n\t%s", "\n\t".join(products)) scene = containers.SwathScene() # Figure out any dependencies raw_products = [] for product_name in products: if product_name not in PRODUCTS: LOG.error("Unknown product name: %s", product_name) raise ValueError("Unknown product name: %s" % (product_name, )) if PRODUCTS[product_name].dependencies: raise NotImplementedError( "Don't know how to handle products dependent on other products" ) raw_products.append(product_name) # Load geographic products - every product needs a geo-product products_created = {} swath_definitions = {} for geo_product_pair in GEO_PAIRS: lon_product_name, lat_product_name = geo_product_pair # longitude if lon_product_name not in products_created: one_lon_swath = self.create_raw_swath_object( lon_product_name, None) products_created[lon_product_name] = one_lon_swath if lon_product_name in raw_products: # only process the geolocation product if the user requested it that way scene[lon_product_name] = one_lon_swath else: one_lon_swath = products_created[lon_product_name] # latitude if lat_product_name not in products_created: one_lat_swath = self.create_raw_swath_object( lat_product_name, None) products_created[lat_product_name] = one_lat_swath if lat_product_name in raw_products: # only process the geolocation product if the user requested it that way scene[lat_product_name] = one_lat_swath else: one_lat_swath = products_created[lat_product_name] swath_definitions[geo_product_pair] = self.create_swath_definition( one_lon_swath, one_lat_swath) # Load raw products for raw_product in raw_products: # FUTURE: Get this info from the product definition geo_pair = GEO_PAIRS[0] if raw_product not in products_created: try: one_lat_swath = self.create_raw_swath_object( raw_product, swath_definitions[geo_pair]) products_created[raw_product] = one_lat_swath scene[raw_product] = one_lat_swath except StandardError: LOG.error("Could not create raw product '%s'", raw_product) if self.exit_on_error: raise continue return scene
def create_scene(self, products=None, **kwargs): LOG.debug("Loading scene data...") # If the user didn't provide the products they want, figure out which ones we can create if products is None: LOG.debug( "No products specified to frontend, will try to load logical defaults products" ) products = self.default_products # Do we actually have all of the files needed to create the requested products? products = self.loadable_products(products) # Needs to be ordered (least-depended product -> most-depended product) products_needed = PRODUCTS.dependency_ordered_products(products) # all of our current products are raw (other frontends have secondary products, but not us) raw_products_needed = products_needed # final scene object we'll be providing to the caller scene = containers.SwathScene() # Dictionary of all products created so far (local variable so we don't hold on to any product objects) products_created = {} # Load geolocation files # normally there are multiple resolutions provided by a frontend, but we only need one # swath_definitions = {} geo_pair_name = BASE_PAIR ### Lon Product ### lon_product_name = GEO_PAIRS[geo_pair_name].lon_product LOG.info("Creating navigation product '%s'", lon_product_name) lon_swath = products_created[ lon_product_name] = self.create_raw_swath_object( lon_product_name, None) if lon_product_name in products: scene[lon_product_name] = lon_swath ### Lat Product ### lat_product_name = GEO_PAIRS[geo_pair_name].lat_product LOG.info("Creating navigation product '%s'", lat_product_name) lat_swath = products_created[ lat_product_name] = self.create_raw_swath_object( lat_product_name, None) if lat_product_name in products: scene[lat_product_name] = lat_swath # Create the SwathDefinition swath_def = self.create_swath_definition(lon_swath, lat_swath) # swath_definitions[swath_def["swath_name"]] = swath_def # Create each raw products (products that are loaded directly from the file) for product_name in raw_products_needed: if product_name in products_created: # already created continue try: LOG.info("Creating data product '%s'", product_name) one_swath = products_created[ product_name] = self.create_raw_swath_object( product_name, swath_def) except (RuntimeError, ValueError, OSError): LOG.error("Could not create raw product '%s'", product_name) if self.exit_on_error: raise continue if product_name in products: # the user wants this product scene[product_name] = one_swath return scene