class ReaderWrapper(roles.FrontendRole):
FILE_EXTENSIONS = []
DEFAULT_READER_NAME = None
DEFAULT_DATASETS = []
# This is temporary until a better solution is found for loading start/end time on init
PRIMARY_FILE_TYPE = None
def __init__(self, **kwargs):
self.reader = kwargs.pop("reader", self.DEFAULT_READER_NAME)
super(ReaderWrapper, self).__init__(**kwargs)
pathnames = self.find_files_with_extensions()
# Create a satpy Scene object
self.scene = Scene(reader=self.reader, filenames=pathnames, reader_kwargs=kwargs)
self._begin_time = self.scene.start_time
self._end_time = self.scene.end_time
self.wishlist = set()
@property
def begin_time(self):
return self._begin_time
@property
def end_time(self):
return self._end_time
@property
def available_product_names(self):
return self.scene.available_dataset_names(reader_name=self.reader, composites=True)
@property
def all_product_names(self):
return self.scene.all_dataset_names(reader_name=self.reader, composites=True)
@property
def default_products(self):
return self.DEFAULT_DATASETS
def filter(self, scene):
pass
def create_scene(self, products=None, **kwargs):
LOG.debug("Loading scene data...")
# If the user didn't provide the products they want, figure out which ones we can create
if products is None:
LOG.debug("No products specified to frontend, will try to load logical defaults products")
products = self.default_products
kwargs.pop("overwrite_existing")
kwargs.pop("exit_on_error")
kwargs.pop("keep_intermediate")
self.scene.load(products, **kwargs)
self.wishlist = self.scene.wishlist
# Apply Filters
self.filter(self.scene)
# Delete the satpy scene so memory is cleared out if it isn't used by the caller
scene = self.scene
self.scene = None
return scene
class ReaderWrapper(roles.FrontendRole):
FILE_EXTENSIONS = []
DEFAULT_READER_NAME = None
DEFAULT_DATASETS = []
# This is temporary until a better solution is found for loading start/end time on init
PRIMARY_FILE_TYPE = None
def __init__(self, **kwargs):
self.reader = kwargs.pop("reader", self.DEFAULT_READER_NAME)
super(ReaderWrapper, self).__init__(**kwargs)
pathnames = self.find_files_with_extensions()
# Create a satpy Scene object
self.scene = Scene(reader=self.reader, filenames=pathnames)
self._begin_time = self.scene.start_time
self._end_time = self.scene.end_time
@property
def begin_time(self):
return self._begin_time
@property
def end_time(self):
return self._end_time
@property
def available_product_names(self):
return self.scene.available_dataset_names(reader_name=self.reader,
composites=True)
@property
def all_product_names(self):
return self.scene.all_dataset_names(reader_name=self.reader,
composites=True)
@property
def default_products(self):
return self.DEFAULT_DATASETS
def filter(self, scene):
pass
def create_scene(self, products=None, **kwargs):
LOG.debug("Loading scene data...")
# If the user didn't provide the products they want, figure out which ones we can create
if products is None:
LOG.debug(
"No products specified to frontend, will try to load logical defaults products"
)
products = self.default_products
kwargs.pop("overwrite_existing")
kwargs.pop("exit_on_error")
kwargs.pop("keep_intermediate")
self.scene.load(products, **kwargs)
self.wishlist = self.scene.wishlist
# Apply Filters
self.filter(self.scene)
# Delete the satpy scene so memory is cleared out if it isn't used by the caller
scene = self.scene
self.scene = None
return scene
class ReaderWrapper(roles.FrontendRole):
FILE_EXTENSIONS = []
DEFAULT_READER_NAME = None
DEFAULT_DATASETS = []
# This is temporary until a better solution is found for loading start/end time on init
PRIMARY_FILE_TYPE = None
GENERATE_COMPOSITES = False
def __init__(self, **kwargs):
self.reader = kwargs.pop("reader", self.DEFAULT_READER_NAME)
super(ReaderWrapper, self).__init__(**kwargs)
pathnames = self.find_files_with_extensions()
# Remove keyword arguments that Satpy won't understand
for key in ('search_paths', 'keep_intermediate', 'overwrite_existing',
'exit_on_error'):
kwargs.pop(key, None)
# Create a satpy Scene object
self.scene = Scene(reader=self.reader,
filenames=pathnames,
reader_kwargs=kwargs)
self._begin_time = self.scene.start_time
self._end_time = self.scene.end_time
self.wishlist = set()
self.missing_datasets = set()
@property
def begin_time(self):
return self._begin_time
@property
def end_time(self):
return self._end_time
@property
def available_product_names(self):
return self.scene.available_dataset_names(reader_name=self.reader,
composites=True)
@property
def all_product_names(self):
return self.scene.all_dataset_names(reader_name=self.reader,
composites=True)
@property
def default_products(self):
return self.DEFAULT_DATASETS
def filter(self, scene):
pass
def _purge_node(self, parent_node, missing_nodes):
"""We no longer need this Node, remove it and any children."""
for child in parent_node.children:
self._purge_node(child, missing_nodes)
if parent_node.name is None:
# root node
return
if all(parent in missing_nodes or parent is None for parent in parent_node.parents) and \
parent_node.name in self.scene:
# we aren't needed by anything
# if not parent_node.parents and parent_node.name in self.scene:
LOG.debug("Removing {} because it is no longer needed".format(
parent_node.name))
del self.scene[parent_node.name]
def _update_filtered_dep_tree(self, parent_node):
"""Update Scene wishlist and needed datasets based on filtered datasets."""
missing_deps = set()
for child in parent_node.children:
_req_deps = self._update_filtered_dep_tree(child)
missing_deps.update(_req_deps)
# we are the root node no need to do the rest of the checks
if parent_node.name is None:
# get rid of any dependencies that are no longer needed
self._purge_node(self.scene.dep_tree, missing_deps)
return None
# if we are missing any of our required dependencies then we can't be made
if missing_deps or (not parent_node.children
and parent_node.name not in self.scene):
missing_deps.add(parent_node)
if missing_deps:
if parent_node.name in self.scene:
LOG.debug(
"Removing {} because it is missing some dependencies".
format(parent_node.name))
del self.scene[parent_node.name]
elif parent_node.name in self.scene.wishlist:
# it was asked for but hasn't been generated yet
LOG.debug(
"Removing {} because it is missing some dependencies".
format(parent_node.name))
self.scene.wishlist.remove(parent_node.name)
return missing_deps
def create_scene(self, products=None, **kwargs):
LOG.debug("Loading scene data...")
# If the user didn't provide the products they want, figure out which ones we can create
if products is None:
LOG.debug(
"No products specified to frontend, will try to load logical defaults products"
)
products = self.default_products
kwargs.pop("overwrite_existing")
kwargs.pop("exit_on_error")
kwargs.pop("keep_intermediate")
self.scene.load(products, generate=self.GENERATE_COMPOSITES, **kwargs)
# Apply Filters
self.filter(self.scene)
if not self.GENERATE_COMPOSITES:
self._update_filtered_dep_tree(self.scene.dep_tree)
self.wishlist = self.scene.wishlist
self.missing_datasets = self.scene.missing_datasets
# Delete the satpy scene so memory is cleared out if it isn't used by the caller
scene = self.scene
self.scene = None
return scene
def msg1Proc1_5(dateSnap, avail_times, fldrs):
"""
What does this definition do?
This script processes the raw MSG-1 Level 1.5 data to produces radiance/reflectance image
files in netCDF,-4 geoTIFF & png file formats.
:param dateSnap:
:param avail_times: A single string NOT an array
:param fldrs:
:return:
"""
#- Start coding
# import necessary modules
import os, sys, glob
from satpy.utils import debug_on
from satpy.scene import Scene
from datetime import datetime
from myDefinitions import nc_write_sat_level_1_5, embellish, imResize
# Start the logic
debug_on()
print("\n \t \t \t STARTING THE msg1Proc1_5 run @ time: %s \t \t \t \n \n" % str(datetime.now()))
print("\n.Processing Date set is: %s" % dateSnap)
# Test whether all data folders are appropriately set or not.
basDir, datDir, outDir, logDir, webDir, geoTdir, GSHHS_ROOT = fldrs
print("\n.Base directory is set to: %s" % basDir)
print("\n.Data directory is set to %s" % datDir)
print("\n.NetCDF output directory is set to: %s" % outDir)
print("\n.Log directory is set to: %s" % logDir)
print("\n.Web directory is set to: %s" % webDir)
print("\n.GeoTiff directory is set to: %s" % geoTdir)
avail_times = str(avail_times).split()
for tt in avail_times:
try:
# Start for-loop-1
print("..Started processing for time: %s" % tt)
searchStr = datDir + 'H-000-MSG1*' + dateSnap + tt + '-*'
files = glob.glob(searchStr)
# for testing
print(">>>>>>>>>> For Testing <<<<<<<<<<")
print("datDir is set to %s: " % datDir)
print("Search string is %s" % searchStr)
print(files)
# Start reading filename in satpy
scn = Scene(filenames=files, reader='hrit_msg')
# Get the dataset names in the scene
allChnls = scn.all_dataset_names()
allChnls.remove('HRV') # due to higher resolution
# Save the individual channels (except HRV) as separate gray-scale GeoTIFF files..
for ii in allChnls:
try:
str(ii).split()
print("Working on channel: %s" % ii)
scn.load(str(ii).split())
indImg = scn.resample('IndiaSC')
# Save as netCDF data
outImgStr1 = outDir + 'ind_MSG1-Band_' + ii + '_' + dateSnap + '_' + tt + '.nc'
nc_write_sat_level_1_5(indImg, outImgStr1, ii)
# Save as Full Resolution GeoTIFF files
outImgStr2 = geoTdir + 'ind_' + ii + '_' + dateSnap + '_' + tt + '.tiff'
indImg.save_dataset(ii, filename = outImgStr2, writer = 'geotiff')
# Add graphics
# img2 = embellish(basDir, GSHHS_ROOT, outImgStr2, ii, dateSnap, tt)
# img2.save(outImgStr2)
# Save the data as resized png files
outImgStr3 = webDir + 'ind_' + ii + '_' + dateSnap + '_' + tt + '.png'
indImg.save_dataset(ii, filename = outImgStr3, writer = "simple_image")
outImgStr3 = imResize(outImgStr3)
# Add graphics
img3 = embellish(basDir, GSHHS_ROOT, outImgStr3, ii, dateSnap, tt)
img3.save(outImgStr3)
# unload the read channel data
scn.unload(str(ii).split())
print("Finished processing for channel: %s " % ii)
except:
print("Something went wrong with this Channel: %s" % ii)
continue
# end try-except block
#end for-loop
print("Finished processing for time-stamp: %s" % tt)
except:
print("Something went wrong with this time: %s" % tt)
continue
