def windspeed(full_xarrays, area_def):
''' Product specification to appropriately sector and plot wind speed data
NOTE in geoips/geoimg/plot/prototypealg.py
scifile is converted to xarray BEFORE being passed
sector is converted to area_def BEFORE being passed
from geoips2.geoips1_utils.scifile import xarray_from_scifile
from geoips2.geoips1_utils.sector import area_def_from_sector
'''
final_products = []
# These use a different area definition - make sure the correct one is recorded.
mtif_final_products = []
text_final_products = []
wind_xarray = None
ir_xarray = None
vis_xarray = None
interp_bg = None
for xobj in full_xarrays:
if 'wind_speed_kts' in xobj.variables.keys():
wind_xarray = xobj
if 'Infrared-Gray' in xobj.variables.keys():
ir_xarray = xobj
ir_covg = percent_unmasked(
ir_xarray['Infrared-Gray'].to_masked_array())
if 'Visible-Gray' in xobj.variables.keys():
vis_xarray = xobj
vis_covg = percent_unmasked(
vis_xarray['Visible-Gray'].to_masked_array())
if vis_xarray and ir_xarray and ir_covg > vis_covg:
vis_xarray = None
[interp_data] = interp_nearest(area_def,
wind_xarray,
varlist=['wind_speed_kts'])
interp_vis = None
interp_ir = None
if vis_xarray is not None:
[interp_vis] = interp_nearest(area_def,
vis_xarray,
varlist=['Visible-Gray'])
elif ir_xarray is not None:
[interp_ir] = interp_nearest(area_def,
ir_xarray,
varlist=['Infrared-Gray'])
mtif_area_def = None
mtif_interp_data = None
text_area_def = None
if is_sector_type(area_def, 'atcf'):
mtif_area_def = set_mtif_area_def(wind_xarray, area_def)
[mtif_interp_data] = interp_nearest(mtif_area_def,
wind_xarray,
varlist=['wind_speed_kts'])
text_area_def = set_text_area_def(wind_xarray, area_def)
if mtif_interp_data is None:
mtif_interp_data = interp_data
covg = percent_unmasked(interp_data)
# Assume this is set in the reader.
# if covg > wind_xarray.minimum_coverage:
# Now that we are checking coverage within 4 deg x 4deg center box, we can now take ANY amount of overall coverage.
if covg > 1:
final_products, mtif_final_products, text_final_products = plot_windspeeds(
wind_xarray,
area_def,
interp_data,
mtif_area_def,
mtif_interp_data,
text_area_def,
vis_data=interp_vis,
ir_data=interp_ir,
vis_xarray=vis_xarray,
ir_xarray=ir_xarray)
else:
LOG.info('SKIPPING Insufficient coverage %s%%, minimum required %s%%',
covg, wind_xarray.minimum_coverage)
# This generates YAML files of sector-related metadata for all products in the final_products list
# NOTE this produces metadata based on "area_def" - if we begin re-centering the storm, ensure this is updated to
# reflect varying area definitions
final_products += produce_all_sector_metadata(final_products, area_def,
wind_xarray)
mtif_final_products += produce_all_sector_metadata(mtif_final_products,
mtif_area_def,
wind_xarray)
text_final_products += produce_all_sector_metadata(text_final_products,
text_area_def,
wind_xarray)
return final_products + mtif_final_products + text_final_products
def pmw_mint(xarray_datasets, area_def, arg_dict=None):
''' Process xarray_dataset (xarray_datasets expected to be length 1 list) over area_def, with optional arg_dict.
input xarray-based variables are defined in the readers with the GEOIPS2 framework
Args:
xarray_datasets (list) : list of xarray Dataset objects - for pmw_mint products, expect a length one list.
area_def (AreaDefinition) : pyresample AreaDefinition specifying initial region to process.
arg_dict (dict) : Dictionary of optional arguments (command_line_args are passed through)
Returns:
(list) : List of full paths to all products produed through pmw_mint processing
'''
LOG.info(arg_dict)
final_products = []
full_xarray = xarray_datasets[0]
# DATASET_INFO is imported from readers.mint_ncdf - contains list of possible variables for each dataset
for varname in DATASET_INFO[full_xarray.dataset_name]:
if varname not in full_xarray.variables.keys():
LOG.info('SKIPPING variable %s, not in current xarray object',
varname)
continue
# Interpolation radius of influence is set for each dataset separately in the mint_ncdf reader - adjust
# in readers/mint_ncdf.py ROI_INFO dictionary
# set_roi(full_xarray, varname)
if area_def.sector_start_datetime:
LOG.info('Trying to sector %s with dynamic time %s, %s points',
area_def.area_id, area_def.sector_start_datetime,
full_xarray['latitude'].size)
else:
LOG.info('Trying to sector %s, %s points', area_def.area_id,
full_xarray['latitude'].size)
# Compile a list of variables that will be used to sector - the current data variable, and we will add in
# the appropriate latitude and longitude variables (of the same shape as data), and if it exists the
# appropriately shaped timestamp array
vars_to_sect = [varname] # create a new sect to list intended products
# we have to have 'latitude','longitude" in the full_xarray, and 'timestamp' if we want temporal sectoring
if 'latitude' in full_xarray.variables.keys():
vars_to_sect += ['latitude']
if 'longitude' in full_xarray.variables.keys():
vars_to_sect += ['longitude']
if 'timestamp' in full_xarray.variables.keys():
vars_to_sect += ['timestamp']
# I believe ARCHER can not have any masked data within the data grid, so create a separate smaller sector for
# running archer. The size of the "new" ARCHER sector could probably use some tweaking, though this worked
# "out of the box" for my test case.
# Probably in the end want to just run ARCHER first, get the new center, then create a new area_def with
# the ARCHER center. and sector / register based on the ARCHER centered area_def. Ok, I'll just do that
# really quickly.
archer_area_def = set_atcf_area_def(area_def.sector_info,
num_lines=500,
num_samples=500,
pixel_width=10000,
pixel_height=10000)
archer_xarray = sector_xarray_dataset(full_xarray, archer_area_def,
vars_to_sect)
try:
from geoips2.sector_utils.atcf_tracks import run_archer
in_dict, out_dict, score_dict = run_archer(archer_xarray, varname)
except ValueError:
from IPython import embed as shell
shell()
continue
recentered_area_def = recenter_area_def(area_def, out_dict)
# The list of variables in vars_to_sect must ALL be the same shape
sect_xarray = sector_xarray_dataset(full_xarray, recentered_area_def,
vars_to_sect)
# numpy arrays fail if numpy_array is None, and xarrays fail if x_array == None
if sect_xarray is None:
LOG.info('No coverage - skipping')
return final_products
sect_xarray.attrs[
'area_def'] = recentered_area_def # add name of this sector to sector attribute
if hasattr(sect_xarray, 'timestamp'):
from geoips2.xarray_utils.timestamp import get_min_from_xarray_timestamp
from geoips2.xarray_utils.timestamp import get_max_from_xarray_timestamp
sect_xarray.attrs[
'start_datetime'] = get_min_from_xarray_timestamp(
sect_xarray, 'timestamp')
sect_xarray.attrs['end_datetime'] = get_max_from_xarray_timestamp(
sect_xarray, 'timestamp')
# Note: need to test whether above two lines can reselect min and max time_info for this sector
LOG.info('Sectored data start/end datetime: %s %s, %s points',
sect_xarray.start_datetime, sect_xarray.end_datetime,
numpy.ma.count(sect_xarray[varname].to_masked_array()))
array_nums = [None] # data points?
if len(sect_xarray[varname].shape) == 3:
array_nums = range(0, sect_xarray[varname].shape[2])
for array_num in array_nums:
# selection of an intepolation scheme
from geoips2.xarray_utils.interpolation import interp_nearest
try:
[interp_data] = interp_nearest(recentered_area_def,
sect_xarray,
varlist=[varname],
array_num=array_num)
except ValueError:
from IPython import embed as shell
shell()
final_products += plot_interp_data(interp_data, sect_xarray,
recentered_area_def, varname)
from geoips2.xarray_utils.interpolation import interp_scipy_grid
interp_data = interp_scipy_grid(recentered_area_def,
sect_xarray,
varname,
array_num=array_num,
method='linear')
prodname = '{0}_{1}_GriddataLinear'.format(sect_xarray.source_name,
varname)
final_products += plot_interp_data(interp_data,
sect_xarray,
recentered_area_def,
varname,
product_name=prodname)
return final_products
def template(xarray_datasets, area_def):
''' Template for creating a simple product from a single channel from each supported data type.
All products found in geoips2/products must have consistent call signatures, and require
no return values. All data processing, plotting, and data and imagery outputs must be completed
within the product algorithm.
Call signature for all products found in geoips2/products/<product_name>.py:
def <product_name>(xarray_datasets, area_def):
NOTE FOR GEOIPS1 Interface: Link directly from geoips/productfiles/<source_name>/<product_name>.xml
to geoips2.products.<product_name>.<product_name>
Must be consistently named.
in geoips/geoimg/plot/prototypealg.py
scifile is converted to xarray BEFORE being passed (standard variables latitude, longitude, timestamp)
sector is converted to area_def BEFORE being passed (standard attributes sector_*)
from geoips2.geoips1_utils.scifile import xarray_from_scifile
from geoips2.geoips1_utils.sector import area_def_from_sector
NOTE FOR GEOIPS2 Prototype Interface: This will be imported and called from modules found in:
geoips2/drivers/
Args:
xarray_datasets (list of xarray.Dataset) : list of xarray Dataset objects including all required variables.
Required variables: 'latitude', 'longitude'
Optional variable; 'timestamp'
Required attributes: 'start_datetime', 'end_datetime',
'source_name', 'platform_name', 'data_provider'
area_def (AreaDefinition) : pyresample AreaDefinition object specifying the current location information.
This includes standard pyresample AreaDefinition attributes, as well as additional
dynamic or static sector info (ie, storm name, basin, etc for TCs)
Extra GeoIPS 2 required attributes:
'sector_info', 'sector_type'
optional attributes (if included, assumed dynamic):
'sector_start_datetime', 'sector_end_datetime'
Returns:
(None). All processing, outputs, and plotting is complete at the end of the product scripts - no return value
'''
final_outputs = []
LOG.info('Running area_def %s', area_def)
# Just write out registered datafile ...
# start_dtstr = xarray_datasets[0].start_datetime.strftime('%Y%m%d.%H%M')
# ncfname = start_dtstr+area_def.area_id+'.nc'
# from geoips2.xarray_utils.outputs import write_xarray_netcdf
from IPython import embed as shell
# write_xarray_netcdf(xarray_datasets[0], ncfname)
full_xarray = None
# Assuming for template, we are just plotting a single variable per file, so loop through until we find it.
# Note some "source_names" can have multiple file types, but there will only be one of
# the listed variables per file.
for xarray_dataset in xarray_datasets:
source_name = xarray_dataset.source_name
for currvarname in VARLIST[source_name]:
if currvarname not in xarray_dataset.variables.keys():
# LOG.info('SKIPPING variable %s, not in current xarray object', currvarname)
continue
LOG.info('FOUND variable %s, in xarray dataset with dims %s',
currvarname, xarray_dataset['latitude'].dims)
varname = currvarname
full_xarray = xarray_dataset
if full_xarray is None:
LOG.info('FAILED variable %s not found in any xarray Datasets')
return final_outputs
# Base product name here is "Template" for plotting titles, etc
product_name = 'Template'
# If we want to override the default radius of influence, pull it from the list above.
if source_name in ROILIST.keys():
full_xarray.attrs['interpolation_radius_of_influence'] = ROILIST[
source_name]
# logging information for what we are sectoring (temporally and spatially for dynamic, just temporally for static
if hasattr(area_def,
'sector_start_datetime') and area_def.sector_start_datetime:
LOG.info('Trying to sector %s with dynamic time %s, %s points',
area_def.area_id, area_def.sector_start_datetime,
full_xarray['latitude'].size)
else:
LOG.info('Trying to sector %s, %s points', area_def.area_id,
full_xarray['latitude'].size)
# Compile a list of variables that will be used to sector - the current data variable, and we will add in
# the appropriate latitude and longitude variables (of the same shape as data), and if it exists the
# appropriately shaped timestamp array
vars_to_sect = [varname, 'latitude', 'longitude']
if 'timestamp' in full_xarray.variables.keys():
vars_to_sect += ['timestamp']
# The list of variables in vars_to_sect must ALL be the same shape
from geoips2.xarray_utils.data import sector_xarray_dataset
sect_xarray = sector_xarray_dataset(full_xarray, area_def, vars_to_sect)
# numpy arrays fail if numpy_array is None, and xarrays fail if x_array == None
if sect_xarray is None:
LOG.info('No coverage - skipping')
return final_outputs
sect_xarray.attrs['area_def'] = area_def
# If you have normal old 2d arrays, you don't have to worry about any of the array_nums pieces.
array_nums = [None]
if len(sect_xarray[varname].shape) == 3:
array_nums = range(0, sect_xarray[varname].shape[2])
from geoips2.xarray_utils.interpolation import interp_nearest
from geoips2.xarray_utils.outputs import output_geoips_fname
from geoips2.data_manipulations.info import percent_unmasked
from geoips2.image_utils.mpl_utils import set_matplotlib_colors_standard
final_products = []
for array_num in array_nums:
# Just pass array_num=None if it is a single 2d array
[interp_data] = interp_nearest(area_def,
sect_xarray,
varlist=[varname],
array_num=array_num)
data_range = [interp_data.min(), interp_data.max()]
cmap_name = None
cbar_label = None
if sect_xarray.source_name in CBAR_LABEL_LIST:
cbar_label = CBAR_LABEL_LIST[sect_xarray.source_name]
if sect_xarray.source_name in CMAP_LIST:
cmap_name = CMAP_LIST[sect_xarray.source_name]
if sect_xarray.source_name in DATA_RANGE_LIST:
data_range = DATA_RANGE_LIST[sect_xarray.source_name]
covg = percent_unmasked(interp_data)
if covg > 0:
from geoips2.sector_utils.utils import is_sector_type
if is_sector_type(area_def, 'atcf'):
# get filename from objects
prodname = '{0}_{1}'.format(product_name, varname)
from geoips2.xarray_utils.outputs import output_atcf_fname
atcf_fname = output_atcf_fname(area_def, sect_xarray, prodname,
covg)
atcf_fname_clean = output_atcf_fname(area_def, sect_xarray,
prodname + 'Clean', covg)
mpl_colors_info = set_matplotlib_colors_standard(
data_range=data_range,
cmap_name=cmap_name,
cbar_label=cbar_label)
from geoips2.output_formats.image import create_standard_imagery
final_products += create_standard_imagery(
area_def,
plot_data=interp_data,
xarray_obj=sect_xarray,
product_name_title=prodname,
clean_fname=atcf_fname_clean,
annotated_fnames=[atcf_fname],
mpl_colors_info=mpl_colors_info)
else:
# Get the output filename from sector, and xarray objects
prodname = '{0}_{1}'.format(product_name, varname)
web_fname = output_geoips_fname(area_def,
sect_xarray,
prodname,
covg,
product_dir=product_name,
source_dir=product_name)
web_fname_clean = output_geoips_fname(area_def,
sect_xarray,
prodname + 'Clean',
covg,
product_dir=product_name,
source_dir=product_name)
mpl_colors_info = set_matplotlib_colors_standard(
data_range=data_range,
cmap_name=cmap_name,
cbar_label=cbar_label)
from geoips2.output_formats.image import create_standard_imagery
final_products += create_standard_imagery(
area_def,
plot_data=interp_data,
xarray_obj=sect_xarray,
product_name_title=prodname,
clean_fname=web_fname_clean,
annotated_fnames=[web_fname],
mpl_colors_info=mpl_colors_info)
else:
LOG.info('Insufficient coverage, skipping')
LOG.info('The following products were successfully produced from %s',
__file__)
for final_product in final_products:
LOG.info('PRODUCTSUCCESS %s', final_product)
from geoips2.output_formats.metadata import produce_all_sector_metadata
final_products += produce_all_sector_metadata(final_products, area_def,
full_xarray)
return final_products
def visir(xarray_datasets, area_def):
'''
This is a template for creating an external algorithm for operating on arbitrary data types from the datafile,
outputting required data products, and plotting as needed. Most of standard GeoIPS processing is bypassed for these
algorithm types.
NOTE in geoips/geoimg/plot/prototypealg.py
scifile is converted to xarray BEFORE being passed (standard variables latitude*, longitude*, timestamp*)
sector is converted to area_def BEFORE being passed (standard attributes sector_*)
from geoips2.geoips1_utils.scifile import xarray_from_scifile
from geoips2.geoips1_utils.sector import area_def_from_sector
Args:
xarray_dataset (xarray.Dataset) : xarray Dataset object including all required variables.
area_def (AreaDefinition) : pyresample AreaDefinition object specifying the current location information.
Returns:
(None). since all processing, outputs, and plotting is complete prior to returning to the GeoIPS 1.0 process
flow. No automated plotting is performed from process.py fo prototype algorithms, all plotting must be
performed from geoips2.r
'''
final_products = []
# Assuming since this is the single channel algorithm, that we are only pulling one variable
for xarray_dataset in xarray_datasets:
for currvarname in VARLIST[xarray_dataset.source_name]:
if currvarname not in xarray_dataset.variables.keys():
LOG.info('%s not in xarray_dataset, skipping', currvarname)
continue
varname = currvarname
LOG.info('Running on variable %s in xarray_dataset with shape %s',
varname, xarray_dataset['latitude'].dims)
product_name = PRODNAME_LIST[xarray_dataset.source_name + varname]
if is_dynamic_sector(area_def):
LOG.info('Trying to sector %s with dynamic time %s, %s points',
area_def.area_id, area_def.sector_start_datetime,
xarray_dataset['latitude'].size)
else:
LOG.info('Trying to sector %s, %s points', area_def.area_id,
xarray_dataset['latitude'].size)
xarray_dataset[varname].attrs['units'] = DATA_UNITSLIST[
product_name]
if DATA_UNITSLIST[product_name] == 'Kelvin' and PRODUCT_UNITS_LIST[
product_name] == 'celsius':
if 'units' in xarray_dataset[varname].attrs and xarray_dataset[
varname].units == 'celsius':
LOG.info('%s already in celsius, not converting', varname)
else:
xarray_dataset[
varname] = xarray_dataset[varname] + KtoC_conversion
xarray_dataset[varname].attrs['units'] = 'celsius'
from geoips2.xarray_utils.data import sector_xarray_dataset
# Pass all 4 variables to sector_xarray_dataset, so they are all masked appropriately for pulling min/max vals
varlist = [varname, 'latitude', 'longitude']
if product_name in MIN_SUNZEN_LIST.keys():
varlist += ['SunZenith']
# Grab an extra +- 3 degrees so if we read in the pre-sectored dataset, we will have extra data for
# re-centering
sect_xarray = sector_xarray_dataset(xarray_dataset,
area_def,
varlist,
lon_pad=3,
lat_pad=3)
# Well this is annoying. numpy arrays fail if numpy_array is None, and xarrays fail if x_array == None
if sect_xarray is None:
continue
sect_xarray.attrs['area_def'] = area_def
sect_xarray.attrs['start_datetime'] = xarray_dataset.start_datetime
sect_xarray.attrs['end_datetime'] = xarray_dataset.end_datetime
LOG.info('Sectored data start/mid/end datetime: %s %s, %s points',
sect_xarray.start_datetime, sect_xarray.end_datetime,
numpy.ma.count(sect_xarray[varname].to_masked_array()))
from geoips2.xarray_utils.interpolation import interp_nearest
from geoips2.data_manipulations.info import percent_unmasked
if product_name in MIN_SUNZEN_LIST.keys():
from geoips2.data_manipulations.corrections import mask_night
LOG.info('Percent unmasked day/night %s',
percent_unmasked(sect_xarray[varname]))
sect_xarray[varname] = xarray.DataArray(
mask_night(sect_xarray[varname].to_masked_array(),
sect_xarray['SunZenith'].to_masked_array(),
MIN_SUNZEN_LIST[product_name]))
LOG.info('Percent unmasked day only %s',
percent_unmasked(sect_xarray[varname]))
if product_name in SUNZEN_CORRECTION_LIST:
from geoips2.data_manipulations.corrections import apply_solar_zenith_correction
sect_xarray[varname] = xarray.DataArray(
apply_solar_zenith_correction(
sect_xarray[varname].to_masked_array(),
sect_xarray['SunZenith'].to_masked_array()))
if product_name in GAMMA_LIST.keys():
from geoips2.data_manipulations.corrections import apply_gamma
for gamma in GAMMA_LIST[product_name]:
sect_xarray[varname] = xarray.DataArray(
apply_gamma(sect_xarray[varname].to_masked_array(),
gamma))
if product_name in SCALEFACTOR_LIST.keys():
from geoips2.data_manipulations.corrections import apply_scale_factor
sect_xarray[varname] = xarray.DataArray(
apply_scale_factor(sect_xarray[varname].to_masked_array(),
SCALEFACTOR_LIST[product_name]))
LOG.info('min/max before: %s to %s', sect_xarray[varname].min(),
sect_xarray[varname].max())
[interp_data] = interp_nearest(area_def,
sect_xarray,
varlist=[varname])
LOG.info('min/max after: %s to %s', interp_data.min(),
interp_data.max())
LOG.info('Percent unmasked before %s',
percent_unmasked(interp_data))
covg = percent_unmasked(interp_data)
LOG.info('Percent unmasked after %s',
percent_unmasked(interp_data))
if covg > 0:
final_products += write_product_xarray(
sect_xarray, [varname, 'latitude', 'longitude'],
[product_name, 'latitude', 'longitude'])
from geoips2.xarray_utils.outputs import output_atcf_fname, output_metoctiff
# from geoips2.output_formats.image import plot_image
from geoips2.image_utils.mpl_utils import set_matplotlib_colors_standard
cbar_label = '{0} ({1})'.format(VARTYPE_TITLE[product_name],
xarray_dataset[varname].units)
LOG.info('data min val %s to max va %s', interp_data.min(),
interp_data.max())
# Create the matplotlib color info dict - the fields in this dictionary (cmap, norm, boundaries,
# etc) will be used in plot_image to ensure the image matches the colorbar.
mpl_colors_info = set_matplotlib_colors_standard(
DATARANGE_LIST[product_name],
cmap_name=CMAPLIST[product_name],
cbar_label=cbar_label)
from geoips2.sector_utils.utils import is_sector_type
if is_sector_type(area_def, 'atcf'):
# from geoips2.sector_utils.atcf_tracks import run_archer
# run_archer(sect_xarray, varname)
# get filename from objects
atcf_fname = output_atcf_fname(
area_def,
sect_xarray,
product_name,
covg,
output_type='png',
output_type_dir=PNGDIRS[sect_xarray.source_name],
product_dir=product_name)
atcf_fname_clean = output_atcf_fname(
area_def,
sect_xarray,
product_name,
covg,
output_type='png',
output_type_dir=PNGCLEANDIRS[sect_xarray.source_name],
product_dir=product_name)
# generate images
if product_name in PRODTYPES['Infrared']:
# setup a special color map for Infrared images at 11 um
from geoips2.image_utils.mpl_utils import set_matplotlib_colors_IR
curr_min_tb = -90
curr_max_tb = 30
mpl_colors_info = set_matplotlib_colors_IR(
min_tb=curr_min_tb, max_tb=curr_max_tb)
# call plotting function
from geoips2.output_formats.image import create_standard_imagery
final_products += create_standard_imagery(
area_def,
plot_data=interp_data,
xarray_obj=sect_xarray,
product_name_title=product_name,
clean_fname=atcf_fname_clean,
annotated_fnames=[atcf_fname],
mpl_colors_info=mpl_colors_info,
boundaries_info=BOUNDARIES_INFO,
gridlines_info=GRIDLINES_INFO,
remove_duplicate_minrange=10)
# output Metoctiff files
final_products += output_metoctiff(
product_name,
area_def,
sect_xarray,
interp_data,
requested_data_min=DATARANGE_LIST[product_name][0],
requested_data_max=DATARANGE_LIST[product_name][1],
scale_data_min=MIN_SCALE_VALUE,
scale_data_max=MAX_SCALE_VALUE,
missing_value=MISSING_SCALE_VALUE,
units=xarray_dataset[varname].units,
mpl_cmap=mpl_colors_info['cmap'],
coverage=covg,
atcf_dir=ATCFDIRS[sect_xarray.source_name])
else:
from geoips2.xarray_utils.outputs import output_geoips_fname
# get filename from objects
web_fname = output_geoips_fname(area_def, sect_xarray,
product_name, covg)
from geoips2.output_formats.image import create_standard_imagery
final_products += create_standard_imagery(
area_def,
plot_data=interp_data,
xarray_obj=sect_xarray,
product_name_title=product_name,
clean_fname=None,
annotated_fnames=[web_fname],
mpl_colors_info=mpl_colors_info)
else:
LOG.info('Insufficient coverage, skipping')
from geoips2.output_formats.metadata import produce_all_sector_metadata
final_products += produce_all_sector_metadata(
final_products,
area_def,
xarray_dataset,
metadata_dir=METADATADIRS[xarray_dataset.source_name])
return final_products
def global_stitched(full_xarray, area_def):
'''
Stitching geostationary and polar imagery into a single product
NOTE in geoips/geoimg/plot/prototypealg.py
scifile is converted to xarray BEFORE being passed
sector is converted to area_def BEFORE being passed
from geoips2.geoips1_utils.scifile import xarray_from_scifile
from geoips2.geoips1_utils.sector import area_def_from_sector
'''
# Run stuff like this to produce a global stitched image over time:
# ./geoips/geoips/driver.py /data/20190529.11/goes16/20190529.1100/ -s globalglobal -p global-stitched
# ./geoips/geoips/driver.py /data/20190529.11/goes17/20190529.1100/ -s globalglobal -p global-stitched
# ./geoips/geoips/driver.py /data/20190529.11/ahi/20190529.1100/ -s globalglobal -p global-stitched
# ./geoips/geoips/driver.py /data/20190529.11/meteo11EU/20190529.1100/ -s globalglobal -p global-stitched
# ./geoips/geoips/driver.py /data/20190529.11/meteo8EU/20190529.1100/ -s globalglobal -p global-stitched
# ./geoips/geoips/driver.py /data/20190529.11/npp/20190529.1100/ -s globalarctic -p global-stitched
# ./geoips/geoips/driver.py /data/20190529.11/jpss/20190529.1100/ -s globalantarctic -p global-stitched
# MLS5 Turn this off completely for now. We can move the return farther and farther down the line as we go!
# return
if full_xarray.source_name in SAT_CONFIG['platforms']['roi'].keys():
roi = SAT_CONFIG['platforms']['roi'][full_xarray.source_name]
full_xarray.attrs['interpolation_radius_of_influence'] = roi
varname = SAT_CONFIG['platforms']['merge_channels'][
full_xarray.source_name]
prodname = SAT_CONFIG['platforms']['merge_channels']['product_name']
from geoips2.xarray_utils.data import sector_xarray_dataset
sect_xarray = sector_xarray_dataset(full_xarray, area_def, [varname])
if sect_xarray is None:
LOG.info('NO COVERAGE, SKIPPING')
return None
sect_xarray.attrs['sector_name'] = area_def.area_id
sect_xarray[prodname] = sect_xarray[varname]
from geoips2.xarray_utils.interpolation import interp_nearest
[interp_data] = interp_nearest(area_def, sect_xarray, varlist=[prodname])
import xarray
alg_xarray = xarray.Dataset()
target_lons, target_lats = area_def.get_lonlats()
alg_xarray[prodname] = xarray.DataArray(interp_data)
alg_xarray['latitude'] = xarray.DataArray(target_lats)
alg_xarray['longitude'] = xarray.DataArray(target_lons)
alg_xarray.attrs = sect_xarray.attrs.copy()
from geoips2.filenames.product_filenames import netcdf_write_filename
# Use %H%M time format so we don't try to match seconds.
ncdf_fname = netcdf_write_filename(
gpaths['PRECALCULATED_DATA_PATH'],
product_name=prodname,
source_name=alg_xarray.source_name,
platform_name=alg_xarray.platform_name,
sector_name=area_def.area_id,
product_datetime=alg_xarray.start_datetime,
time_format='%H%M')
from geoips2.xarray_utils.outputs import write_xarray_netcdf
write_xarray_netcdf(alg_xarray, ncdf_fname)
if alg_xarray.source_name in SAT_CONFIG['platforms']['run_on_sources']:
from geoips2.data_manipulations.merge import get_matching_files, merge_data
import geoips.sectorfile as sectorfile
for primary_sector_name in SAT_CONFIG['sectors'].keys():
# Use %H%M time format so we don't try to match seconds, and so we match written filename above.
match_fnames = get_matching_files(
primary_sector_name,
subsector_names=SAT_CONFIG['sectors'][primary_sector_name]
['merge_lines'],
platforms=SAT_CONFIG['platforms']['merge_platforms'],
sources=SAT_CONFIG['platforms']['merge_sources'],
max_time_diffs=SAT_CONFIG['platforms']['merge_max_time_diffs'],
basedir=gpaths['PRECALCULATED_DATA_PATH'],
merge_datetime=full_xarray.start_datetime,
product_name=prodname,
time_format='%H%M')
# Open the primary_sector_name to get the lat/lons and resolution of the overall sector
main_sector = sectorfile.open(sectorlist=[primary_sector_name]
).open_sector(primary_sector_name)
sectlons, sectlats = main_sector.area_definition.get_lonlats()
finaldata, attrs = merge_data(
match_fnames,
sectlons.shape,
variable_name=prodname,
merge_samples=SAT_CONFIG['sectors'][primary_sector_name]
['merge_samples'],
merge_lines=SAT_CONFIG['sectors'][primary_sector_name]
['merge_lines'])
stitched_xarray = xarray.Dataset()
stitched_xarray[prodname] = xarray.DataArray(finaldata).astype(
numpy.float32)
latdims = stitched_xarray.dims.keys()[0]
londims = stitched_xarray.dims.keys()[1]
stitched_xarray['latitude'] = xarray.DataArray(
sectlats[:, 0], dims=latdims).astype(numpy.float32)
stitched_xarray['longitude'] = xarray.DataArray(
sectlons[0, :], dims=londims).astype(numpy.float32)
stitched_xarray.attrs = attrs
stitched_xarray.attrs[
'start_datetime'] = full_xarray.start_datetime
stitched_xarray.attrs['platform_name'] = 'stitched'
stitched_xarray.attrs['source_name'] = 'stitched'
stitched_xarray.attrs['data_provider'] = 'stitched'
# ## from geoips2.output_formats.image import plot_image, set_plotting_params, coverage
# ## set_plotting_params(geoimg_obj,
# ## finaldata,
# ## cmap_name='Blues',
# ## title_str='This is\nMy Title',
# ## is_final=True)
from geoips2.data_manipulations.info import percent_unmasked
from geoips2.xarray_utils.outputs import output_geoips_fname
web_fname = output_geoips_fname(area_def, stitched_xarray,
prodname,
percent_unmasked(finaldata))
web_fname_clear = output_geoips_fname(area_def, stitched_xarray,
prodname + 'Clear',
percent_unmasked(finaldata))
from geoips2.image_utils.mpl_utils import create_figure_and_main_ax_and_mapobj
from geoips2.image_utils.mpl_utils import set_matplotlib_colors_standard
from geoips2.image_utils.mpl_utils import plot_image, save_image, plot_overlays
from geoips2.image_utils.mpl_utils import get_title_string_from_objects, set_title
from geoips2.image_utils.mpl_utils import create_colorbar
# Create matplotlib figure and main axis, where the main image will be plotted
fig, main_ax, mapobj = create_figure_and_main_ax_and_mapobj(
area_def.x_size, area_def.y_size, area_def)
# Create the matplotlib color info dict - the fields in this dictionary (cmap, norm, boundaries,
# etc) will be used in plot_image to ensure the image matches the colorbar.
mpl_colors_info = set_matplotlib_colors_standard(
data_range=[finaldata.min(), finaldata.max()],
cmap_name=None,
cbar_label=None)
# Plot the actual data on a basemap or cartopy instance
plot_image(main_ax,
finaldata,
mapobj,
mpl_colors_info=mpl_colors_info)
# Save the clean image with no gridlines or coastlines
save_image(fig, web_fname_clear, is_final=False)
# Set the title for final image
title_string = get_title_string_from_objects(
area_def, sect_xarray, prodname)
set_title(main_ax, title_string, area_def.y_size)
# Create the colorbar to match the mpl_colors
create_colorbar(fig, mpl_colors_info)
# Plot gridlines and boundaries overlays
plot_overlays(mapobj,
main_ax,
area_def,
boundaries_info=None,
gridlines_info=None)
# Save the final image
save_image(fig, web_fname, is_final=True)
from geoips2.filenames.product_filenames import netcdf_write_filename
del (stitched_xarray.attrs['start_datetime'])
stitched_xarray.attrs[
'valid_datetime'] = full_xarray.start_datetime
# Use %H%M time format so we don't try to match seconds.
ncdf_fname = netcdf_write_filename(
gpaths['MAPROOMDROPBOX'],
product_name=prodname,
source_name=full_xarray.source_name,
platform_name=full_xarray.platform_name,
sector_name=area_def.area_id,
product_datetime=stitched_xarray.valid_datetime,
time_format='%H%M')
from geoips2.xarray_utils.outputs import write_xarray_netcdf
write_xarray_netcdf(full_xarray, ncdf_fname)
else:
LOG.info('SKIPPING Not specified to run on %s, exiting',
alg_xarray.source_name)
return None