def interp_nearest(area_def, xarray_obj, array_num=None, varlist=None):
''' Set up the call to interp_kd_tree using standard attributes and variables in a given xarray object.
Returns:
list of numpy.ma.MaskedArray'''
LOG.info('Interpolating nearest using standard scifile register method: kd_tree nearest')
vars_to_interp = []
if array_num is not None:
lons = xarray_obj['longitude'].to_masked_array()[:, :, array_num]
lats = xarray_obj['latitude'].to_masked_array()[:, :, array_num]
for varname in varlist:
vars_to_interp += [xarray_obj[varname].to_masked_array()[:, :, array_num]]
else:
lons = xarray_obj['longitude'].to_masked_array()
lats = xarray_obj['latitude'].to_masked_array()
for varname in varlist:
vars_to_interp += [xarray_obj[varname].to_masked_array()]
# Use standard scifile / pyresample registration
from geoips2.interpolation.interp_pyresample import interp_kd_tree, get_data_box_definition
data_box_definition = get_data_box_definition(xarray_obj.source_name,
lons,
lats)
interp_data = interp_kd_tree(vars_to_interp,
area_def,
data_box_definition,
float(xarray_obj.interpolation_radius_of_influence),
interp_type='nearest')
from geoips2.data_manipulations.info import percent_unmasked
for arr, orig, varname in zip(interp_data, vars_to_interp, varlist):
LOG.info('%s min/max before: %s to %s', varname, orig.min(), orig.max())
LOG.info('%s min/max after: %s to %s', varname, arr.min(), arr.max())
LOG.info('%s Percent unmasked before %s', varname, percent_unmasked(orig))
LOG.info('%s Percent unmasked after %s', varname, percent_unmasked(arr))
return interp_data
def check_center_coverage(xarray_obj, area_def, varlist, covg_varname, width_degrees=8, height_degrees=8, verbose=False,
hours_before_sector_time=18, hours_after_sector_time=6):
''' Check if there is any data covering the center of the sector '''
# Do not provide any longitude padding for coverage check sectoring - we want to see if there is any data within
# the exact center box, not within +- 3 degrees of the center box.
covg_area_def = set_tc_coverage_check_area_def(area_def, width_degrees=8, height_degrees=8)
from geoips2.xarray_utils.data import sector_xarray_dataset
covg_xarray = sector_xarray_dataset(xarray_obj,
covg_area_def,
varlist,
lon_pad=0,
verbose=verbose,
hours_before_sector_time=hours_before_sector_time,
hours_after_sector_time=hours_after_sector_time)
from geoips2.data_manipulations.info import percent_unmasked
if covg_xarray is None or percent_unmasked(covg_xarray[covg_varname].to_masked_array()) == 0:
return False, covg_xarray
return True, covg_xarray
def plot_windspeeds(wind_xarray,
area_def,
interp_data,
mtif_area_def=None,
mtif_interp_data=None,
text_area_def=None,
vis_data=None,
ir_data=None,
vis_xarray=None,
ir_xarray=None):
''' Plot wind speed files, based on the current data and area definitions
Args:
wind_xarray (Dataset) : xarray Dataset containing metadata information
area_def (AreaDefinition) : pyresample AreaDefintion for full data array
interp_data (ndarray) : numpy.ma.MaskedArray of data to plot, relating to area_def
mtif_area_def (AreaDefinition) : pyresample AreaDefinition pertaining to native resolution mtif data
mtif_interp_data (ndarray) : numpy.ma.MaskedArray of data to plot, relating to mtif_area_def
text_area_def (AreaDefinition) : pyresample AreaDefinition pertaining to non-interpolated sectored data
Returns:
(list, list, list) : List of strings of full final products , List of strings of mtif final products
List of strings for text final products
'''
final_products = []
mtif_final_products = []
text_final_products = []
product_name = 'windspeed'
product_name_title = 'Winds'
product_datatype_title = None
if wind_xarray.source_name in PRODUCT_DATATYPE_TITLE:
product_datatype_title = PRODUCT_DATATYPE_TITLE[
wind_xarray.source_name]
covg = percent_unmasked(interp_data)
bg_mpl_colors_info = None
bg_data = None
bg_xarray = None
bg_product_name_title = None
if vis_data is not None:
from geoips2.products.visir import DATARANGE_LIST, CMAPLIST
bg_data = vis_data
bg_xarray = vis_xarray
bg_product_name_title = 'Visible'
# Create the matplotlib color info dict - the fields in this dictionary (cmap, norm, boundaries,
# etc) will be used in plot_image
bg_mpl_colors_info = set_matplotlib_colors_standard(
DATARANGE_LIST['Visible-Gray'],
cmap_name=CMAPLIST['Visible-Gray'],
cbar_label=None,
create_colorbar=False)
elif ir_data is not None:
from geoips2.products.visir import DATARANGE_LIST, CMAPLIST
# Create the matplotlib color info dict - the fields in this dictionary (cmap, norm, boundaries,
# etc) will be used in plot_image
bg_data = ir_data
bg_xarray = ir_xarray
bg_product_name_title = 'Infrared'
bg_mpl_colors_info = set_matplotlib_colors_standard(
DATARANGE_LIST['Infrared-Gray'],
cmap_name=CMAPLIST['Infrared-Gray'],
cbar_label=None,
create_colorbar=False)
if is_sector_type(area_def, 'atcf'):
mtif_covg = percent_unmasked(mtif_interp_data)
# get filename from objects
atcf_fname = output_atcf_fname(area_def,
wind_xarray,
product_name,
covg,
output_type='png',
output_type_dir='png',
product_dir=product_name)
old_tcweb_fname = output_atcf_fname(area_def,
wind_xarray,
product_name,
covg,
output_type='jpg',
product_dir=product_name,
output_old_tc_web=True)
atcf_fname_clean = output_atcf_fname(area_def,
wind_xarray,
product_name + 'Clean',
covg,
output_type='png',
output_type_dir='png_clean',
product_dir=product_name)
annotated_fnames = [atcf_fname]
if old_tcweb_fname is not None and wind_xarray.source_name in OLDWEB and product_name in OLDWEB[
wind_xarray.source_name]:
annotated_fnames += [old_tcweb_fname]
# atcf_fname_clean = None
# Create the matplotlib color info dict - the fields in this dictionary (cmap, norm, boundaries,
# etc) will be used in plot_image to ensure the image matches the colorbar.
mpl_colors_info = set_matplotlib_colors_winds(
min_wind_speed=PNG_MIN_WIND_SPEED,
max_wind_speed=PNG_MAX_WIND_SPEED)
final_products += create_standard_imagery(
area_def,
plot_data=interp_data,
xarray_obj=wind_xarray,
product_name_title=product_name_title,
clean_fname=atcf_fname_clean,
annotated_fnames=annotated_fnames,
mpl_colors_info=mpl_colors_info,
boundaries_info=BOUNDARIES_INFO,
gridlines_info=GRIDLINES_INFO,
product_datatype_title=product_datatype_title,
bg_data=bg_data,
bg_mpl_colors_info=bg_mpl_colors_info,
bg_xarray=bg_xarray,
bg_product_name_title=bg_product_name_title)
units = 'kts'
if 'units' in wind_xarray['wind_speed_kts'].attrs.keys():
units = wind_xarray['wind_speed_kts'].attrs['units']
# Different color bars for MTIFs and PNGs - if the min/max wind speed in the color bar lines up with the
# min/max scale values, then the colors will line up exactly with the 8 bit integer values.
mpl_colors_info_mtif = set_matplotlib_colors_winds(
min_wind_speed=MTIF_MIN_WIND_SPEED,
max_wind_speed=MTIF_MAX_WIND_SPEED)
mtif_final_products += output_metoctiff(
product_name,
mtif_area_def,
wind_xarray,
mtif_interp_data,
requested_data_min=MTIF_MIN_WIND_SPEED,
requested_data_max=MTIF_MAX_WIND_SPEED,
scale_data_min=MTIF_SCALE_DATA_MIN,
scale_data_max=MTIF_SCALE_DATA_MAX,
missing_value=MTIF_MISSING_VALUE,
coverage=mtif_covg,
mpl_cmap=mpl_colors_info_mtif['cmap'],
units=units)
text_final_products += output_windspeed_text(wind_xarray,
area_def=area_def)
else:
# get filename from objects
web_fname = output_geoips_fname(area_def, wind_xarray, product_name,
covg)
# Create the matplotlib color info dict - the fields in this dictionary (cmap, norm, boundaries,
# etc) will be used in plot_image to ensure the image matches the colorbar.
mpl_colors_info = set_matplotlib_colors_winds(
min_wind_speed=PNG_MIN_WIND_SPEED,
max_wind_speed=PNG_MAX_WIND_SPEED)
final_products += create_standard_imagery(
area_def,
plot_data=interp_data,
xarray_obj=wind_xarray,
product_name_title=product_name_title,
clean_fname=None,
annotated_fnames=[web_fname],
mpl_colors_info=mpl_colors_info,
boundaries_info=BOUNDARIES_INFO,
bg_data=bg_data,
bg_mpl_colors_info=bg_mpl_colors_info,
bg_xarray=bg_xarray)
return final_products, mtif_final_products, text_final_products
def windspeed(full_xarrays, area_def):
''' Product specification to appropriately sector and plot wind speed data
NOTE in geoips/geoimg/plot/prototypealg.py
scifile is converted to xarray BEFORE being passed
sector is converted to area_def BEFORE being passed
from geoips2.geoips1_utils.scifile import xarray_from_scifile
from geoips2.geoips1_utils.sector import area_def_from_sector
'''
final_products = []
# These use a different area definition - make sure the correct one is recorded.
mtif_final_products = []
text_final_products = []
wind_xarray = None
ir_xarray = None
vis_xarray = None
interp_bg = None
for xobj in full_xarrays:
if 'wind_speed_kts' in xobj.variables.keys():
wind_xarray = xobj
if 'Infrared-Gray' in xobj.variables.keys():
ir_xarray = xobj
ir_covg = percent_unmasked(
ir_xarray['Infrared-Gray'].to_masked_array())
if 'Visible-Gray' in xobj.variables.keys():
vis_xarray = xobj
vis_covg = percent_unmasked(
vis_xarray['Visible-Gray'].to_masked_array())
if vis_xarray and ir_xarray and ir_covg > vis_covg:
vis_xarray = None
[interp_data] = interp_nearest(area_def,
wind_xarray,
varlist=['wind_speed_kts'])
interp_vis = None
interp_ir = None
if vis_xarray is not None:
[interp_vis] = interp_nearest(area_def,
vis_xarray,
varlist=['Visible-Gray'])
elif ir_xarray is not None:
[interp_ir] = interp_nearest(area_def,
ir_xarray,
varlist=['Infrared-Gray'])
mtif_area_def = None
mtif_interp_data = None
text_area_def = None
if is_sector_type(area_def, 'atcf'):
mtif_area_def = set_mtif_area_def(wind_xarray, area_def)
[mtif_interp_data] = interp_nearest(mtif_area_def,
wind_xarray,
varlist=['wind_speed_kts'])
text_area_def = set_text_area_def(wind_xarray, area_def)
if mtif_interp_data is None:
mtif_interp_data = interp_data
covg = percent_unmasked(interp_data)
# Assume this is set in the reader.
# if covg > wind_xarray.minimum_coverage:
# Now that we are checking coverage within 4 deg x 4deg center box, we can now take ANY amount of overall coverage.
if covg > 1:
final_products, mtif_final_products, text_final_products = plot_windspeeds(
wind_xarray,
area_def,
interp_data,
mtif_area_def,
mtif_interp_data,
text_area_def,
vis_data=interp_vis,
ir_data=interp_ir,
vis_xarray=vis_xarray,
ir_xarray=ir_xarray)
else:
LOG.info('SKIPPING Insufficient coverage %s%%, minimum required %s%%',
covg, wind_xarray.minimum_coverage)
# This generates YAML files of sector-related metadata for all products in the final_products list
# NOTE this produces metadata based on "area_def" - if we begin re-centering the storm, ensure this is updated to
# reflect varying area definitions
final_products += produce_all_sector_metadata(final_products, area_def,
wind_xarray)
mtif_final_products += produce_all_sector_metadata(mtif_final_products,
mtif_area_def,
wind_xarray)
text_final_products += produce_all_sector_metadata(text_final_products,
text_area_def,
wind_xarray)
return final_products + mtif_final_products + text_final_products
def plot_interp_data(interp_data,
xarray_obj,
area_def,
varname,
product_name=None):
''' Plot the current interpolated data array, using metadata found in xarray_obj and area_def
Args:
interp_data (numpy.ndarray) : ndarray or MaskedArray of data to plot
xarray_obj (Dataset) : xarray Dataset containing appropriate metadata for naming files, etc
area_def (AreaDefinition) : Pyresample AreaDefinition specifying the region to plot
varname (str) : Name of variable that we are plotting out of xarray_obj
Returns:
(list) : List of strings containing full paths to all output products created
'''
final_products = []
if not product_name:
product_name = '{0}_{1}'.format(xarray_obj.source_name, varname)
from geoips2.data_manipulations.info import percent_unmasked
covg = percent_unmasked(interp_data)
if covg > 0: # test a data coverage criterion
if is_sector_type(area_def, 'atcf'):
# Get the output filename from sector, and xarray objects
atcf_fname = output_atcf_fname(area_def, xarray_obj, product_name,
covg)
# product_dir=product_name,
# source_dir=product_name)
atcf_fname_clean = output_atcf_fname(area_def, xarray_obj,
product_name + 'Clear', covg)
# product_dir=product_name,
# source_dir=product_name)
# Create the matplotlib color info dict - the fields in this dictionary (cmap, norm, boundaries,
# etc) will be used in plot_image to ensure the image matches the colorbar.
# selelec a relative color scheme depending on Hi or Lo frequency channels
if varname in ['H19', 'V19', 'H37', 'V37', 'tb37H', 'tb19H']:
# color scheme for low frequency
from geoips2.image_utils.mpl_utils import set_matplotlib_colors_37H
mpl_colors_info = set_matplotlib_colors_37H(min_tb=MIN_TB_LO,
max_tb=MAX_TB_LO)
else:
if varname in ['Chan2_AT', 'Chan3_AT', 'Chan4_AT', 'Chan5_AT']:
# default for high frequency (150-190 GHz) color scheme
from geoips2.image_utils.mpl_utils import set_matplotlib_colors_150H
mpl_colors_info = set_matplotlib_colors_150H(min_tb=110,
max_tb=310)
else:
# default for high frequency (85-91 GHz) color scheme
from geoips2.image_utils.mpl_utils import set_matplotlib_colors_89H
mpl_colors_info = set_matplotlib_colors_89H(
min_tb=MIN_TB_HI, max_tb=MAX_TB_HI)
from geoips2.output_formats.image import create_standard_imagery
final_products += create_standard_imagery(
area_def,
plot_data=interp_data,
xarray_obj=xarray_obj,
product_name_title=product_name,
clean_fname=atcf_fname_clean,
annotated_fname=atcf_fname,
mpl_colors_info=mpl_colors_info,
boundaries_info=BOUNDARIES_INFO,
gridlines_info=GRIDLINES_INFO)
else:
LOG.info('Insufficient coverage, skipping')
# This generates YAML files of sector-related metadata for all products in the final_products list
final_products += produce_all_sector_metadata(final_products, area_def,
xarray_obj)
return final_products
def visir(xarray_datasets, area_def):
'''
This is a template for creating an external algorithm for operating on arbitrary data types from the datafile,
outputting required data products, and plotting as needed. Most of standard GeoIPS processing is bypassed for these
algorithm types.
NOTE in geoips/geoimg/plot/prototypealg.py
scifile is converted to xarray BEFORE being passed (standard variables latitude*, longitude*, timestamp*)
sector is converted to area_def BEFORE being passed (standard attributes sector_*)
from geoips2.geoips1_utils.scifile import xarray_from_scifile
from geoips2.geoips1_utils.sector import area_def_from_sector
Args:
xarray_dataset (xarray.Dataset) : xarray Dataset object including all required variables.
area_def (AreaDefinition) : pyresample AreaDefinition object specifying the current location information.
Returns:
(None). since all processing, outputs, and plotting is complete prior to returning to the GeoIPS 1.0 process
flow. No automated plotting is performed from process.py fo prototype algorithms, all plotting must be
performed from geoips2.r
'''
final_products = []
# Assuming since this is the single channel algorithm, that we are only pulling one variable
for xarray_dataset in xarray_datasets:
for currvarname in VARLIST[xarray_dataset.source_name]:
if currvarname not in xarray_dataset.variables.keys():
LOG.info('%s not in xarray_dataset, skipping', currvarname)
continue
varname = currvarname
LOG.info('Running on variable %s in xarray_dataset with shape %s',
varname, xarray_dataset['latitude'].dims)
product_name = PRODNAME_LIST[xarray_dataset.source_name + varname]
if is_dynamic_sector(area_def):
LOG.info('Trying to sector %s with dynamic time %s, %s points',
area_def.area_id, area_def.sector_start_datetime,
xarray_dataset['latitude'].size)
else:
LOG.info('Trying to sector %s, %s points', area_def.area_id,
xarray_dataset['latitude'].size)
xarray_dataset[varname].attrs['units'] = DATA_UNITSLIST[
product_name]
if DATA_UNITSLIST[product_name] == 'Kelvin' and PRODUCT_UNITS_LIST[
product_name] == 'celsius':
if 'units' in xarray_dataset[varname].attrs and xarray_dataset[
varname].units == 'celsius':
LOG.info('%s already in celsius, not converting', varname)
else:
xarray_dataset[
varname] = xarray_dataset[varname] + KtoC_conversion
xarray_dataset[varname].attrs['units'] = 'celsius'
from geoips2.xarray_utils.data import sector_xarray_dataset
# Pass all 4 variables to sector_xarray_dataset, so they are all masked appropriately for pulling min/max vals
varlist = [varname, 'latitude', 'longitude']
if product_name in MIN_SUNZEN_LIST.keys():
varlist += ['SunZenith']
# Grab an extra +- 3 degrees so if we read in the pre-sectored dataset, we will have extra data for
# re-centering
sect_xarray = sector_xarray_dataset(xarray_dataset,
area_def,
varlist,
lon_pad=3,
lat_pad=3)
# Well this is annoying. numpy arrays fail if numpy_array is None, and xarrays fail if x_array == None
if sect_xarray is None:
continue
sect_xarray.attrs['area_def'] = area_def
sect_xarray.attrs['start_datetime'] = xarray_dataset.start_datetime
sect_xarray.attrs['end_datetime'] = xarray_dataset.end_datetime
LOG.info('Sectored data start/mid/end datetime: %s %s, %s points',
sect_xarray.start_datetime, sect_xarray.end_datetime,
numpy.ma.count(sect_xarray[varname].to_masked_array()))
from geoips2.xarray_utils.interpolation import interp_nearest
from geoips2.data_manipulations.info import percent_unmasked
if product_name in MIN_SUNZEN_LIST.keys():
from geoips2.data_manipulations.corrections import mask_night
LOG.info('Percent unmasked day/night %s',
percent_unmasked(sect_xarray[varname]))
sect_xarray[varname] = xarray.DataArray(
mask_night(sect_xarray[varname].to_masked_array(),
sect_xarray['SunZenith'].to_masked_array(),
MIN_SUNZEN_LIST[product_name]))
LOG.info('Percent unmasked day only %s',
percent_unmasked(sect_xarray[varname]))
if product_name in SUNZEN_CORRECTION_LIST:
from geoips2.data_manipulations.corrections import apply_solar_zenith_correction
sect_xarray[varname] = xarray.DataArray(
apply_solar_zenith_correction(
sect_xarray[varname].to_masked_array(),
sect_xarray['SunZenith'].to_masked_array()))
if product_name in GAMMA_LIST.keys():
from geoips2.data_manipulations.corrections import apply_gamma
for gamma in GAMMA_LIST[product_name]:
sect_xarray[varname] = xarray.DataArray(
apply_gamma(sect_xarray[varname].to_masked_array(),
gamma))
if product_name in SCALEFACTOR_LIST.keys():
from geoips2.data_manipulations.corrections import apply_scale_factor
sect_xarray[varname] = xarray.DataArray(
apply_scale_factor(sect_xarray[varname].to_masked_array(),
SCALEFACTOR_LIST[product_name]))
LOG.info('min/max before: %s to %s', sect_xarray[varname].min(),
sect_xarray[varname].max())
[interp_data] = interp_nearest(area_def,
sect_xarray,
varlist=[varname])
LOG.info('min/max after: %s to %s', interp_data.min(),
interp_data.max())
LOG.info('Percent unmasked before %s',
percent_unmasked(interp_data))
covg = percent_unmasked(interp_data)
LOG.info('Percent unmasked after %s',
percent_unmasked(interp_data))
if covg > 0:
final_products += write_product_xarray(
sect_xarray, [varname, 'latitude', 'longitude'],
[product_name, 'latitude', 'longitude'])
from geoips2.xarray_utils.outputs import output_atcf_fname, output_metoctiff
# from geoips2.output_formats.image import plot_image
from geoips2.image_utils.mpl_utils import set_matplotlib_colors_standard
cbar_label = '{0} ({1})'.format(VARTYPE_TITLE[product_name],
xarray_dataset[varname].units)
LOG.info('data min val %s to max va %s', interp_data.min(),
interp_data.max())
# Create the matplotlib color info dict - the fields in this dictionary (cmap, norm, boundaries,
# etc) will be used in plot_image to ensure the image matches the colorbar.
mpl_colors_info = set_matplotlib_colors_standard(
DATARANGE_LIST[product_name],
cmap_name=CMAPLIST[product_name],
cbar_label=cbar_label)
from geoips2.sector_utils.utils import is_sector_type
if is_sector_type(area_def, 'atcf'):
# from geoips2.sector_utils.atcf_tracks import run_archer
# run_archer(sect_xarray, varname)
# get filename from objects
atcf_fname = output_atcf_fname(
area_def,
sect_xarray,
product_name,
covg,
output_type='png',
output_type_dir=PNGDIRS[sect_xarray.source_name],
product_dir=product_name)
atcf_fname_clean = output_atcf_fname(
area_def,
sect_xarray,
product_name,
covg,
output_type='png',
output_type_dir=PNGCLEANDIRS[sect_xarray.source_name],
product_dir=product_name)
# generate images
if product_name in PRODTYPES['Infrared']:
# setup a special color map for Infrared images at 11 um
from geoips2.image_utils.mpl_utils import set_matplotlib_colors_IR
curr_min_tb = -90
curr_max_tb = 30
mpl_colors_info = set_matplotlib_colors_IR(
min_tb=curr_min_tb, max_tb=curr_max_tb)
# call plotting function
from geoips2.output_formats.image import create_standard_imagery
final_products += create_standard_imagery(
area_def,
plot_data=interp_data,
xarray_obj=sect_xarray,
product_name_title=product_name,
clean_fname=atcf_fname_clean,
annotated_fnames=[atcf_fname],
mpl_colors_info=mpl_colors_info,
boundaries_info=BOUNDARIES_INFO,
gridlines_info=GRIDLINES_INFO,
remove_duplicate_minrange=10)
# output Metoctiff files
final_products += output_metoctiff(
product_name,
area_def,
sect_xarray,
interp_data,
requested_data_min=DATARANGE_LIST[product_name][0],
requested_data_max=DATARANGE_LIST[product_name][1],
scale_data_min=MIN_SCALE_VALUE,
scale_data_max=MAX_SCALE_VALUE,
missing_value=MISSING_SCALE_VALUE,
units=xarray_dataset[varname].units,
mpl_cmap=mpl_colors_info['cmap'],
coverage=covg,
atcf_dir=ATCFDIRS[sect_xarray.source_name])
else:
from geoips2.xarray_utils.outputs import output_geoips_fname
# get filename from objects
web_fname = output_geoips_fname(area_def, sect_xarray,
product_name, covg)
from geoips2.output_formats.image import create_standard_imagery
final_products += create_standard_imagery(
area_def,
plot_data=interp_data,
xarray_obj=sect_xarray,
product_name_title=product_name,
clean_fname=None,
annotated_fnames=[web_fname],
mpl_colors_info=mpl_colors_info)
else:
LOG.info('Insufficient coverage, skipping')
from geoips2.output_formats.metadata import produce_all_sector_metadata
final_products += produce_all_sector_metadata(
final_products,
area_def,
xarray_dataset,
metadata_dir=METADATADIRS[xarray_dataset.source_name])
return final_products
def template(xarray_datasets, area_def):
''' Template for creating a simple product from a single channel from each supported data type.
All products found in geoips2/products must have consistent call signatures, and require
no return values. All data processing, plotting, and data and imagery outputs must be completed
within the product algorithm.
Call signature for all products found in geoips2/products/<product_name>.py:
def <product_name>(xarray_datasets, area_def):
NOTE FOR GEOIPS1 Interface: Link directly from geoips/productfiles/<source_name>/<product_name>.xml
to geoips2.products.<product_name>.<product_name>
Must be consistently named.
in geoips/geoimg/plot/prototypealg.py
scifile is converted to xarray BEFORE being passed (standard variables latitude, longitude, timestamp)
sector is converted to area_def BEFORE being passed (standard attributes sector_*)
from geoips2.geoips1_utils.scifile import xarray_from_scifile
from geoips2.geoips1_utils.sector import area_def_from_sector
NOTE FOR GEOIPS2 Prototype Interface: This will be imported and called from modules found in:
geoips2/drivers/
Args:
xarray_datasets (list of xarray.Dataset) : list of xarray Dataset objects including all required variables.
Required variables: 'latitude', 'longitude'
Optional variable; 'timestamp'
Required attributes: 'start_datetime', 'end_datetime',
'source_name', 'platform_name', 'data_provider'
area_def (AreaDefinition) : pyresample AreaDefinition object specifying the current location information.
This includes standard pyresample AreaDefinition attributes, as well as additional
dynamic or static sector info (ie, storm name, basin, etc for TCs)
Extra GeoIPS 2 required attributes:
'sector_info', 'sector_type'
optional attributes (if included, assumed dynamic):
'sector_start_datetime', 'sector_end_datetime'
Returns:
(None). All processing, outputs, and plotting is complete at the end of the product scripts - no return value
'''
final_outputs = []
LOG.info('Running area_def %s', area_def)
# Just write out registered datafile ...
# start_dtstr = xarray_datasets[0].start_datetime.strftime('%Y%m%d.%H%M')
# ncfname = start_dtstr+area_def.area_id+'.nc'
# from geoips2.xarray_utils.outputs import write_xarray_netcdf
from IPython import embed as shell
# write_xarray_netcdf(xarray_datasets[0], ncfname)
full_xarray = None
# Assuming for template, we are just plotting a single variable per file, so loop through until we find it.
# Note some "source_names" can have multiple file types, but there will only be one of
# the listed variables per file.
for xarray_dataset in xarray_datasets:
source_name = xarray_dataset.source_name
for currvarname in VARLIST[source_name]:
if currvarname not in xarray_dataset.variables.keys():
# LOG.info('SKIPPING variable %s, not in current xarray object', currvarname)
continue
LOG.info('FOUND variable %s, in xarray dataset with dims %s',
currvarname, xarray_dataset['latitude'].dims)
varname = currvarname
full_xarray = xarray_dataset
if full_xarray is None:
LOG.info('FAILED variable %s not found in any xarray Datasets')
return final_outputs
# Base product name here is "Template" for plotting titles, etc
product_name = 'Template'
# If we want to override the default radius of influence, pull it from the list above.
if source_name in ROILIST.keys():
full_xarray.attrs['interpolation_radius_of_influence'] = ROILIST[
source_name]
# logging information for what we are sectoring (temporally and spatially for dynamic, just temporally for static
if hasattr(area_def,
'sector_start_datetime') and area_def.sector_start_datetime:
LOG.info('Trying to sector %s with dynamic time %s, %s points',
area_def.area_id, area_def.sector_start_datetime,
full_xarray['latitude'].size)
else:
LOG.info('Trying to sector %s, %s points', area_def.area_id,
full_xarray['latitude'].size)
# Compile a list of variables that will be used to sector - the current data variable, and we will add in
# the appropriate latitude and longitude variables (of the same shape as data), and if it exists the
# appropriately shaped timestamp array
vars_to_sect = [varname, 'latitude', 'longitude']
if 'timestamp' in full_xarray.variables.keys():
vars_to_sect += ['timestamp']
# The list of variables in vars_to_sect must ALL be the same shape
from geoips2.xarray_utils.data import sector_xarray_dataset
sect_xarray = sector_xarray_dataset(full_xarray, area_def, vars_to_sect)
# numpy arrays fail if numpy_array is None, and xarrays fail if x_array == None
if sect_xarray is None:
LOG.info('No coverage - skipping')
return final_outputs
sect_xarray.attrs['area_def'] = area_def
# If you have normal old 2d arrays, you don't have to worry about any of the array_nums pieces.
array_nums = [None]
if len(sect_xarray[varname].shape) == 3:
array_nums = range(0, sect_xarray[varname].shape[2])
from geoips2.xarray_utils.interpolation import interp_nearest
from geoips2.xarray_utils.outputs import output_geoips_fname
from geoips2.data_manipulations.info import percent_unmasked
from geoips2.image_utils.mpl_utils import set_matplotlib_colors_standard
final_products = []
for array_num in array_nums:
# Just pass array_num=None if it is a single 2d array
[interp_data] = interp_nearest(area_def,
sect_xarray,
varlist=[varname],
array_num=array_num)
data_range = [interp_data.min(), interp_data.max()]
cmap_name = None
cbar_label = None
if sect_xarray.source_name in CBAR_LABEL_LIST:
cbar_label = CBAR_LABEL_LIST[sect_xarray.source_name]
if sect_xarray.source_name in CMAP_LIST:
cmap_name = CMAP_LIST[sect_xarray.source_name]
if sect_xarray.source_name in DATA_RANGE_LIST:
data_range = DATA_RANGE_LIST[sect_xarray.source_name]
covg = percent_unmasked(interp_data)
if covg > 0:
from geoips2.sector_utils.utils import is_sector_type
if is_sector_type(area_def, 'atcf'):
# get filename from objects
prodname = '{0}_{1}'.format(product_name, varname)
from geoips2.xarray_utils.outputs import output_atcf_fname
atcf_fname = output_atcf_fname(area_def, sect_xarray, prodname,
covg)
atcf_fname_clean = output_atcf_fname(area_def, sect_xarray,
prodname + 'Clean', covg)
mpl_colors_info = set_matplotlib_colors_standard(
data_range=data_range,
cmap_name=cmap_name,
cbar_label=cbar_label)
from geoips2.output_formats.image import create_standard_imagery
final_products += create_standard_imagery(
area_def,
plot_data=interp_data,
xarray_obj=sect_xarray,
product_name_title=prodname,
clean_fname=atcf_fname_clean,
annotated_fnames=[atcf_fname],
mpl_colors_info=mpl_colors_info)
else:
# Get the output filename from sector, and xarray objects
prodname = '{0}_{1}'.format(product_name, varname)
web_fname = output_geoips_fname(area_def,
sect_xarray,
prodname,
covg,
product_dir=product_name,
source_dir=product_name)
web_fname_clean = output_geoips_fname(area_def,
sect_xarray,
prodname + 'Clean',
covg,
product_dir=product_name,
source_dir=product_name)
mpl_colors_info = set_matplotlib_colors_standard(
data_range=data_range,
cmap_name=cmap_name,
cbar_label=cbar_label)
from geoips2.output_formats.image import create_standard_imagery
final_products += create_standard_imagery(
area_def,
plot_data=interp_data,
xarray_obj=sect_xarray,
product_name_title=prodname,
clean_fname=web_fname_clean,
annotated_fnames=[web_fname],
mpl_colors_info=mpl_colors_info)
else:
LOG.info('Insufficient coverage, skipping')
LOG.info('The following products were successfully produced from %s',
__file__)
for final_product in final_products:
LOG.info('PRODUCTSUCCESS %s', final_product)
from geoips2.output_formats.metadata import produce_all_sector_metadata
final_products += produce_all_sector_metadata(final_products, area_def,
full_xarray)
return final_products
def global_stitched(full_xarray, area_def):
'''
Stitching geostationary and polar imagery into a single product
NOTE in geoips/geoimg/plot/prototypealg.py
scifile is converted to xarray BEFORE being passed
sector is converted to area_def BEFORE being passed
from geoips2.geoips1_utils.scifile import xarray_from_scifile
from geoips2.geoips1_utils.sector import area_def_from_sector
'''
# Run stuff like this to produce a global stitched image over time:
# ./geoips/geoips/driver.py /data/20190529.11/goes16/20190529.1100/ -s globalglobal -p global-stitched
# ./geoips/geoips/driver.py /data/20190529.11/goes17/20190529.1100/ -s globalglobal -p global-stitched
# ./geoips/geoips/driver.py /data/20190529.11/ahi/20190529.1100/ -s globalglobal -p global-stitched
# ./geoips/geoips/driver.py /data/20190529.11/meteo11EU/20190529.1100/ -s globalglobal -p global-stitched
# ./geoips/geoips/driver.py /data/20190529.11/meteo8EU/20190529.1100/ -s globalglobal -p global-stitched
# ./geoips/geoips/driver.py /data/20190529.11/npp/20190529.1100/ -s globalarctic -p global-stitched
# ./geoips/geoips/driver.py /data/20190529.11/jpss/20190529.1100/ -s globalantarctic -p global-stitched
# MLS5 Turn this off completely for now. We can move the return farther and farther down the line as we go!
# return
if full_xarray.source_name in SAT_CONFIG['platforms']['roi'].keys():
roi = SAT_CONFIG['platforms']['roi'][full_xarray.source_name]
full_xarray.attrs['interpolation_radius_of_influence'] = roi
varname = SAT_CONFIG['platforms']['merge_channels'][
full_xarray.source_name]
prodname = SAT_CONFIG['platforms']['merge_channels']['product_name']
from geoips2.xarray_utils.data import sector_xarray_dataset
sect_xarray = sector_xarray_dataset(full_xarray, area_def, [varname])
if sect_xarray is None:
LOG.info('NO COVERAGE, SKIPPING')
return None
sect_xarray.attrs['sector_name'] = area_def.area_id
sect_xarray[prodname] = sect_xarray[varname]
from geoips2.xarray_utils.interpolation import interp_nearest
[interp_data] = interp_nearest(area_def, sect_xarray, varlist=[prodname])
import xarray
alg_xarray = xarray.Dataset()
target_lons, target_lats = area_def.get_lonlats()
alg_xarray[prodname] = xarray.DataArray(interp_data)
alg_xarray['latitude'] = xarray.DataArray(target_lats)
alg_xarray['longitude'] = xarray.DataArray(target_lons)
alg_xarray.attrs = sect_xarray.attrs.copy()
from geoips2.filenames.product_filenames import netcdf_write_filename
# Use %H%M time format so we don't try to match seconds.
ncdf_fname = netcdf_write_filename(
gpaths['PRECALCULATED_DATA_PATH'],
product_name=prodname,
source_name=alg_xarray.source_name,
platform_name=alg_xarray.platform_name,
sector_name=area_def.area_id,
product_datetime=alg_xarray.start_datetime,
time_format='%H%M')
from geoips2.xarray_utils.outputs import write_xarray_netcdf
write_xarray_netcdf(alg_xarray, ncdf_fname)
if alg_xarray.source_name in SAT_CONFIG['platforms']['run_on_sources']:
from geoips2.data_manipulations.merge import get_matching_files, merge_data
import geoips.sectorfile as sectorfile
for primary_sector_name in SAT_CONFIG['sectors'].keys():
# Use %H%M time format so we don't try to match seconds, and so we match written filename above.
match_fnames = get_matching_files(
primary_sector_name,
subsector_names=SAT_CONFIG['sectors'][primary_sector_name]
['merge_lines'],
platforms=SAT_CONFIG['platforms']['merge_platforms'],
sources=SAT_CONFIG['platforms']['merge_sources'],
max_time_diffs=SAT_CONFIG['platforms']['merge_max_time_diffs'],
basedir=gpaths['PRECALCULATED_DATA_PATH'],
merge_datetime=full_xarray.start_datetime,
product_name=prodname,
time_format='%H%M')
# Open the primary_sector_name to get the lat/lons and resolution of the overall sector
main_sector = sectorfile.open(sectorlist=[primary_sector_name]
).open_sector(primary_sector_name)
sectlons, sectlats = main_sector.area_definition.get_lonlats()
finaldata, attrs = merge_data(
match_fnames,
sectlons.shape,
variable_name=prodname,
merge_samples=SAT_CONFIG['sectors'][primary_sector_name]
['merge_samples'],
merge_lines=SAT_CONFIG['sectors'][primary_sector_name]
['merge_lines'])
stitched_xarray = xarray.Dataset()
stitched_xarray[prodname] = xarray.DataArray(finaldata).astype(
numpy.float32)
latdims = stitched_xarray.dims.keys()[0]
londims = stitched_xarray.dims.keys()[1]
stitched_xarray['latitude'] = xarray.DataArray(
sectlats[:, 0], dims=latdims).astype(numpy.float32)
stitched_xarray['longitude'] = xarray.DataArray(
sectlons[0, :], dims=londims).astype(numpy.float32)
stitched_xarray.attrs = attrs
stitched_xarray.attrs[
'start_datetime'] = full_xarray.start_datetime
stitched_xarray.attrs['platform_name'] = 'stitched'
stitched_xarray.attrs['source_name'] = 'stitched'
stitched_xarray.attrs['data_provider'] = 'stitched'
# ## from geoips2.output_formats.image import plot_image, set_plotting_params, coverage
# ## set_plotting_params(geoimg_obj,
# ## finaldata,
# ## cmap_name='Blues',
# ## title_str='This is\nMy Title',
# ## is_final=True)
from geoips2.data_manipulations.info import percent_unmasked
from geoips2.xarray_utils.outputs import output_geoips_fname
web_fname = output_geoips_fname(area_def, stitched_xarray,
prodname,
percent_unmasked(finaldata))
web_fname_clear = output_geoips_fname(area_def, stitched_xarray,
prodname + 'Clear',
percent_unmasked(finaldata))
from geoips2.image_utils.mpl_utils import create_figure_and_main_ax_and_mapobj
from geoips2.image_utils.mpl_utils import set_matplotlib_colors_standard
from geoips2.image_utils.mpl_utils import plot_image, save_image, plot_overlays
from geoips2.image_utils.mpl_utils import get_title_string_from_objects, set_title
from geoips2.image_utils.mpl_utils import create_colorbar
# Create matplotlib figure and main axis, where the main image will be plotted
fig, main_ax, mapobj = create_figure_and_main_ax_and_mapobj(
area_def.x_size, area_def.y_size, area_def)
# Create the matplotlib color info dict - the fields in this dictionary (cmap, norm, boundaries,
# etc) will be used in plot_image to ensure the image matches the colorbar.
mpl_colors_info = set_matplotlib_colors_standard(
data_range=[finaldata.min(), finaldata.max()],
cmap_name=None,
cbar_label=None)
# Plot the actual data on a basemap or cartopy instance
plot_image(main_ax,
finaldata,
mapobj,
mpl_colors_info=mpl_colors_info)
# Save the clean image with no gridlines or coastlines
save_image(fig, web_fname_clear, is_final=False)
# Set the title for final image
title_string = get_title_string_from_objects(
area_def, sect_xarray, prodname)
set_title(main_ax, title_string, area_def.y_size)
# Create the colorbar to match the mpl_colors
create_colorbar(fig, mpl_colors_info)
# Plot gridlines and boundaries overlays
plot_overlays(mapobj,
main_ax,
area_def,
boundaries_info=None,
gridlines_info=None)
# Save the final image
save_image(fig, web_fname, is_final=True)
from geoips2.filenames.product_filenames import netcdf_write_filename
del (stitched_xarray.attrs['start_datetime'])
stitched_xarray.attrs[
'valid_datetime'] = full_xarray.start_datetime
# Use %H%M time format so we don't try to match seconds.
ncdf_fname = netcdf_write_filename(
gpaths['MAPROOMDROPBOX'],
product_name=prodname,
source_name=full_xarray.source_name,
platform_name=full_xarray.platform_name,
sector_name=area_def.area_id,
product_datetime=stitched_xarray.valid_datetime,
time_format='%H%M')
from geoips2.xarray_utils.outputs import write_xarray_netcdf
write_xarray_netcdf(full_xarray, ncdf_fname)
else:
LOG.info('SKIPPING Not specified to run on %s, exiting',
alg_xarray.source_name)
return None