def get(self, var, varInFile=None, region={}, *args, **kargs):
if region is None:
region = {}
self.variable = var
if varInFile is None:
varInFile = var
# First extract data
f = cdms2.open(os.path.abspath(self()))
out = f(varInFile, *args, **kargs)
f.close()
# Now are we masking anything?
value = region.get("value", None)
if value is not None: # Indeed we are
if self.mask is None:
if isinstance(self.file_mask_template, basestring):
self.file_mask_template = Base(
self.root, self.file_mask_template,
{"domain": region.get("domain", None)})
try:
oMask = self.file_mask_template.get("sftlf")
# ok that failed falling back on autogenerate
except:
oMask = cdutil.generateLandSeaMask(
out, regridTool=self.regridTool).filled(1.) * 100.
oMask = MV2.array(oMask)
oMask.setAxis(-1, out.getLongitude())
oMask.setAxis(-2, out.getLatitude())
self.mask = oMask
if self.mask.shape != out.shape:
dum, msk = genutil.grower(out, self.mask)
else:
msk = self.mask
msk = MV2.not_equal(msk, value)
out = MV2.masked_where(msk, out)
if self.targetGrid is not None:
out = out.regrid(self.targetGrid,
regridTool=self.regridTool,
regridMethod=self.regridMethod,
coordSys='deg',
diag={},
periodicity=1)
if self.targetMask is not None:
if self.targetMask.shape != out.shape:
dum, msk = genutil.grower(out, self.targetMask)
else:
msk = self.targetMask
out = MV2.masked_where(msk, out)
# Now are we looking at a region in particular?
domain = region.get("domain", None)
if domain is not None: # Ok we are subsetting
if isinstance(domain, dict):
out = out(**domain)
elif isinstance(domain, (list, tuple)):
out = out(*domain)
elif isinstance(domain, cdms2.selectors.Selector):
domain.id = region.get("id", "region")
out = out(*[domain])
return out
def surface_maskvariable(var, surf_mask, surf_type=None):
""" Quick script that takes as inputs, a masked variable
and a surface mask to create a masked variable, where the
area not of interest is masked out.
Assumes that the surface mask has values with range 0 to 1.
REMEMBER that the surf_mask denotes area that you want to
have masked out, NOT the area that you want plotted.
Also note: make sure that the mask and the variable have
the same shape
"""
#If no surface mask is given, create mask
if surf_mask is None:
surf_mask = cdutil.generateLandSeaMask(data)
if surf_type is 'land':
surf_mask = surf_mask / 100.
if surf_type is 'ocean':
surf_mask = (100. - surf_mask) / 100.
#Mask the variable according to the surf_mask - assumes that surf_mask is taken from 'LANDFRAC'
# check to see if the surface mask and variable have the same shape, otherwise regrid surf_mask
if surf_mask.shape != var.shape:
var_grid = var.getGrid()
surf_mask = surf_mask.regrid(var_grid)
condition = MV2.greater(surf_mask, 0.5)
var_masked = MV2.masked_where(condition, var)
return var_masked
def surface_maskvariable(var, surf_mask, surf_type=None):
""" Quick script that takes as inputs, a masked variable
and a surface mask to create a masked variable, where the
area not of interest is masked out.
Assumes that the surface mask has values with range 0 to 1.
REMEMBER that the surf_mask denotes area that you want to
have masked out, NOT the area that you want plotted.
Also note: make sure that the mask and the variable have
the same shape
"""
#If no surface mask is given, create mask
if surf_mask is None:
surf_mask = cdutil.generateLandSeaMask(data)
if surf_type is 'land':
surf_mask=surf_mask/100.
if surf_type is 'ocean':
surf_mask=(100.-surf_mask)/100.
#Mask the variable according to the surf_mask - assumes that surf_mask is taken from 'LANDFRAC'
# check to see if the surface mask and variable have the same shape, otherwise regrid surf_mask
if surf_mask.shape!=var.shape:
var_grid=var.getGrid()
surf_mask=surf_mask.regrid(var_grid)
condition=MV2.greater(surf_mask,0.5)
var_masked=MV2.masked_where(condition,var)
return var_masked
def create_sftlf_model_raw(self, var_in_file):
"""For the self.obs_or_model from the initializer, create a landSeaMask
from cdutil for self.sftlf[self.obs_or_model]['raw'] value."""
if (not hasattr(self.parameter, "generate_sftlf")
or self.parameter.generate_sftlf is False):
logging.getLogger("pcmdi_metrics").info(
"Model %s does not have sftlf, skipping region: %s" %
(self.obs_or_model, self.region))
raise RuntimeError(
"Model %s does not have sftlf, skipping region: %s" %
(self.obs_or_model, self.region))
else:
logging.getLogger("pcmdi_metrics").info(
"Auto generating sftlf for model %s" % self._model_file())
if os.path.exists(self._model_file()):
var_file = cdms2.open(self._model_file())
var = var_file[var_in_file]
n = var.rank() - 2 # Minus lat and long
sft = cdutil.generateLandSeaMask(
var(*(slice(0, 1), ) * n)) * 100.0
sft[:] = sft.filled(100.0)
self.sftlf[self.obs_or_model]["raw"] = sft
var_file.close()
logging.getLogger("pcmdi_metrics").info(
"Auto generated sftlf for model %s" % self.obs_or_model)
def get_mask_from_var(self, var):
try:
o_mask = self.file_mask_template.get('sftlf')
except Exception:
o_mask = cdutil.generateLandSeaMask(
var, regridTool=self.regrid_tool).filled(1.) * 100.
o_mask = MV2.array(o_mask)
o_mask.setAxis(-1, var.getLongitude())
o_mask.setAxis(-2, var.getLatitude())
return o_mask
def get_mask_from_var(self, var):
try:
o_mask = self.file_mask_template.get('sftlf')
except Exception:
o_mask = cdutil.generateLandSeaMask(
var, regridTool=self.regrid_tool).filled(1.) * 100.
o_mask = MV2.array(o_mask)
o_mask.setAxis(-1, var.getLongitude())
o_mask.setAxis(-2, var.getLatitude())
return o_mask
def test1(self):
"""
Test cdutil.generateLandSeaMask, using a smaller dataset
"""
f = cdms2.open(sys.prefix + "/sample_data/clt.nc")
s = f("clt")
print s.shape
print dir(cdutil.create_landsea_mask)
# this will call the regrid method
m = cdutil.generateLandSeaMask(s)
def test1(self):
"""
Test cdutil.generateLandSeaMask, using a smaller dataset
"""
f = cdms2.open(cdat_info.get_sampledata_path() + "/clt.nc")
s = f("clt")
print s.shape
print dir(cdutil.create_landsea_mask)
# this will call the regrid method
m = cdutil.generateLandSeaMask(s)
def test1(self):
"""
Test cdutil.generateLandSeaMask, using a smaller dataset
"""
f = cdms2.open(cdat_info.get_sampledata_path() + "/clt.nc")
s = f("clt")
print s.shape
print dir(cdutil.create_landsea_mask)
# this will call the regrid method
m = cdutil.generateLandSeaMask(s)
def estimate_landmask(d):
print("Estimate landmask")
n = 1
sft = cdutil.generateLandSeaMask(d(*(slice(0, 1), ) * n)) * 100.0
sft[:] = sft.filled(100.0)
d2 = sft
d2.setAxis(0, d.getAxis(1))
d2.setAxis(1, d.getAxis(2))
d2.id = "sftlf"
return d2
def test1(self):
"""
Test cdutil.generateLandSeaMask, using a smaller dataset
"""
f = cdms2.open(sys.prefix + "/sample_data/clt.nc")
s = f("clt")
print s.shape
print dir(cdutil.create_landsea_mask)
# this will call the regrid method
m = cdutil.generateLandSeaMask(s)
def Xtest0(self):
"""
Test cdutil.generateLandSeaMask
"""
f = cdms2.open(sys.prefix + \
"/sample_data/so_Omon_GISS-E2-R_historicalNat_r5i1p1_185001-187512_2timesteps.nc")
s = f("so")
print s.shape
print dir(cdutil.create_landsea_mask)
# this will call the regrid method
m = cdutil.generateLandSeaMask(s)
def Xtest0(self):
"""
Test cdutil.generateLandSeaMask
"""
f = cdms2.open(cdat_info.get_sampledata_path() + \
"/so_Omon_GISS-E2-R_historicalNat_r5i1p1_185001-187512_2timesteps.nc")
s = f("so")
print s.shape
print dir(cdutil.create_landsea_mask)
# this will call the regrid method
m = cdutil.generateLandSeaMask(s)
def apply_land_mask_v2(data):
"""
apply land mask (data):
this will read in and reshape the land-sea mask to match data
"""
# Call the cdutil function to generate a mask, 0 for ocean, 1 for land.
mask = cdutil.generateLandSeaMask(data)
# Match dimension using "grower" function
data, mask2 = grower(data, mask)
ocean_data = MV.masked_where(mask2, data)
land_data = MV.masked_where(mask2 == 0., data)
return (ocean_data, land_data)
def create_sftlf(parameter):
"""Create the sftlf file from the parameter."""
sftlf = {}
for test in parameter.test_data_set:
tmp_name = getattr(parameter, "sftlf_filename_template")
if tmp_name is None: # Not defined from commandline or param file
tmp_name = parameter.filename_template
sft = Base(parameter.test_data_path, tmp_name)
sft.model_version = test
sft.table = "fx"
sft.realm = "atmos"
sft.period = getattr(parameter, "period", "")
sft.ext = "nc"
sft.case_id = getattr(parameter, "case_id", "")
sft.target_grid = None
sft.realization = "r0i0p0"
DataSet.apply_custom_keys(sft, parameter.custom_keys, "sftlf")
try:
sftlf[test] = {"raw": sft.get("sftlf")}
sftlf[test]["filename"] = os.path.basename(sft())
sftlf[test]["md5"] = sft.hash()
except Exception:
sftlf[test] = {"raw": None}
sftlf[test]["filename"] = None
sftlf[test]["md5"] = None
if parameter.target_grid == "2.5x2.5":
t_grid = cdms2.createUniformGrid(-88.875, 72, 2.5, 0, 144, 2.5)
else:
t_grid = parameter.target_grid
sft = cdutil.generateLandSeaMask(t_grid)
sft[:] = sft.filled(1.0) * 100.0
sftlf["target_grid"] = sft
return sftlf
def create_sftlf(parameter):
''' Create the sftlf file from the parameter. '''
sftlf = {}
for test in parameter.test_data_set:
tmp_name = getattr(parameter, "sftlf_filename_template")
if tmp_name is None: # Not defined from commandline or param file
tmp_name = parameter.filename_template
sft = Base(parameter.test_data_path, tmp_name)
sft.model_version = test
sft.table = "fx"
sft.realm = "atmos"
sft.period = getattr(parameter, 'period', '')
sft.ext = "nc"
sft.case_id = getattr(parameter, 'case_id', '')
sft.target_grid = None
sft.realization = "r0i0p0"
DataSet.apply_custom_keys(sft, parameter.custom_keys, "sftlf")
try:
sftlf[test] = {"raw": sft.get("sftlf")}
sftlf[test]["filename"] = os.path.basename(sft())
sftlf[test]["md5"] = sft.hash()
except Exception:
sftlf[test] = {"raw": None}
sftlf[test]["filename"] = None
sftlf[test]["md5"] = None
if parameter.target_grid == "2.5x2.5":
t_grid = cdms2.createUniformGrid(-88.875, 72, 2.5, 0, 144, 2.5)
else:
t_grid = parameter.target_grid
sft = cdutil.generateLandSeaMask(t_grid)
sft[:] = sft.filled(1.0) * 100.0
sftlf["target_grid"] = sft
return sftlf
def main():
P.add_argument(
"-j",
"--outnamejson",
type=str,
dest='outnamejson',
default='pr_%(month)_%(firstyear)-%(lastyear)_savg_DiurnalFourier.json',
help="Output name for jsons")
P.add_argument("--lat1", type=float, default=-50., help="First latitude")
P.add_argument("--lat2", type=float, default=50., help="Last latitude")
P.add_argument("--lon1", type=float, default=0., help="First longitude")
P.add_argument("--lon2", type=float, default=360., help="Last longitude")
P.add_argument("--region_name",
type=str,
default="TRMM",
help="name for the region of interest")
P.add_argument(
"-t",
"--filename_template",
default="pr_%(model)_%(month)_%(firstyear)-%(lastyear)_S.nc",
help="template for getting at amplitude files")
P.add_argument(
"--filename_template_tS",
default="pr_%(model)_%(month)_%(firstyear)-%(lastyear)_tS.nc",
help="template for phase files")
P.add_argument(
"--filename_template_sftlf",
default=
"cmip5.%(model).%(experiment).r0i0p0.fx.atm.fx.sftlf.%(version).latestX.xml",
help="template for sftlf file names")
P.add_argument("--model", default="*")
args = P.get_parameter()
month = args.month
monthname = monthname_d[month]
startyear = args.firstyear
finalyear = args.lastyear
years = "%s-%s" % (startyear, finalyear) # noqa: F841
print('Specifying latitude / longitude domain of interest ...')
# TRMM (observed) domain:
latrange = (args.lat1, args.lat2)
lonrange = (args.lon1, args.lon2)
region = cdutil.region.domain(latitude=latrange, longitude=lonrange)
if args.region_name == "":
region_name = "{:g}_{:g}&{:g}_{:g}".format(*(latrange + lonrange))
else:
region_name = args.region_name
# Amazon basin:
# latrange = (-15.0, -5.0)
# lonrange = (285.0, 295.0)
# Functions to convert phase between angle-in-radians and hours, for
# either a 12- or 24-hour clock, i.e. for clocktype = 12 or 24:
def hrs_to_rad(hours, clocktype):
import MV2
return 2 * MV2.pi * hours / clocktype
def rad_to_hrs(phase, clocktype):
import MV2
return phase * clocktype / 2 / MV2.pi
def vectoravg(hr1, hr2, clocktype):
'Function to test vector-averaging of two time values:'
import MV2
sin_avg = (MV2.sin(hrs_to_rad(hr1, clocktype)) +
MV2.sin(hrs_to_rad(hr2, clocktype))) / 2
cos_avg = (MV2.cos(hrs_to_rad(hr1, clocktype)) +
MV2.cos(hrs_to_rad(hr2, clocktype))) / 2
return rad_to_hrs(MV2.arctan2(sin_avg, cos_avg), clocktype)
def spacevavg(tvarb1, tvarb2, sftlf, model):
'''
Given a "root filename" and month/year specifications, vector-average lat/lon arrays in an (amplitude, phase)
pair of input data files. Each input data file contains diurnal (24h), semidiurnal (12h) and terdiurnal (8h)
Fourier harmonic components of the composite mean day/night cycle.
Vector-averaging means we consider the input data to be readings on an 8-, 12- or 24-hour clock and separately
average the Cartesian components (called "cosine" and "sine" below). Then the averaged components are combined
back into amplitude and phase values and returned.
Space-averaging is done globally, as well as separately for land and ocean areas.
'''
glolf = cdutil.averager(sftlf, axis='xy')
print(' Global mean land fraction = %5.3f' % glolf)
outD = {} # Output dictionary to be returned by this function
harmonics = [1, 2, 3]
for harmonic in harmonics:
ampl = tvarb1[harmonic - 1]
tmax = tvarb2[harmonic - 1]
# print ampl[:, :]
# print tmax[:, :]
clocktype = 24 / harmonic
cosine = MV2.cos(hrs_to_rad(tmax, clocktype)) * ampl # X-component
sine = MV2.sin(hrs_to_rad(tmax, clocktype)) * ampl # Y-component
print(
'Area-averaging globally, over land only, and over ocean only ...'
)
# Average Cartesian components ...
cos_avg_glo = cdutil.averager(cosine, axis='xy')
sin_avg_glo = cdutil.averager(sine, axis='xy')
cos_avg_lnd = cdutil.averager(cosine * sftlf, axis='xy')
sin_avg_lnd = cdutil.averager(sine * sftlf, axis='xy')
cos_avg_ocn = cos_avg_glo - cos_avg_lnd
sin_avg_ocn = sin_avg_glo - sin_avg_lnd
# ... normalized by land-sea fraction:
cos_avg_lnd /= glolf
sin_avg_lnd /= glolf
cos_avg_ocn /= (1 - glolf)
sin_avg_ocn /= (1 - glolf)
# Amplitude and phase:
# * 86400 Convert kg/m2/s -> mm/d?
amp_avg_glo = MV2.sqrt(sin_avg_glo**2 + cos_avg_glo**2)
# * 86400 Convert kg/m2/s -> mm/d?
amp_avg_lnd = MV2.sqrt(sin_avg_lnd**2 + cos_avg_lnd**2)
# * 86400 Convert kg/m2/s -> mm/d?
amp_avg_ocn = MV2.sqrt(sin_avg_ocn**2 + cos_avg_ocn**2)
pha_avg_glo = MV2.remainder(
rad_to_hrs(MV2.arctan2(sin_avg_glo, cos_avg_glo), clocktype),
clocktype)
pha_avg_lnd = MV2.remainder(
rad_to_hrs(MV2.arctan2(sin_avg_lnd, cos_avg_lnd), clocktype),
clocktype)
pha_avg_ocn = MV2.remainder(
rad_to_hrs(MV2.arctan2(sin_avg_ocn, cos_avg_ocn), clocktype),
clocktype)
if 'CMCC-CM' in model:
# print '** Correcting erroneous time recording in ', rootfname
pha_avg_lnd -= 3.0
pha_avg_lnd = MV2.remainder(pha_avg_lnd, clocktype)
elif 'BNU-ESM' in model or 'CCSM4' in model or 'CNRM-CM5' in model:
# print '** Correcting erroneous time recording in ', rootfname
pha_avg_lnd -= 1.5
pha_avg_lnd = MV2.remainder(pha_avg_lnd, clocktype)
print(
'Converting singleton transient variables to plain floating-point numbers ...'
)
amp_avg_glo = float(amp_avg_glo)
pha_avg_glo = float(pha_avg_glo)
amp_avg_lnd = float(amp_avg_lnd)
pha_avg_lnd = float(pha_avg_lnd)
amp_avg_ocn = float(amp_avg_ocn)
pha_avg_ocn = float(pha_avg_ocn)
print(
'%s %s-harmonic amplitude, phase = %7.3f mm/d, %7.3f hrsLST averaged globally'
% (monthname, harmonic, amp_avg_glo, pha_avg_glo))
print(
'%s %s-harmonic amplitude, phase = %7.3f mm/d, %7.3f hrsLST averaged over land'
% (monthname, harmonic, amp_avg_lnd, pha_avg_lnd))
print(
'%s %s-harmonic amplitude, phase = %7.3f mm/d, %7.3f hrsLST averaged over ocean'
% (monthname, harmonic, amp_avg_ocn, pha_avg_ocn))
# Sub-dictionaries, one for each harmonic component:
outD['harmonic' + str(harmonic)] = {}
outD['harmonic' + str(harmonic)]['amp_avg_lnd'] = amp_avg_lnd
outD['harmonic' + str(harmonic)]['pha_avg_lnd'] = pha_avg_lnd
outD['harmonic' + str(harmonic)]['amp_avg_ocn'] = amp_avg_ocn
outD['harmonic' + str(harmonic)]['pha_avg_ocn'] = pha_avg_ocn
return outD
print('Preparing to write output to JSON file ...')
if not os.path.exists(args.results_dir):
os.makedirs(args.results_dir)
jsonFile = populateStringConstructor(args.outnamejson, args)
jsonFile.month = monthname
jsonname = os.path.join(os.path.abspath(args.results_dir), jsonFile())
if not os.path.exists(jsonname) or args.append is False:
print('Initializing dictionary of statistical results ...')
stats_dic = {}
metrics_dictionary = collections.OrderedDict()
else:
with open(jsonname) as f:
metrics_dictionary = json.load(f)
stats_dic = metrics_dictionary["RESULTS"]
OUT = pcmdi_metrics.io.base.Base(os.path.abspath(args.results_dir),
os.path.basename(jsonname))
try:
egg_pth = pkg_resources.resource_filename(
pkg_resources.Requirement.parse("pcmdi_metrics"), "share/pmp")
except Exception:
# python 2 seems to fail when ran in home directory of source?
egg_pth = os.path.join(os.getcwd(), "share", "pmp")
disclaimer = open(os.path.join(egg_pth, "disclaimer.txt")).read()
metrics_dictionary["DISCLAIMER"] = disclaimer
metrics_dictionary[
"REFERENCE"] = "The statistics in this file are based on Covey et al., J Climate 2016"
# Accumulate output from each model (or observed) data source in the
# Python dictionary.
template_S = populateStringConstructor(args.filename_template, args)
template_S.month = monthname
template_tS = populateStringConstructor(args.filename_template_tS, args)
template_tS.month = monthname
template_sftlf = populateStringConstructor(args.filename_template_sftlf,
args)
template_sftlf.month = monthname
print("TEMPLATE:", template_S())
files_S = glob.glob(os.path.join(args.modpath, template_S()))
print(files_S)
for file_S in files_S:
print('Reading Amplitude from %s ...' % file_S)
reverted = template_S.reverse(os.path.basename(file_S))
model = reverted["model"]
try:
template_tS.model = model
template_sftlf.model = model
S = cdms2.open(file_S)("S", region)
print('Reading Phase from %s ...' %
os.path.join(args.modpath, template_tS()))
tS = cdms2.open(os.path.join(args.modpath, template_tS()))("tS",
region)
print('Reading sftlf from %s ...' %
os.path.join(args.modpath, template_sftlf()))
try:
sftlf_fnm = glob.glob(
os.path.join(args.modpath, template_sftlf()))[0]
sftlf = cdms2.open(sftlf_fnm)("sftlf", region) / 100.
except BaseException as err:
print('Failed reading sftlf from file (error was: %s)' % err)
print('Creating one for you')
sftlf = cdutil.generateLandSeaMask(S.getGrid())
if model not in stats_dic:
stats_dic[model] = {
region_name: spacevavg(S, tS, sftlf, model)
}
else:
stats_dic[model].update(
{region_name: spacevavg(S, tS, sftlf, model)})
print(stats_dic)
except Exception as err:
print("Failed for model %s with error %s" % (model, err))
# Write output to JSON file.
metrics_dictionary["RESULTS"] = stats_dic
rgmsk = metrics_dictionary.get("RegionalMasking", {})
nm = region_name
region.id = nm
rgmsk[nm] = {"id": nm, "domain": region}
metrics_dictionary["RegionalMasking"] = rgmsk
OUT.write(metrics_dictionary,
json_structure=["model", "domain", "harmonic", "statistic"],
indent=4,
separators=(',', ': '))
print('done')
sftlf[model_version]["filename"] = os.path.basename(sft())
sftlf[model_version]["md5"] = sft.hash()
except:
# Hum no sftlf...
dup.tb = args.traceback
dup("No mask for ", sft())
dup.tb = False
sftlf[model_version] = {"raw": None}
sftlf[model_version]["filename"] = None
sftlf[model_version]["md5"] = None
if parameters.targetGrid == "2.5x2.5":
tGrid = cdms2.createUniformGrid(-88.875, 72, 2.5, 0, 144, 2.5)
else:
tGrid = parameters.targetGrid
sft = cdutil.generateLandSeaMask(tGrid)
sft[:] = sft.filled(1.) * 100.
sftlf["targetGrid"] = sft
# At this point we need to create the tuples var/region to know if a
# variable needs to be ran over a specific region or global or both
regions = getattr(parameters, "regions", {})
vars = []
# Update/overwrite default region_values keys with user ones
regions_values = {}
regions_values.update(getattr(parameters, "regions_values", {}))
# need to convert from old format regions_values to newer region_specs
for reg in regions_values:
dic = {"value": regions_values[reg]}
def generateSurfaceTypeByRegionMask(mask,
sftbyrgn=None,
sftbyrgnmask=215,
regions=range(201, 223),
maximum_regions_per_cell=4,
extend_up_to=3,
verbose=True):
""" Maps a "types" dataset onto a landsea mask
Usage:
mapped,found = generateSurfaceTypeByRegionMask(mask,sftbyrgn,sftbyrgnmask=None,regions=None,maximum_regions_per_cell=4,extend_up_to=3,verbode=True)
Input:
mask : land/sea mask (100/0) onto you wish to map our grid (will generate a ld/sea mask for you)
sftbyrgn: mask you wish to map
if None then uses our own "sftbyrgn" dataset (old ezget type)
sftbyrgnmask: land/sea mask for sftbyrgn
or a number specifying limit in values of sftbygrn
which indicate the threshold land/sea (greater values are land)
regions: Numbers from sftbyrgn array that you want to map onto mask
maximum_regions_per_cell: maximum number f regions concidered in a cell
extend_up_to : how many grid cells away around a cell can we extent to identify a guess
verbose: prints to the screen what's going on (default is True)
Output:
mapped : mapped input mask
found : ???
"""
## OK first determine which regions are available
## Must be integer values
if isinstance(mask, cdms2.grid.TransientRectGrid):
mask = cdutil.generateLandSeaMask(mask) * 100.
if sftbyrgn is None:
sftbyrgn = cdms2.open(
os.path.join(sys.prefix, 'sample_data', 'sftbyrgn.nc'))('sftbyrgn')
if regions is None:
if verbose: print 'Preparing regions'
## regions = range(201,223)
regions = []
for i in range(0, 10000):
genutil.statusbar(i, 9999)
c = float(MV2.sum(MV2.ravel(MV2.equal(sftbyrgn, i)), 0))
if c != 0: regions.append(i)
if verbose: print 'Regions:', regions
## If no mask passed fr sftbyrgn, assumes everything greater 5000 is land)
if isinstance(sftbyrgnmask, int):
split = sftbyrgnmask
n = MV2.maximum(mask)
sftbyrgnmask = MV2.greater_equal(sftbyrgn, sftbyrgnmask) * n
else:
split = MV2.maximum(sftbyrgnmask) / 2.
## Now guess the type for each regions
keys = {}
## ## Nice way to do it
## for r in regions:
## c=MV2.not_equal(sftbyrgn,r)
## c=MV2.masked_where(c,sftbyrgnmask)
## n=MV2.count(c)
## c=float(MV2.sum(MV2.ravel(c),0)/n)
## print r,c,n
## keys[r]=c
## Fast but not so "general" way to do it
for r in regions:
if r < split:
keys[r] = 0.
else:
keys[r] = 100.
sh = list(mask.shape)
sh.insert(0, maximum_regions_per_cell)
potential = MV2.ones(sh, dtype='d') * -999
potential_reg = MV2.ones(sh, dtype='d') * -999
g1 = sftbyrgn.getGrid()
g2 = mask.getGrid()
r1 = regrid2.Regridder(g1, g2)
w = cdutil.area_weights(sftbyrgn)
if verbose: print 'First pass'
itmp = 0.
for ireg in keys.keys():
genutil.statusbar(itmp, len(keys.keys()) - 1)
itmp += 1.
c = MV2.equal(sftbyrgn, ireg)
w2 = 1. - c * w
s2, w3 = r1(sftbyrgn, mask=w2.filled(), returnTuple=1)
c2 = MV2.equal(mask, keys[ireg])
loop(potential, potential_reg, c2, w3, ireg)
found = MV2.zeros(sh[1:], typecode='f')
for i in range(maximum_regions_per_cell):
found = found + MV2.not_equal(potential[i], -999)
sh2 = list(sh)
for k in range(extend_up_to):
sh2[1] = sh[1] + 2 * (k + 1)
sh2[2] = sh[2] + 2 * (k + 1)
## Form the possible i/j couples !
s = MV2.sum(MV2.ravel(MV2.equal(potential[0], -999)), 0)
if verbose:
print 'Expanding up to', k + 1, 'cells while trying to fix', s, 'cells'
## if dump:
## f=cdms2.open('tmp_'+str(k)+'.nc','w')
## f.write(sumregions(potential_reg,potential).astype('f'),id='sftbyrgn',axes=mask.getAxisList())
## f.close()
## g=sumregions(potential_reg,potential).astype('d')
## g=MV2.masked_equal(g,-999)
## g=MV2.greater(g,4999)*100.
## g=MV2.absolute(mask-g)
## g=MV2.masked_equal(g,0.)
## print 'Number of differences:',MV2.count(g)
if float(s) != 0:
c0 = MV2.equal(potential[0], -999)
couples = []
sft2 = MV2.zeros(sh2[1:], dtype='d') - 888.
sft2[k + 1:-k - 1, k + 1:-k - 1] = mask
for i in range(-k - 1, k + 2):
for j in range(-k - 1, k + 2):
if abs(i) > k or abs(j) > k: couples.append([i, j])
ntot = len(keys.keys()) * len(couples) - 1
itmp = 0
for ireg in keys.keys():
c = MV2.equal(sftbyrgn, ireg)
w2 = 1. - c * w
s2, w3 = r1(sftbyrgn, mask=w2.filled(), returnTuple=1)
w4 = MV2.zeros(sh2[1:], typecode='d')
w4[k + 1:-k - 1, k + 1:-k - 1] = w3
for i, j in couples:
if verbose: genutil.statusbar(itmp, ntot)
itmp += 1.
c2 = MV2.equal(
sft2[j + k + 1:j + k + 1 + sh[1],
i + k + 1:i + k + 1 + sh[2]], keys[ireg])
c3 = MV2.equal(
sft2[j + k + 1:j + k + 1 + sh[1],
i + k + 1:i + k + 1 + sh[2]], mask)
c2 = MV2.logical_and(c2, c3)
c2 = MV2.logical_and(c2, c0)
loop(
potential, potential_reg, c2,
w4[j + k + 1:j + k + 1 + sh[1],
i + k + 1:i + k + 1 + sh[2]], ireg)
found = MV2.where(MV2.equal(potential[0], -999), found - 1, found)
out = sumregions(potential_reg, potential)
out.setAxisList(mask.getAxisList())
found.setAxisList(mask.getAxisList())
found = found.astype('i')
found.missing_value = -999
found.id = 'found'
out.id = 'sftbyrgn'
out = out.astype('i')
out.missing_value = -999
del (out.name)
del (found.name)
return out, found
import cdutil
import genutil
import MV2
# Read data
var = 'pr'
data = 'GPCP-2-3'
period = '197901-201907'
ver = 'v20200421'
dir = '/p/user_pub/PCMDIobs/PCMDIobs2_clims/atmos/' + var + '/' + data + '/'
nc = var + '_mon_' + data + '_BE_gn_' + period + '.' + ver + '.AC.nc'
f = cdms.open(dir + nc)
# Land/Sea masking
d = f[var]
mask = cdutil.generateLandSeaMask(d[0])
d, mask2 = genutil.grower(d, mask)
d_ocean = MV2.masked_where(mask2 == 1., d)
d_land = MV2.masked_where(mask2 == 0., d)
# Write data
out = cdms.open(
var + '_mon_' + data + '_BE_gn_' + period + '.' + ver +
'.AC_land.sea.mask.nc', 'w')
out.write(mask2[0])
out.close()
# In[2]:
get_ipython().system('jupyter nbconvert --to script cdat_land.sea.mask.ipynb')
def get(self, var, varInFile=None, region={}, *args, **kargs):
if region is None:
region = {}
self.variable = var
if varInFile is None:
varInFile = var
# First extract data
f = cdms2.open(os.path.abspath(self()))
out = f(varInFile, *args, **kargs)
f.close()
# Now are we masking anything?
value = region.get("value", None)
if value is not None: # Indeed we are
if self.mask is None:
if isinstance(self.file_mask_template, basestring):
self.file_mask_template = Base(
self.root, self.file_mask_template, {
"domain": region.get(
"domain", None)})
try:
oMask = self.file_mask_template.get("sftlf")
# ok that failed falling back on autogenerate
except:
oMask = cdutil.generateLandSeaMask(
out,
regridTool=self.regridTool).filled(1.) * 100.
oMask = MV2.array(oMask)
oMask.setAxis(-1, out.getLongitude())
oMask.setAxis(-2, out.getLatitude())
self.mask = oMask
if self.mask.shape != out.shape:
dum, msk = genutil.grower(out, self.mask)
else:
msk = self.mask
msk = MV2.not_equal(msk, value)
out = MV2.masked_where(msk, out)
if self.targetGrid is not None:
out = out.regrid(
self.targetGrid,
regridTool=self.regridTool,
regridMethod=self.regridMethod,
coordSys='deg',
diag={},
periodicity=1)
if self.targetMask is not None:
if self.targetMask.shape != out.shape:
dum, msk = genutil.grower(out, self.targetMask)
else:
msk = self.targetMask
out = MV2.masked_where(msk, out)
# Now are we looking at a region in particular?
domain = region.get("domain", None)
if domain is not None: # Ok we are subsetting
if isinstance(domain, dict):
out = out(**domain)
elif isinstance(domain, (list, tuple)):
out = out(*domain)
elif isinstance(domain, cdms2.selectors.Selector):
domain.id = region.get("id", "region")
out = out(*[domain])
return out
sftlf[model_version]["filename"] = os.path.basename(sft())
sftlf[model_version]["md5"] = sft.hash()
except:
# Hum no sftlf...
dup.tb = args.traceback
dup("No mask for ", sft())
dup.tb = False
sftlf[model_version] = {"raw": None}
sftlf[model_version]["filename"] = None
sftlf[model_version]["md5"] = None
if parameters.targetGrid == "2.5x2.5":
tGrid = cdms2.createUniformGrid(-88.875, 72, 2.5, 0, 144, 2.5)
else:
tGrid = parameters.targetGrid
sft = cdutil.generateLandSeaMask(tGrid)
sft[:] = sft.filled(1.) * 100.
sftlf["targetGrid"] = sft
# At this point we need to create the tuples var/region to know if a
# variable needs to be ran over a specific region or global or both
regions = getattr(parameters, "regions", {})
vars = []
# Update/overwrite default region_values keys with user ones
regions_values = {}
regions_values.update(getattr(parameters, "regions_values", {}))
# need to convert from old format regions_values to newer region_specs
for reg in regions_values:
dic = {"value": regions_values[reg]}
import cdms2,sys,cdutil,os,cdat_info
f=cdms2.open(os.path.join(cdat_info.get_prefix(),"sample_data","navy_land.nc"))
navy_frac = f("sftlf")/100.
target = cdms2.open(os.path.join(cdat_info.get_prefix(),'sample_data','clt.nc'))("clt",slice(0,1)).getGrid()
mask = cdutil.generateLandSeaMask(target,navy_frac)
target = cdms2.open(os.path.join(cdat_info.get_prefix(),'sample_data','clt.nc'))("clt",slice(0,1))
mask = cdutil.generateLandSeaMask(target,navy_frac)
target=cdms2.createGaussianGrid(64)
mask = cdutil.generateLandSeaMask(target)
target = cdms2.open(os.path.join(cdat_info.get_prefix(),'sample_data','clt.nc'))("clt",slice(0,1),latitude=(15,85),longitude=(-175,-65)).getGrid()
mask = cdutil.generateLandSeaMask(target)
#import vcs
#x=vcs.init()
#x.plot(mask)
#raw_input()
def testGenmask(self):
f = cdms2.open(os.path.join(egg_path, "navy_land.nc"))
navy_frac_one = f('sftlf')
g = navy_frac_one.getGrid()
print(("SOURCE GRID:", g))
print(("LON:", g.getLongitude()))
print(("LAT:", g.getLatitude()))
navy_frac = navy_frac_one / 100.
g = navy_frac.getGrid()
print(("SOURCE GRID 2:", g))
print(("LON:", g.getLongitude()))
print(("LAT:", g.getLatitude()))
target = cdms2.open(
os.path.join(cdat_info.get_sampledata_path(),
'clt.nc'))("clt", slice(0, 1)).getGrid()
print(("TARGET:", navy_frac.getGrid()))
mask = cdutil.generateLandSeaMask(target, navy_frac)
target = cdms2.open(
os.path.join(cdat_info.get_sampledata_path(),
'clt.nc'))("clt", slice(0, 1))
mask = cdutil.generateLandSeaMask(target, navy_frac)
target = cdms2.createGaussianGrid(64)
mask = cdutil.generateLandSeaMask(target)
target = cdms2.open(
os.path.join(cdat_info.get_sampledata_path(),
'clt.nc'))("clt",
slice(0, 1),
latitude=(15, 85),
longitude=(-175, -65)).getGrid()
mask = cdutil.generateLandSeaMask(target)
self.assertEqual(mask.shape, (17, 23))
good = numpy.array([
[
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0
],
[
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0
],
[
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0
],
[
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0
],
[
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0
],
[
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0,
1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0
],
[
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0,
1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0
],
[
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0,
1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 1.0, 1.0, 1.0, 1.0
],
[
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0,
1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0
],
[
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0,
1.0, 1.0, 1.0, 0.0, 1.0, 1.0, 1.0, 0.0, 1.0, 1.0, 1.0
],
[
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0,
1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0
],
[
0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0
],
[
1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0
],
[
0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0,
1.0, 1.0, 1.0, 0.0, 1.0, 0.0, 1.0, 1.0, 1.0, 1.0, 0.0
],
[
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0,
1.0, 0.0, 0.0, 1.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0
],
[
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 1.0
],
[
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0
]
])
#good = numpy.ma.masked_equal(good,1.0)
self.assertTrue(numpy.ma.allclose(mask.filled(3.) - good, 0))
# Adapted for numpy/ma/cdms2 by convertcdms.py
"""
This script converts a land sea mask to a stbyrgn mask
Input:
Land/sea mask
Original sftbyrgn
dctionary of values in sftbyrgn/type (ld or water)
Ouptput:
Newsftbyrgn
"""
import cdms2, cdutil, MV2, os, sys, cdat_info
bg = 0
din = cdms2.open(os.path.join(cdat_info.get_sampledata_path(),
"clt.nc"))("clt", slice(0, 1))
print 'generating mask'
sftlf = cdutil.generateLandSeaMask(din) * 100.
print 'done:', sftlf.shape
newsftbyrgn, n = cdutil.generateSurfaceTypeByRegionMask(sftlf)
#import vcs
#x=vcs.init()
#x.plot(newsftbyrgn,bg=bg)
#x.clear()
#x.plot(n,bg=bg)
#raw_input()
def __init__(self, data):
self.mask = cdutil.generateLandSeaMask(data[0])
self.d, self.mask2 = genutil.grower(data, self.mask)
os.path.join(
args.modpath,
template_tS()))(
"tS",
region)
print('Reading sftlf from %s ...' % os.path.join(args.modpath, template_sftlf()))
try:
sftlf_fnm = glob.glob(
os.path.join(
args.modpath,
template_sftlf()))[0]
sftlf = cdms2.open(sftlf_fnm)("sftlf", region) / 100.
except BaseException as err:
print('Failed reading sftlf from file (error was: %s)' % err)
print('Creating one for you')
sftlf = cdutil.generateLandSeaMask(S.getGrid())
if model not in stats_dic:
stats_dic[model] = {region_name: spacevavg(S, tS, sftlf, model)}
else:
stats_dic[model].update(
{region_name: spacevavg(S, tS, sftlf, model)})
print(stats_dic)
except Exception as err:
print("Failed for model %s with error %s" % (model, err))
# Write output to JSON file.
metrics_dictionary["RESULTS"] = stats_dic
rgmsk = metrics_dictionary.get("RegionalMasking", {})
nm = region_name
region.id = nm
def generateSurfaceTypeByRegionMask(mask,sftbyrgn=None,sftbyrgnmask=215,regions = range(201,223), maximum_regions_per_cell=4,extend_up_to=3,verbose=True):
""" Maps a "types" dataset onto a landsea mask
Usage:
mapped,found = generateSurfaceTypeByRegionMask(mask,sftbyrgn,sftbyrgnmask=None,regions=None,maximum_regions_per_cell=4,extend_up_to=3,verbode=True)
Input:
mask : land/sea mask (100/0) onto you wish to map our grid (will generate a ld/sea mask for you)
sftbyrgn: mask you wish to map
if None then uses our own "sftbyrgn" dataset (old ezget type)
sftbyrgnmask: land/sea mask for sftbyrgn
or a number specifying limit in values of sftbygrn
which indicate the threshold land/sea (greater values are land)
regions: Numbers from sftbyrgn array that you want to map onto mask
maximum_regions_per_cell: maximum number f regions concidered in a cell
extend_up_to : how many grid cells away around a cell can we extent to identify a guess
verbose: prints to the screen what's going on (default is True)
Output:
mapped : mapped input mask
found : ???
"""
## OK first determine which regions are available
## Must be integer values
if isinstance(mask, cdms2.grid.TransientRectGrid):
mask = cdutil.generateLandSeaMask(mask)*100.
if sftbyrgn is None:
sftbyrgn = cdms2.open(os.path.join(sys.prefix,'sample_data','sftbyrgn.nc'))('sftbyrgn')
if regions is None:
if verbose: print 'Preparing regions'
## regions = range(201,223)
regions=[]
for i in range(0,10000):
genutil.statusbar(i,9999)
c=float(MV2.sum(MV2.ravel(MV2.equal(sftbyrgn,i)),0))
if c!=0: regions.append(i)
if verbose: print 'Regions:',regions
## If no mask passed fr sftbyrgn, assumes everything greater 5000 is land)
if isinstance(sftbyrgnmask,int):
split = sftbyrgnmask
n=MV2.maximum(mask)
sftbyrgnmask=MV2.greater_equal(sftbyrgn,sftbyrgnmask)*n
else:
split = MV2.maximum(sftbyrgnmask)/2.
## Now guess the type for each regions
keys={}
## ## Nice way to do it
## for r in regions:
## c=MV2.not_equal(sftbyrgn,r)
## c=MV2.masked_where(c,sftbyrgnmask)
## n=MV2.count(c)
## c=float(MV2.sum(MV2.ravel(c),0)/n)
## print r,c,n
## keys[r]=c
## Fast but not so "general" way to do it
for r in regions:
if r< split:
keys[r]=0.
else:
keys[r]=100.
sh=list(mask.shape)
sh.insert(0,maximum_regions_per_cell)
potential=MV2.ones(sh,dtype='d')*-999
potential_reg=MV2.ones(sh,dtype='d')*-999
g1=sftbyrgn.getGrid()
g2=mask.getGrid()
r1=regrid2.Regridder(g1,g2)
w=cdutil.area_weights(sftbyrgn)
if verbose: print 'First pass'
itmp=0.
for ireg in keys.keys():
genutil.statusbar(itmp,len(keys.keys())-1)
itmp+=1.
c=MV2.equal(sftbyrgn,ireg)
w2=1.-c*w
s2,w3=r1(sftbyrgn,mask=w2.filled(),returnTuple=1)
c2=MV2.equal(mask,keys[ireg])
loop(potential,potential_reg,c2,w3,ireg)
found=MV2.zeros(sh[1:],typecode='f')
for i in range(maximum_regions_per_cell):
found=found+MV2.not_equal(potential[i],-999)
sh2=list(sh)
for k in range(extend_up_to):
sh2[1]=sh[1]+2*(k+1)
sh2[2]=sh[2]+2*(k+1)
## Form the possible i/j couples !
s=MV2.sum(MV2.ravel(MV2.equal(potential[0],-999)),0)
if verbose: print 'Expanding up to',k+1,'cells while trying to fix',s,'cells'
## if dump:
## f=cdms2.open('tmp_'+str(k)+'.nc','w')
## f.write(sumregions(potential_reg,potential).astype('f'),id='sftbyrgn',axes=mask.getAxisList())
## f.close()
## g=sumregions(potential_reg,potential).astype('d')
## g=MV2.masked_equal(g,-999)
## g=MV2.greater(g,4999)*100.
## g=MV2.absolute(mask-g)
## g=MV2.masked_equal(g,0.)
## print 'Number of differences:',MV2.count(g)
if float(s)!=0:
c0=MV2.equal(potential[0],-999)
couples=[]
sft2=MV2.zeros(sh2[1:],dtype='d')-888.
sft2[k+1:-k-1,k+1:-k-1]=mask
for i in range(-k-1,k+2):
for j in range(-k-1,k+2):
if abs(i)>k or abs(j)>k: couples.append([i,j])
ntot=len(keys.keys())*len(couples)-1
itmp=0
for ireg in keys.keys():
c=MV2.equal(sftbyrgn,ireg)
w2=1.-c*w
s2,w3=r1(sftbyrgn,mask=w2.filled(),returnTuple=1)
w4=MV2.zeros(sh2[1:],typecode='d')
w4[k+1:-k-1,k+1:-k-1]=w3
for i,j in couples:
if verbose: genutil.statusbar(itmp,ntot)
itmp+=1.
c2=MV2.equal(sft2[j+k+1:j+k+1+sh[1],i+k+1:i+k+1+sh[2]],keys[ireg])
c3=MV2.equal(sft2[j+k+1:j+k+1+sh[1],i+k+1:i+k+1+sh[2]],mask)
c2=MV2.logical_and(c2,c3)
c2=MV2.logical_and(c2,c0)
loop(potential,potential_reg,c2,w4[j+k+1:j+k+1+sh[1],i+k+1:i+k+1+sh[2]],ireg)
found=MV2.where(MV2.equal(potential[0],-999),found-1,found)
out=sumregions(potential_reg,potential)
out.setAxisList(mask.getAxisList())
found.setAxisList(mask.getAxisList())
found=found.astype('i')
found.missing_value=-999
found.id='found'
out.id='sftbyrgn'
out=out.astype('i')
out.missing_value=-999
del(out.name)
del(found.name)
return out,found
sft.ext = "nc"
sft.targetGrid = None
sft.realization="r0i0p0"
applyCustomKeys(sft,parameters.custom_keys,"sftlf")
try:
sftlf[model_version] = {"raw":sft.get("sftlf")}
except:
#Hum no sftlf...
dup("No mask for ",sft())
sftlf[model_version] = {"raw":None}
if parameters.targetGrid == "2.5x2.5":
tGrid = cdms2.createUniformGrid(-88.875,72,2.5,0,144,2.5)
else:
tGrid = parameters.targetGrid
sftlf["targetGrid"] = cdutil.generateLandSeaMask(tGrid)*100.
#At this point we need to create the tuples var/region to know if a variable needs to be ran over a specific region or global or both
regions = getattr(parameters,"regions",{})
vars = []
#Update/overwrite defsult region_values keys with user ones
regions_values.update(getattr(parameters,"regions_values",{}))
regions_dict = {}
for var in parameters.vars:
vr = var.split("_")[0]
rg = regions.get(vr,[None,])
if not isinstance(rg,(list,tuple)):
import cdms2, sys, cdutil, os, cdat_info
f = cdms2.open(os.path.join(cdat_info.get_sampledata_path(), "navy_land.nc"))
navy_frac = f("sftlf") / 100.
target = cdms2.open(os.path.join(cdat_info.get_sampledata_path(),
'clt.nc'))("clt", slice(0, 1)).getGrid()
mask = cdutil.generateLandSeaMask(target, navy_frac)
target = cdms2.open(os.path.join(cdat_info.get_sampledata_path(),
'clt.nc'))("clt", slice(0, 1))
mask = cdutil.generateLandSeaMask(target, navy_frac)
target = cdms2.createGaussianGrid(64)
mask = cdutil.generateLandSeaMask(target)
target = cdms2.open(os.path.join(cdat_info.get_sampledata_path(),
'clt.nc'))("clt",
slice(0, 1),
latitude=(15, 85),
longitude=(-175, -65)).getGrid()
mask = cdutil.generateLandSeaMask(target)
#import vcs
#x=vcs.init()
#x.plot(mask)
#raw_input()
try:
sftlf[model_version] = {"raw": sft.get("sftlf")}
sftlf[model_version]["filename"] = os.path.basename(sft())
sftlf[model_version]["md5"] = sft.hash()
except:
# Hum no sftlf...
dup("No mask for ", sft())
sftlf[model_version] = {"raw": None}
sftlf[model_version]["filename"] = None
sftlf[model_version]["md5"] = None
if parameters.targetGrid == "2.5x2.5":
tGrid = cdms2.createUniformGrid(-88.875, 72, 2.5, 0, 144, 2.5)
else:
tGrid = parameters.targetGrid
sft = cdutil.generateLandSeaMask(tGrid)
sft[:] = sft.filled(1.) * 100.
sftlf["targetGrid"] = sft
# At this point we need to create the tuples var/region to know if a
# variable needs to be ran over a specific region or global or both
regions = getattr(parameters, "regions", {})
vars = []
# Update/overwrite defsult region_values keys with user ones
regions_values.update(getattr(parameters, "regions_values", {}))
regions_dict = {}
for var in parameters.vars:
sft.ext = "nc"
sft.targetGrid = None
sft.realization="r0i0p0"
applyCustomKeys(sft,parameters.custom_keys,"sftlf")
try:
sftlf[model_version] = {"raw":sft.get("sftlf")}
except:
#Hum no sftlf...
dup("No mask for ",sft())
sftlf[model_version] = {"raw":None}
if parameters.targetGrid == "2.5x2.5":
tGrid = cdms2.createUniformGrid(-88.875,72,2.5,0,144,2.5)
else:
tGrid = parameters.targetGrid
sftlf["targetGrid"] = cdutil.generateLandSeaMask(tGrid)*100.
#At this point we need to create the tuples var/region to know if a variable needs to be ran over a specific region or global or both
regions = getattr(parameters,"regions",{})
vars = []
#Update/overwrite defsult region_values keys with user ones
regions_values.update(getattr(parameters,"regions_values",{}))
regions_dict = {}
for var in parameters.vars:
vr = var.split("_")[0]
rg = regions.get(vr,[None,])
if not isinstance(rg,(list,tuple)):
template_sftlf.model = model
S = cdms2.open(file_S)("S", region)
print 'Reading Phase from %s ...' % os.path.join(
args.modroot, template_tS())
tS = cdms2.open(os.path.join(args.modroot, template_tS()))("tS",
region)
print 'Reading sftlf from %s ...' % os.path.join(
args.modroot, template_sftlf())
try:
sftlf_fnm = glob.glob(os.path.join(args.modroot,
template_sftlf()))[0]
sftlf = cdms2.open(sftlf_fnm)("sftlf", region) / 100.
except BaseException as err:
print 'Failed reading sftlf from file (error was: %s)' % err
print 'Creating one for you'
sftlf = cdutil.generateLandSeaMask(S.getGrid())
if model not in stats_dic:
stats_dic[model] = {region_name: spacevavg(S, tS, sftlf, model)}
else:
stats_dic[model].update(
{region_name: spacevavg(S, tS, sftlf, model)})
print stats_dic
except Exception as err:
print "Failed for model %s with error %s" % (model, err)
# Write output to JSON file.
metrics_dictionary["RESULTS"] = stats_dic
rgmsk = metrics_dictionary.get("RegionalMasking", {})
nm = region_name
region.id = nm
# Adapted for numpy/ma/cdms2 by convertcdms.py
"""
This script converts a land sea mask to a stbyrgn mask
Input:
Land/sea mask
Original sftbyrgn
dctionary of values in sftbyrgn/type (ld or water)
Ouptput:
Newsftbyrgn
"""
import cdms2,cdutil,MV2,os,sys,cdat_info
bg=0
din=cdms2.open(os.path.join(cdat_info.get_prefix(),"sample_data","clt.nc"))("clt",slice(0,1))
print 'generating mask'
sftlf = cdutil.generateLandSeaMask(din)*100.
print 'done:',sftlf.shape
newsftbyrgn,n=cdutil.generateSurfaceTypeByRegionMask(sftlf)
#import vcs
#x=vcs.init()
#x.plot(newsftbyrgn,bg=bg)
#x.clear()
#x.plot(n,bg=bg)
#raw_input()
def generateSurfaceTypeByRegionMask(mask,sftbyrgn=None,sftbyrgnmask=215,regions=range(201,223),maximum_regions_per_cell=4,extend_up_to=3,verbose=True):
"""
Maps a "regions" dataset onto a user provided land/sea mask or grid
Usage:
-----
mapped,found = generateSurfaceTypeByRegionMask(mask,sftbyrgn=None,sftbyrgnmask=None,regions=None,maximum_regions_per_cell=4,extend_up_to=3,verbose=True)
Input:
-----
mask User provided land/sea mask (100/0) or grid (the land/sea mask will be generated automagically) which will be mapped using the "sftbyrgn" internal dataset (will generate a land/sea mask for you)
sftbyrgn Mask you wish to map onto your grid (if None uses internal "sftbyrgn" dataset (old ezget type))
sftbyrgnmask Land/sea mask for sftbyrgn (or a number specifying value limits for sftbyrgn which indicates land/sea threshold (greater values are land) - see URL below for integer region map)
regions Numbers from sftbyrgn array that you want to map onto mask (integers from 201-222)
maximum_regions_per_cell Maximum number of regions considered for a single cell
extend_up_to How many grid cells around a cell can we extend to identify a guess
verbose Prints to the screen what's going on (default is True)
Output:
-----
mapped Mapped input grid/mask using provided (or default) regions - sftbyrgn -> user provided grid/mask
found Matrix containing number of regions matched for each output cell
Notes:
-----
- More detailed information, including a region map and tabulated region numbers are available from http://www-pcmdi.llnl.gov/publications/pdf/34.pdf
"""
cdat_info.pingPCMDIdb("cdat","cdutil.generateSurfaceTypeByRegionMask")
## OK first determine which regions are available
## Must be integer values
if isinstance(mask,cdms2.grid.TransientRectGrid):
mask = cdutil.generateLandSeaMask(mask)*100.
if sftbyrgn is None:
sftbyrgn = cdms2.open(os.path.join(cdat_info.get_prefix(),'share','cdutil','sftbyrgn.nc'))('sftbyrgn')
if regions is None:
if verbose: print 'Preparing regions'
#regions = range(201,223)
regions = []
for i in range(0,10000):
genutil.statusbar(i,9999)
c = float(MV2.sum(MV2.ravel(MV2.equal(sftbyrgn,i)),0))
if c != 0: regions.append(i)
if verbose: print 'Regions:',regions
## If no mask passed fr sftbyrgn, assumes everything greater 5000 is land)
if isinstance(sftbyrgnmask,int):
split = sftbyrgnmask
n = MV2.maximum(mask)
sftbyrgnmask = MV2.greater_equal(sftbyrgn,sftbyrgnmask)*n
else:
split = MV2.maximum(sftbyrgnmask)/2.
## Now guess the type for each regions
keys = {}
## ## Nice way to do it
## for r in regions:
## c=MV2.not_equal(sftbyrgn,r)
## c=MV2.masked_where(c,sftbyrgnmask)
## n=MV2.count(c)
## c=float(MV2.sum(MV2.ravel(c),0)/n)
## print r,c,n
## keys[r]=c
## Fast but not so "general" way to do it
for r in regions:
if r< split:
keys[r] = 0.
else:
keys[r] = 100.
sh = list(mask.shape)
sh.insert(0,maximum_regions_per_cell)
potential = MV2.ones(sh,dtype='d')*-999
potential_reg = MV2.ones(sh,dtype='d')*-999
g1 = sftbyrgn.getGrid()
g2 = mask.getGrid()
r1 = regrid2.Horizontal(g1,g2)
w = cdutil.area_weights(sftbyrgn)
if verbose: print 'First pass'
itmp = 0.
for ireg in keys.keys():
genutil.statusbar(itmp,len(keys.keys())-1)
itmp += 1.
c = MV2.equal(sftbyrgn,ireg)
w2 = 1.-c*w
s2,w3 = r1(sftbyrgn,mask=w2.filled(),returnTuple=1)
c2 = MV2.equal(mask,keys[ireg])
loop(potential,potential_reg,c2,w3,ireg)
found = MV2.zeros(sh[1:],typecode='f')
for i in range(maximum_regions_per_cell):
found = found+MV2.not_equal(potential[i],-999)
sh2 = list(sh)
for k in range(extend_up_to):
sh2[1] = sh[1]+2*(k+1)
sh2[2] = sh[2]+2*(k+1)
## Form the possible i/j couples !
s = MV2.sum(MV2.ravel(MV2.equal(potential[0],-999)),0)
if verbose: print 'Expanding up to',k+1,'cells while trying to fix',s,'cells'
#if dump:
#f=cdms2.open('tmp_'+str(k)+'.nc','w')
#f.write(sumregions(potential_reg,potential).astype('f'),id='sftbyrgn',axes=mask.getAxisList())
#f.close()
#g=sumregions(potential_reg,potential).astype('d')
#g=MV2.masked_equal(g,-999)
#g=MV2.greater(g,4999)*100.
#g=MV2.absolute(mask-g)
#g=MV2.masked_equal(g,0.)
#print 'Number of differences:',MV2.count(g)
if float(s) != 0:
c0 = MV2.equal(potential[0],-999)
couples = []
sft2 = MV2.zeros(sh2[1:],dtype='d')-888.
sft2[k+1:-k-1,k+1:-k-1] = mask
for i in range(-k-1,k+2):
for j in range(-k-1,k+2):
if abs(i)>k or abs(j)>k: couples.append([i,j])
ntot = len(keys.keys())*len(couples)-1
itmp = 0
for ireg in keys.keys():
c = MV2.equal(sftbyrgn,ireg)
w2 = 1.-c*w
s2,w3 = r1(sftbyrgn,mask=w2.filled(),returnTuple=1)
w4 = MV2.zeros(sh2[1:],typecode='d')
w4[k+1:-k-1,k+1:-k-1] = w3
for i,j in couples:
if verbose: genutil.statusbar(itmp,ntot)
itmp += 1.
c2 = MV2.equal(sft2[j+k+1:j+k+1+sh[1],i+k+1:i+k+1+sh[2]],keys[ireg])
c3 = MV2.equal(sft2[j+k+1:j+k+1+sh[1],i+k+1:i+k+1+sh[2]],mask)
c2 = MV2.logical_and(c2,c3)
c2 = MV2.logical_and(c2,c0)
loop(potential,potential_reg,c2,w4[j+k+1:j+k+1+sh[1],i+k+1:i+k+1+sh[2]],ireg)
found = MV2.where(MV2.equal(potential[0],-999),found-1,found)
out = sumregions(potential_reg,potential)
out.setAxisList(mask.getAxisList())
out.id = 'sftbyrgn'
out = out.astype('i')
out.missing_value = -999
found.setAxisList(mask.getAxisList())
found.id = 'found'
found = found.astype('i')
found.missing_value = -999
del(out.name)
del(found.name)
return out,found
def Avg_PS_DomFrq(d, frequency, ntd, dat, mip, frc):
"""
Domain and Frequency average of spectral power
Input
- d: spectral power with lon, lat, and frequency dimensions (cdms)
- ntd: number of time steps per day (daily: 1, 3-hourly: 8)
- frequency: frequency
- dat: model name
- mip: mip name
- frc: forced or unforced
Output
- psdmfm: Domain and Frequency averaged of spectral power (json)
"""
domains = [
"Total_50S50N",
"Ocean_50S50N",
"Land_50S50N",
"Total_30N50N",
"Ocean_30N50N",
"Land_30N50N",
"Total_30S30N",
"Ocean_30S30N",
"Land_30S30N",
"Total_50S30S",
"Ocean_50S30S",
"Land_50S30S",
]
if ntd == 8: # 3-hourly data
frqs_forced = ["semi-diurnal", "diurnal", "semi-annual", "annual"]
frqs_unforced = [
"sub-daily",
"synoptic",
"sub-seasonal",
"seasonal-annual",
"interannual",
]
elif ntd == 1: # daily data
frqs_forced = ["semi-annual", "annual"]
frqs_unforced = [
"synoptic", "sub-seasonal", "seasonal-annual", "interannual"
]
else:
sys.exit("ERROR: ntd " + ntd + " is not defined!")
if frc == "forced":
frqs = frqs_forced
elif frc == "unforced":
frqs = frqs_unforced
else:
sys.exit("ERROR: frc " + frc + " is not defined!")
mask = cdutil.generateLandSeaMask(d[0])
d, mask2 = genutil.grower(d, mask)
d_ocean = MV.masked_where(mask2 == 1.0, d)
d_land = MV.masked_where(mask2 == 0.0, d)
psdmfm = {}
for dom in domains:
psdmfm[dom] = {}
if "Ocean" in dom:
dmask = d_ocean
elif "Land" in dom:
dmask = d_land
else:
dmask = d
if "50S50N" in dom:
am = cdutil.averager(dmask(latitude=(-50, 50)), axis="xy")
if "30N50N" in dom:
am = cdutil.averager(dmask(latitude=(30, 50)), axis="xy")
if "30S30N" in dom:
am = cdutil.averager(dmask(latitude=(-30, 30)), axis="xy")
if "50S30S" in dom:
am = cdutil.averager(dmask(latitude=(-50, -30)), axis="xy")
am = np.array(am)
for frq in frqs:
if frq == "semi-diurnal": # pr=0.5day
idx = prdday_to_frqidx(0.5, frequency, ntd)
amfm = am[idx]
elif frq == "diurnal": # pr=1day
idx = prdday_to_frqidx(1, frequency, ntd)
amfm = am[idx]
elif frq == "semi-annual": # 180day=<pr=<183day
idx2 = prdday_to_frqidx(180, frequency, ntd)
idx1 = prdday_to_frqidx(183, frequency, ntd)
amfm = np.amax(am[idx1:idx2 + 1])
elif frq == "annual": # 360day=<pr=<366day
idx2 = prdday_to_frqidx(360, frequency, ntd)
idx1 = prdday_to_frqidx(366, frequency, ntd)
amfm = np.amax(am[idx1:idx2 + 1])
elif frq == "sub-daily": # pr<1day
idx1 = prdday_to_frqidx(1, frequency, ntd)
amfm = cdutil.averager(am[idx1 + 1:], weights="unweighted")
elif frq == "synoptic": # 1day=<pr<20day
idx2 = prdday_to_frqidx(1, frequency, ntd)
idx1 = prdday_to_frqidx(20, frequency, ntd)
amfm = cdutil.averager(am[idx1 + 1:idx2 + 1],
weights="unweighted")
elif frq == "sub-seasonal": # 20day=<pr<90day
idx2 = prdday_to_frqidx(20, frequency, ntd)
idx1 = prdday_to_frqidx(90, frequency, ntd)
amfm = cdutil.averager(am[idx1 + 1:idx2 + 1],
weights="unweighted")
elif frq == "seasonal-annual": # 90day=<pr<365day
idx2 = prdday_to_frqidx(90, frequency, ntd)
idx1 = prdday_to_frqidx(365, frequency, ntd)
amfm = cdutil.averager(am[idx1 + 1:idx2 + 1],
weights="unweighted")
elif frq == "interannual": # 365day=<pr
idx2 = prdday_to_frqidx(365, frequency, ntd)
amfm = cdutil.averager(am[:idx2 + 1], weights="unweighted")
psdmfm[dom][frq] = amfm.tolist()
print("Complete domain and frequency average of spectral power")
return psdmfm
def testSftbyrgn(self):
din = cdms2.open(
os.path.join(cdat_info.get_sampledata_path(),
"clt.nc"))("clt", slice(0, 1))
sftlf = cdutil.generateLandSeaMask(din) * 100.
newsftbyrgn, n = cdutil.generateSurfaceTypeByRegionMask(sftlf)