def compute(self, dataset_pool):
constants = dataset_pool.get_dataset('constants')
footprint = constants["FOOTPRINT"]
covertypes_of_interest = self._cover_type_translation(constants)
lct = ma.filled(self.get_dataset().get_2d_attribute(self.land_cover_type), 0)
is_lct_of_interest = reduce(lambda prev_answer, lct_num: logical_or(prev_answer, lct==lct_num),
covertypes_of_interest,
zeros(shape=lct.shape, dtype=int32))
# maxg: maximum number of possible like-adjacencies within a window
# a_i: # of cells of target covertypes within moving window
a_i = correlate(ma.filled(is_lct_of_interest.astype(int32), 0),
footprint, mode="reflect", cval=0.0)
n = floor(sqrt(a_i))
m = a_i - power(n, 2)
maxg = 2*n*(n-1)
maxg += where(m==0, 0, where(m<=n, (2*m-1), where(m>n, (2*m-2), 0)))
# detect all adjacencies by systematically checking each possibility
# within the 5x5 footprint window
gii = ma.masked_array(zeros(shape=lct.shape, dtype=int32))
for ri in range(0, footprint.shape[0]):
for ci in range(0, footprint.shape[1]-1):
gii += self._find_adjacency(is_lct_of_interest, (ri,ci), (ri,ci+1), footprint)
for ri in range(0, footprint.shape[0]-1):
for ci in range(0, footprint.shape[1]):
gii += self._find_adjacency(is_lct_of_interest, (ri,ci), (ri+1,ci), footprint)
#assert ma.alltrue(ravel(gii<=maxg))
return self.get_dataset().flatten_by_id(arcsin(sqrt(ma.filled(gii/maxg, 0))))
def compute(self, dataset_pool):
constants = dataset_pool.get_dataset('constants')
footprint = constants["FOOTPRINT"]
lct = ma.filled(self.get_dataset().get_2d_attribute(self.land_cover_types), 0)
temp = equal(lct, constants[self.lct_type.upper()])
values = correlate(temp.astype(int32), footprint, mode="reflect")
return self.get_dataset().flatten_by_id(ma.filled(values, 0))
def ncf2ffi1001(f, outpath, mode = 'w'):
outfile = open(outpath, mode)
header_keys = "PI_CONTACT_INFO PLATFORM LOCATION ASSOCIATED_DATA INSTRUMENT_INFO DATA_INFO UNCERTAINTY ULOD_FLAG ULOD_VALUE LLOD_FLAG LLOD_VALUE DM_CONTACT_INFO PROJECT_INFO STIPULATIONS_ON_USE OTHER_COMMENTS REVISION".split()
print('%d, %d' % (len(f.ncattrs()) + len(f.variables), 1001), file = outfile)
print(getattr(f, 'PI_NAME', 'Unknown'), file = outfile)
print(getattr(f, 'ORGANIZATION_NAME', 'Unknown'), file = outfile)
print(getattr(f, 'SOURCE_DESCRIPTION', 'Unknown'), file = outfile)
print(getattr(f, 'VOLUME_INFO', 'Unknown'), file = outfile)
print(f.SDATE, f.WDATE, file = outfile)
print(f.TIME_INTERVAL, file = outfile)
print(f.INDEPENDENT_VARIABLE, file = outfile)
print('%d' % len(f.variables), file = outfile)
for key, var in f.variables.items():
print('%s, %s' % (key, getattr(var, 'units', 'unknown')), file = outfile)
print(len(f.ncattrs()), file = outfile)
for key in f.ncattrs():
print('%s: %s' % (key, getattr(f, key, '')), file = outfile)
vals = [filled(f.variables[f.INDEPENDENT_VARIABLE][:]).ravel()]
keys = [f.INDEPENDENT_VARIABLE]
for key, var in f.variables.items():
if key == f.INDEPENDENT_VARIABLE: continue
keys.append(key)
vals.append(filled(var[:]).ravel())
print(', '.join(keys), file = outfile)
for row in array(vals).T:
row.tofile(outfile, format = '%.6e', sep = ', ')
print('', file = outfile)
def extract(self,PSF):
#Placeholder
pixsig = 1.
sh = self.sh
npsf = PSF.size
flux = np.zeros(sh)
uflux = np.zeros(sh)
sky1d = np.zeros(sh)
usky1d = np.zeros(sh)
im = self.Order.image_rect
uim = self.Order.uimage_rect
sky = self.Order.sky_rect
usky = self.Order.usky_rect
for i in np.arange(sh):
# flux[i] = (PSF*im[i,:]).sum() / (PSF**2).sum()
flux[i] = (PSF*im[i,:]/uim[i,:]**2).sum() / (PSF**2/uim[i,:]**2).sum()
uflux[i] = np.sqrt(1.0/(PSF**2/uim[i,:]**2.).sum())
sky1d[i] = (np.abs(PSF)*sky[i,:]).sum() / (PSF**2).sum()
usky1d[i] = np.sqrt(1.0/(PSF**2/usky[i,:]**2.).sum())
flux = ma.masked_invalid(flux)
flux = ma.filled(flux,1.)
uflux = ma.masked_invalid(uflux)
uflux = ma.filled(uflux,1000.)
sky1d = ma.masked_invalid(sky1d)
sky1d = ma.filled(sky1d,1.)
usky1d = ma.masked_invalid(usky1d)
usky1d = ma.filled(usky1d,1000.)
sky_cont = self._fitCont(self.wave,sky1d)
return flux,uflux,sky1d-sky_cont,usky1d
def _compute_shannon_evenness_index(self, lct, footprint, lct_mask):
"""Compute Shannon's evenness index"""
cover_types_list = self._get_cover_types(lct)
numerator_sum = ma.masked_array(zeros(shape=lct.shape, dtype=float32),
mask=lct_mask)
for cover_type in cover_types_list:
pi = self._compute_pct_cover_type_within_footprint(lct, cover_type, footprint, lct_mask)
numerator_sum += pi*ma.filled(ma.log(pi), 0)
m = self._count_covertypes_within_window(lct, cover_types_list, footprint)
return ma.filled(-numerator_sum / ma.log(m), 0).astype(float32)
def compute(self, dataset_pool):
ds = self.get_dataset()
age = ds.get_attribute(self.building_age)
result = ma.filled(age, -1)
mask = ma.getmask(age)
if mask is not ma.nomask and mask.any():
# If there are bad ages then mask won't be ma.nomask and will have some non-False values. In this case compute
# the average age for all buildings, then find the ages of buildings within walking distance, and use those
# to fill in the bad values.
whole_area_average_age = int(age.sum()/float(ds.size() - age.mask.sum()))
age_wwd_values = ma.filled(ds.get_attribute(self.age_wwd), whole_area_average_age)
result[mask] = age_wwd_values[mask]
return result
def reduce (self, target, axis=0):
"""Reduce target along the given axis."""
axes, attributes, id, grid = _extractMetadata(target, omit=axis)
a = _makeMaskedArg(target)
m = getmask(a)
if m is nomask:
t = filled(a)
return masked_array (numpy.maximum.reduce (t, axis))
else:
t = numpy.maximum.reduce(filled(a, numpy.ma.maximum_fill_value(a)), axis)
m = numpy.logical_and.reduce(m, axis)
return TransientVariable(t, mask=m, fill_value=fill_value(a),
axes = axes, grid=grid, id=id)
def compute(self, dataset_pool):
constants = dataset_pool.get_dataset('constants')
fs = ma.filled(self.get_dataset().get_2d_attribute(self.footprint_size).astype(float32), 0)
lct = ma.filled(self.get_dataset().get_2d_attribute(self.land_cover_type), 0)
x = zeros(shape=lct.shape, dtype=float32)
max_type = int(maximum.reduce(ravel(lct)))
for itype in range(1, max_type+1):
temp = equal(lct, itype).astype(int32)
summed = correlate(ma.filled(temp, 0.0),
constants['FOOTPRINT'],
mode="reflect")
x += temp * ma.filled(summed / ma.masked_where(fs==0, fs), 0.0)
return self.get_dataset().flatten_by_id(arcsin(sqrt(x)))
def calcDeltaMags():
'''
Creates an array with the delta magnitudes of WDS objects.
'''
# Calculating magnitude difference for each object
# Make nice formats...
Pri_mag_g = ma.filled(wdsInteresting[priMag],[-999])
Sec_mag_g = ma.filled(wdsInteresting[secMag],[-999])
Pri_mag_gg = np.asarray(Pri_mag_g, dtype = 'float_' )
Sec_mag_gg = np.asarray(Sec_mag_g, dtype = 'float_' )
# Then calculate the actual delta mags
Delta_mag = np.subtract(Pri_mag_gg, Sec_mag_gg)
return Delta_mag
def outer (self, a, b):
"Return the function applied to the outer product of a and b."
a = _makeMaskedArg(a)
b = _makeMaskedArg(b)
ma = getmask(a)
mb = getmask(b)
if ma is nomask and mb is nomask:
m = None
else:
ma = getmaskarray(a)
mb = getmaskarray(b)
m = logical_or.outer(ma, mb)
d = numpy.maximum.outer(filled(a), filled(b))
return TransientVariable(d, mask=m)
def test_testMasked(self):
# Test of masked element
xx = arange(6)
xx[1] = masked
self.assertTrue(str(masked) == '--')
self.assertTrue(xx[1] is masked)
self.assertEqual(filled(xx[1], 0), 0)
def compute_lambda_for_vacancy(grouping_location_set, units_variable, vacant_units_variable, movers_variable,
secondary_residence_variable=None, multiplicator=1.0, resources=None):
grouping_location_set.compute_variables([units_variable, vacant_units_variable, movers_variable], resources=resources)
if secondary_residence_variable is not None:
grouping_location_set.compute_variables([secondary_residence_variable], resources=resources)
unitssecondary = grouping_location_set.get_attribute(secondary_residence_variable)
else:
unitssecondary = zeros(grouping_location_set.size())
unitsvacant = grouping_location_set.get_attribute(vacant_units_variable)
units = grouping_location_set.get_attribute(units_variable)
movers = grouping_location_set.get_attribute(movers_variable)
tsv = units - unitssecondary - unitsvacant
lambda_value = ma.filled((units - unitssecondary).astype(float32)/ ma.masked_where(tsv==0, tsv),0) \
- ma.filled(unitsvacant.astype(float32) / ma.masked_where(movers==0, movers),0)
lambda_value = lambda_value * multiplicator
return lambda_value
def compute(self, dataset_pool):
hh_wwd = self.get_dataset().get_attribute(self.number_of_households_within_walking_distance)
return 100.0 * ma.filled(
self.get_dataset().get_attribute(self.number_of_young_households_within_walking_distance)
/ ma.masked_where(hh_wwd == 0, hh_wwd.astype(float32)),
0.0,
)
def compute(self, dataset_pool):
constants = dataset_pool.get_dataset('constants')
footprint = constants["FOOTPRINT"]
lct = ma.filled(self.get_dataset().get_2d_attribute(self.land_cover_type), 0)
lct_mask = self.get_dataset().get_mask(is_2d_version=True)
shei = self._compute_shannon_evenness_index(lct, footprint, lct_mask)
return self.get_dataset().flatten_by_id(arcsin(sqrt(shei)))
def __setslice__(self, i, j, value):
"Sets the slice described by [i,j] to `value`."
_localdict = self.__dict__
d = self._data
m = _localdict['_fieldmask']
names = self.dtype.names
if value is masked:
for n in names:
m[i:j][n] = True
elif not self._hardmask:
fval = filled(value)
mval = getmaskarray(value)
for n in names:
d[n][i:j] = fval
m[n][i:j] = mval
else:
mindx = getmaskarray(self)[i:j]
dval = np.asarray(value)
valmask = getmask(value)
if valmask is nomask:
for n in names:
mval = mask_or(m[n][i:j], valmask)
d[n][i:j][~mval] = value
elif valmask.size > 1:
for n in names:
mval = mask_or(m[n][i:j], valmask)
d[n][i:j][~mval] = dval[~mval]
m[n][i:j] = mask_or(m[n][i:j], mval)
self._fieldmask = m
def __init__(self, MetricTable, opts):
epsilon = 0.0
if opts.cpu:
epsilon = 0.01
# Create empty ratio table
nprobs = MetricTable.nprobs
nsolvs = MetricTable.nsolvs
self.ratios = ma.zeros((nprobs, nsolvs), dtype=numpy.float)
# Compute best relative performance ratios across
# solvers for each problem
for prob in range(nprobs):
metrics = MetricTable.prob_mets(prob) + epsilon
best_met = ma.minimum(metrics)
self.ratios[prob,:] = metrics * (1.0 / best_met)
# Sort each solvers performance ratios
for solv in range(nsolvs):
self.ratios[:,solv] = ma.sort(self.ratios[:,solv])
# Compute largest ratio and use to replace failure entries
self.maxrat = ma.maximum(self.ratios)
self.ratios = ma.filled(self.ratios, 10 * self.maxrat)
def run_chunk(self, agents_index, agent_set, specification, coefficients):
# unplaced agents in agents_index
location_id_name = self.choice_set.get_id_name()[0]
agent_set.set_values_of_one_attribute(location_id_name, resize(array([-1]), agents_index.size),
agents_index)
## capacity may need to be re-computed for every chunk
if self.compute_capacity_flag:
self.capacity = ma.filled(self.determine_capacity(capacity_string=self.run_config.get("capacity_string", None),
agent_set=agent_set,
agents_index=agents_index),
0.0)
if self.capacity is not None:
logger.log_status("Available capacity: %s units." % self.capacity.sum())
self.run_config.merge({"capacity":self.capacity})
if self.run_config.get("agent_units_string", None):
self.run_config["agent_units_all"] = agent_set.get_attribute_by_index(self.run_config["agent_units_string"], agents_index)
choices = ChoiceModel.run_chunk(self, agents_index, agent_set, specification, coefficients)
## this is done in choice_model
#modify locations
#agent_set.set_values_of_one_attribute(location_id_name, choices, agents_index)
if self.run_config.has_key("capacity"):
del self.run_config["capacity"]
return choices
def compute(self, dataset_pool):
cellsize = dataset_pool.get_dataset('constants')['CELLSIZE']
fpdimension = int(self.footprint_width / cellsize)
fp = ones((fpdimension, fpdimension), dtype="int32")
summed = correlate( ma.filled( self.get_dataset().get_2d_attribute( self.comm_add ), 0.0 ), \
fp, mode="reflect" )
return ln(self.get_dataset().flatten_by_id(summed)+1)/10.0
def mag_to_flux(mag, limMag=False):
"""
Converts magnitudes to fluxes using the following law (from Kessler+2010):
flux = 10^(-0.4 * m + 11)
If the magnitude is above the limit of the instrument, returns the
corresponding limiting flux.
INPUT:
mag: numpy array of magnitude
limMag: specifies the limiting magnitude
OUTPUT:
flux: flux-converted magnitudes
"""
if limMag:
exponent = (-0.4 * limMag) + 11
limFlux = np.power(10, exponent)
# masked arrays if magnitude is grater then limit
maMag = ma.masked_where(mag > limMag, mag)
maExponent = (-0.4 * maMag) + 11
maFlux = np.power(10, exponent)
maFlux.set_fill_value(limFlux)
flux = ma.filled(maFlux)
else:
exponent = (-0.4 * mag) + 11
flux = 10**exponent
return flux
def compute(self, dataset_pool):
recent_years = dataset_pool.get_dataset('urbansim_constant')["recent_years"]
buildings = dataset_pool.get_dataset('building')
age = buildings.get_attribute("building_age")
values = buildings.get_attribute(self.units)
values = where(ma.filled(age, recent_years+1) <= recent_years, values, 0)
return self.get_dataset().sum_over_ids(buildings.get_attribute("grid_id"), values)
def create_gap_mask(self, wave):
"""
Use the gap configuration and a wavelength vector, wave, to build a masked array that
masks out the location of the gap. Wavelengths between and including both gap endpoints
will be masked out.
Parameters
----------
wave: numpy.ndarray
Wavelength vector to construct mask from
Returns
-------
mask: numpy.ma 1D masked array
1D array masked at the locations within the configured detector gap and 1.0 elsewhere
"""
disperser = self.instrument['disperser']
aperture = self.instrument['aperture']
filt = self.instrument['filter']
gap = self.aperture_config[aperture]['gap'][disperser]
if filt in gap:
gap_start = gap[filt]['gap_start']
gap_end = gap[filt]['gap_end']
else:
gap_start = gap['gap_start']
gap_end = gap['gap_end']
if gap_start is not None and gap_end is not None:
masked_wave = ma.masked_inside(wave, gap_start, gap_end)
mask = masked_wave / wave
mask = ma.filled(mask, 0.0)
else:
mask = 1.0
return mask
def run(self, utilities, resources=None):
""" Compute probabilities of a discrete choice model from the given utitlities.
'utilities' is a 2D array (nobservations x nequations).
The return value is a 2D array (same shape as utilities).
"""
if utilities.ndim < 2:
raise StandardError, "Argument 'utilities' must be a 2D numpy array."
util_min = utilities.min()
util_max = utilities.max()
if (util_min < self.computable_range[0]) or (util_max > self.computable_range[1]):
# shift utilities to zero (maximum is at zero)
to_be_transformed=where((utilities < self.computable_range[0]) + (utilities > self.computable_range[1]))
to_be_transformed=unique(to_be_transformed[0])
for idx in arange(to_be_transformed.size):
i = to_be_transformed[idx]
this_max = utilities[i,:].max()
utilities[i,:]=utilities[i,:]-this_max
availability = resources.get('availability', None)
if availability is None:
exponentiated_utility = exp(utilities)
else:
exponentiated_utility = exp(utilities) * (availability).astype('b')
sum_exponentiated_utility = sum(exponentiated_utility, axis=1, dtype="float64")
sum_exponentiated_utility = ma.masked_where(sum_exponentiated_utility==0, sum_exponentiated_utility)
return ma.filled(exponentiated_utility/reshape(sum_exponentiated_utility,(utilities.shape[0], 1)), 0)
def __init__(self, MetricTable):
# Create empty ratio table
nprobs = MetricTable.nprobs
nsolvs = MetricTable.nsolvs
self.ratios = ma.masked_array(1.0 * ma.zeros((nprobs+1, nsolvs)))
# Compute best relative performance ratios across
# solvers for each problem
for prob in range(nprobs):
metrics = MetricTable.prob_mets(prob)
best_met = ma.minimum(metrics)
if (ma.count(metrics)==nsolvs and
ma.maximum(metrics)<=opts.minlimit):
self.ratios[prob+1,:] = 1.0;
else:
self.ratios[prob+1,:] = metrics * (1.0 / best_met)
# Sort each solvers performance ratios
for solv in range(nsolvs):
self.ratios[:,solv] = ma.sort(self.ratios[:,solv])
# Compute largest ratio and use to replace failures entries
self.maxrat = ma.maximum(self.ratios)
self.ratios = ma.filled(self.ratios, 1.01 * self.maxrat)
def compute(self, dataset_pool):
sqft = self.get_dataset().get_attribute(self.sqft)
num_of_job_spaces = ma.filled(self.get_dataset().get_attribute(self.commercial_sqft)/
ma.masked_where(sqft == 0, sqft.astype(float32)), 0.0).astype(int32)
return clip_to_zero_if_needed(num_of_job_spaces -
self.get_dataset().get_attribute(self.number_of_commercial_jobs),
'vacant_commercial_job_space')
def get_average_omega(self, omega, probability, index, nsupply, demand):
omega_prob = omega[:, newaxis]*probability
omega_prob_sum_over_i = array(ndimage_sum(omega_prob, labels=index+1, index=arange(nsupply)+1))
prob_sum_over_i = array(ndimage_sum(probability, labels=index+1, index=arange(nsupply)+1))
average_omega = ma.filled(omega_prob_sum_over_i/
ma.masked_where(prob_sum_over_i==0, prob_sum_over_i), 0.0)
return average_omega
def identify(cls, variables, ignore=None, target=None, warn=True, monotonic=False):
result = {}
ignore, target = cls._identify_common(variables, ignore, target)
# Identify all CF coordinate variables.
for nc_var_name, nc_var in target.iteritems():
if nc_var_name in ignore:
continue
# String variables can't be coordinates
if np.issubdtype(nc_var.dtype, np.str):
continue
# Restrict to one-dimensional with name as dimension OR zero-dimensional scalar
if not ((nc_var.ndim == 1 and nc_var_name in nc_var.dimensions) or (nc_var.ndim == 0)):
continue
# Restrict to monotonic?
if monotonic:
data = nc_var[:]
# Gracefully fill a masked coordinate.
if ma.isMaskedArray(data):
data = ma.filled(data)
if nc_var.shape == () or nc_var.shape == (1,) or iris.util.monotonic(data):
result[nc_var_name] = CFCoordinateVariable(nc_var_name, nc_var)
else:
result[nc_var_name] = CFCoordinateVariable(nc_var_name, nc_var)
return result
def compute(self, dataset_pool):
buildings = self.get_dataset()
non_residential_sqft = buildings.get_attribute("non_residential_sqft")
building_sqft_per_job = buildings.get_attribute("building_sqft_per_job")
remaining_space = clip(non_residential_sqft - buildings.get_attribute("occupied_building_sqft_by_jobs"),
0, non_residential_sqft)
return ma.filled(ma.masked_where(building_sqft_per_job==0, remaining_space / building_sqft_per_job), 0)
def flux_to_mag(flux, limFlux=False):
"""
Converts fluxes to magnitudes using the following law (from Kessler+2010):
flux = 10^(-0.4 * m + 11) => m = -2.5 * (log_10(flux) - 11)
If the flux is below the limit of the instrument, returns the
corresponding limiting magnitude.
INPUT:
flux: numpy array of fluxes
limFlux: specifies the limiting flux
OUTPUT:
mag: magnitude-converted fluxes
"""
if limFlux:
limMag = -2.5 * (-11 + np.log10(limFlux))
# applying the mask to detection below the limiting flux
maFlux = ma.masked_where(flux < limFlux, flux)
# to avoid warnings due to values passed to np.log10
# fluxMask = maFlux.mask
# maMag = -2.5 * (-11.0 + np.log10(ma.filled(maFlux,1)))
maMag = -2.5 * (-11.0 + np.log10(maFlux))
mag = ma.filled(maMag, limMag)
else:
if flux > 0:
mag = -2.5 * (-11 + np.log10(flux))
else:
mag = None
return mag
def test_testMasked(self):
# Test of masked element
xx = arange(6)
xx[1] = masked
assert_(str(masked) == '--')
assert_(xx[1] is masked)
assert_equal(filled(xx[1], 0), 0)
def compute(self, dataset_pool):
nj = self.get_dataset().get_attribute(self.number_of_industrial_jobs_wwd)
sqft = self.get_dataset().get_attribute(self.industrial_sqft_within_walking_distance)
regional_average = self.get_dataset().get_attribute(self.industrial_sqft).sum() / float(
self.get_dataset().get_attribute(self.number_of_industrial_jobs).sum()
)
return where(sqft < 5000, regional_average, ma.filled(sqft / ma.masked_where(nj == 0, nj.astype(float32)), 0))
def from_surface(self, surf, zname="Z_TVDSS"):
"""Get points as X Y Value from a surface object nodes.
Note that undefined surface nodes will not be included.
Args:
surf (RegularSurface): A XTGeo RegularSurface object instance.
zname (str): Name of value column (3'rd column)
Example::
topx = RegularSurface('topx.gri')
topx_aspoints = Points()
topx_aspoints.from_surface(topx)
# alternative shortform:
topx_aspoints = Points(topx) # get an instance directly
topx_aspoints.to_file('mypoints.poi') # export as XYZ file
"""
# check if surf is instance from RegularSurface
if not isinstance(surf, xtgeo.surface.RegularSurface):
raise ValueError("Given surf is not a RegularSurface object")
val = surf.values
xc, yc = surf.get_xy_values()
coor = []
for vv in [xc, yc, val]:
vv = ma.filled(vv.flatten(order="C"), fill_value=np.nan)
vv = vv[~np.isnan(vv)]
coor.append(vv)
# now populate the dataframe:
xc, yc, val = coor # pylint: disable=unbalanced-tuple-unpacking
ddatas = OrderedDict()
ddatas[self._xname] = xc
ddatas[self._yname] = yc
ddatas[self._zname] = val
self._df = pd.DataFrame(ddatas)
self.zname = zname
def extract_snow_accum(self):
""" Retrieves snow accumulation.
Returns
-------
list
"""
nan = float('nan')
if self.snow_accum == None:
if self.tmt_site_index != ([],):
snow_accum_var = self.tmt_nc.variables["snow_depth_inches"]
snow_accum = ma.filled(snow_accum_var[self.tmt_site_index], nan).flatten()
snow_accum[snow_accum==9.9692099683868690e+36] = nan
snow_accum = snow_accum[self.current_utc_hour:]
self.snow_accum = snow_accum
else:
self.snow_accum = []
return self.snow_accum
def test_testBasic2d(self):
# Test of basic array creation and properties in 2 dimensions.
for s in [(4, 3), (6, 2)]:
(x, y, a10, m1, m2, xm, ym, z, zm, xf, s) = self.d
x.shape = s
y.shape = s
xm.shape = s
ym.shape = s
xf.shape = s
assert_(not isMaskedArray(x))
assert_(isMaskedArray(xm))
assert_equal(shape(xm), s)
assert_equal(xm.shape, s)
assert_equal(xm.size, reduce(lambda x, y: x * y, s))
assert_equal(count(xm), len(m1) - reduce(lambda x, y: x + y, m1))
assert_(eq(xm, xf))
assert_(eq(filled(xm, 1.e20), xf))
assert_(eq(x, xm))
self.setup()
def extract_road_temps(self):
""" Retrieves road temperatures.
Returns
-------
list
"""
nan = float('nan')
if self.road_temps == None:
if self.tmt_site_index != ([],):
road_temp_var = self.tmt_nc.variables[backend_sys_path.Met_vars.tmt_road_temp_var]
road_temps = ma.filled(road_temp_var[self.tmt_site_index], nan).flatten()
road_temps[road_temps==9.9692099683868690e+36] = nan
road_temps = road_temps[self.current_utc_hour:]
self.road_temps = road_temps
else:
self.road_temps = []
return self.road_temps
def _compute_patch_size_of_cover_types(self, lct, footprint):
"""Computes the mean size of all patches of covertypes of interest
that are (partially) within each cell's footprint"""
eightway_structure = ones(
(3, 3),
dtype="int32") # put this to class variable will result in error
patchsizes = zeros(shape=lct.shape, dtype=float32)
patchcount = zeros(shape=lct.shape, dtype=float32)
labels, n = label(lct, eightway_structure)
slices = find_objects(labels)
# Summing the 0/1 mask gives the patch size
for ip in range(n):
locmask = where(labels[slices[ip]] == (ip + 1), lct[slices[ip]], 0)
patchcount[slices[ip]] += locmask
patchsizes[slices[ip]] += locmask.sum() * locmask
pcount_corr = ma.masked_array(
correlate(patchcount, footprint, mode="reflect", cval=0.0))
psize_corr = correlate(patchsizes, footprint, mode="reflect", cval=0.0)
return ma.filled(psize_corr / pcount_corr, 0)
def get_mask_of_attribute(self, attribute, is_2d_version=False):
"""Get mask for lct."""
if attribute is None:
raise StandardError, "No non-attributes found. Cannot create mask."
if is_2d_version:
values = self.get_2d_attribute(attribute)
else:
values = self.get_attribute(attribute)
nodata_val = self.get_nodata_value()
missingdata_val = self.get_missing_data_value()
self.resources["NODATA_value"] = nodata_val
self.resources["MISSING_value"] = missingdata_val
values = ma.filled(values, nodata_val)
nodata_filter = where(values == nodata_val, True, False)
return logical_or(where(values == missingdata_val, True, False),
nodata_filter)
def nowyield(year, couna, counb):
bb = year - 1901
region1 = NetCDFFile(
'/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/HistoricalGLM_crop_150901.nc',
'r')
maitrop = region1.variables['maize_trop'][99, :, :]
maitemp = region1.variables['maize_temp'][99, :, :]
maitropi = region1.variables['maize_trop_irrig'][99, :, :]
maitempi = region1.variables['maize_temp_irrig'][99, :, :]
gridarea = region1.variables['area'][:, :]
maitrop = ma.masked_where(maitrop <= 0, maitrop)
maitrop = ma.filled(maitrop, fill_value=0.)
maitemp = ma.masked_where(maitemp <= 0, maitemp)
maitemp = ma.filled(maitemp, fill_value=0.)
maitropi = ma.masked_where(maitropi <= 0, maitropi)
maitropi = ma.filled(maitropi, fill_value=0.)
maitempi = ma.masked_where(maitempi <= 0, maitempi)
maitempi = ma.filled(maitempi, fill_value=0.)
maizetro = maitrop + maitropi
maizetem = maitemp + maitempi
maizetor = maitrop + maitemp
maizetoi = maitropi + maitempi
maizeto = maitrop + maitemp + maitropi + maitempi
clm = NetCDFFile(
'/scratch2/scratchdirs/tslin2/isam/cheyenne/plot/finalyield/clm/clm45his_maiscaleifyield_luc2000.nc',
'r')
clmyield = clm.variables['yield'][bb, :, :]
isam = NetCDFFile(
'/scratch2/scratchdirs/tslin2/isam/cheyenne/plot/finalyield/isam/heat/isamhis_maiscaleifyield_heat_luc2000.nc',
'r')
isamyield = isam.variables['yield'][bb, :, :]
clmyield[N.isnan(clmyield)] = -9999
clmyield = ma.masked_where(clmyield <= 0, clmyield)
clmyield = ma.masked_where(maizeto <= 0, clmyield)
clmyield = ma.filled(clmyield, fill_value=0.)
isamyield[N.isnan(isamyield)] = -9999
isamyield = ma.masked_where(isamyield <= 0, isamyield)
isamyield = ma.masked_where(maizeto <= 0, isamyield)
isamyield = ma.filled(isamyield, fill_value=0.)
return clmyield, isamyield, maizeto
def normalize(weight_array):
"""Normalize a weight_array of either 1 or 2 dimensions"""
prob_array = None
if weight_array is None:
return prob_array
# dimensions of weight array
try:
d = weight_array.ndim
except:
d = 0
lt0_index = weight_array < 0
if lt0_index.sum() > 0:
raise ValueError, "some values of rows %s in weight_array are less than 0" % where(lt0_index)[0]
if d > 2:
raise ValueError, "Can't handle a weight_array with more than 2d"
elif d == 2:
weight_array_sum = sum(weight_array, axis=1, dtype=float32)[:, newaxis]
zero_index = weight_array_sum<=0
if zero_index.sum() > 0:
logger.log_warning("Rows %s of weight_array sum to 0 or less" % where(zero_index)[0])
prob_array = ma.filled( weight_array/ma.masked_where(weight_array_sum<=0, weight_array_sum), 0.0)
else:
prob_array = weight_array/weight_array_sum
elif d == 1:
weight_array_sum = weight_array.sum()
if weight_array_sum > 0:
prob_array = weight_array / float(weight_array_sum)
else:
logger.log_warning("weight_array sums to 0 or less")
prob_array = weight_array * 0
else:
raise ValueError, "Can't handle this weight_array"
return prob_array
def Main(ZRef_File, Depth_File, N_File, Background_N):
# Define constants
Cu = 4.5
a = 1.
g = 9.8 # m/s^2
# Load in values from rasters as arrays
zref = LoadZRef(ZRef_File)
depth = LoadDepth(Depth_File)
depth = depth[:-1, :-1]
print(str(depth.max()) + ' ' + str(depth.mean()) + ' ' + str(depth.min()))
# Produced masked array of roughness
eps = Epsilon(zref, depth)
print(str(zref.max()) + ' ' + str(zref.mean()) + ' ' + str(zref.min()))
print(str(eps.max()) + ' ' + str(eps.mean()) + ' ' + str(eps.min()))
func = Func(a, eps)
n = NCalc(Cu, g, depth, func)
print(str(n.max()) + ' ' + str(n.mean()) + ' ' + str(n.min()))
# Fill masked values with default of 0.04
Background_N = float(Background_N)
n = ma.filled(n, fill_value=Background_N)
print(str(n.max()) + ' ' + str(n.mean()) + ' ' + str(n.min()))
ReadWriteRaster(ZRef_File, N_File, n)
def write(Maps, file_name, feedback=2):
"""Write a map to fits file.
Map should be a map_data.MapData object.
"""
# If only a single Map was passed, make it iterable.
if not hasattr(Maps, '__iter__'):
Maps = (Maps, )
# First create a primary with the history and such:
prihdu = pyfits.PrimaryHDU()
# Add history to the primary.
fname_abbr = ku.abbreviate_file_path(file_name)
# Add final history entry and store in the primary header.
history = base_data.merge_histories(*Maps)
history.add('Written to file.', 'File name: ' + fname_abbr)
bf.write_history_header(prihdu.header, history)
list_of_hdus = [prihdu]
for ii, Map in enumerate(Maps):
# Creat an image HDU.
map = sp.array(ma.filled(Map.data, float('nan')))
imhdu = pyfits.ImageHDU(sp.swapaxes(map, 0, 2), name='MAP%d' % ii)
# Add extra data to the HDU
for key in Map.field.iterkeys():
if Map.field_axes[key] != ():
raise ce.DataError('Only 0D data can be written to a Fits Map '
'Header.')
card = pyfits.Card(key, Map.field[key].item())
imhdu.header.ascardlist().append(card)
list_of_hdus.append(imhdu)
# Creat the HDU list and write to file.
hdulist = pyfits.HDUList(list_of_hdus)
hdulist.writeto(file_name, clobber=True)
if feedback > 0:
print 'Wrote data to file: ' + fname_abbr
def test_testCopySize(self):
# Tests of some subtle points of copying and sizing.
n = [0, 0, 1, 0, 0]
m = make_mask(n)
m2 = make_mask(m)
self.assertTrue(m is m2)
m3 = make_mask(m, copy=1)
self.assertTrue(m is not m3)
x1 = np.arange(5)
y1 = array(x1, mask=m)
self.assertTrue(y1._data is not x1)
self.assertTrue(allequal(x1, y1._data))
self.assertTrue(y1.mask is m)
y1a = array(y1, copy=0)
self.assertTrue(y1a.mask is y1.mask)
y2 = array(x1, mask=m, copy=0)
self.assertTrue(y2.mask is m)
self.assertTrue(y2[2] is masked)
y2[2] = 9
self.assertTrue(y2[2] is not masked)
self.assertTrue(y2.mask is not m)
self.assertTrue(allequal(y2.mask, 0))
y3 = array(x1 * 1.0, mask=m)
self.assertTrue(filled(y3).dtype is (x1 * 1.0).dtype)
x4 = arange(4)
x4[2] = masked
y4 = resize(x4, (8, ))
self.assertTrue(eq(concatenate([x4, x4]), y4))
self.assertTrue(eq(getmask(y4), [0, 0, 1, 0, 0, 0, 1, 0]))
y5 = repeat(x4, (2, 2, 2, 2), axis=0)
self.assertTrue(eq(y5, [0, 0, 1, 1, 2, 2, 3, 3]))
y6 = repeat(x4, 2, axis=0)
self.assertTrue(eq(y5, y6))
def Multimode(D, graph=False):
"""
Calculates the multimode of the masked array D.
graph plots the image with the differently colored regions on top of it.
It's not super optimized, it makes a new plot every time it finds a new
maximum multimode value.
"""
N = D.count()
Ny, Nx = D.shape
RawData = ma.compressed(D)
QuartileArray = [
np.percentile(RawData, p) for p in np.linspace(1, 100, 201)
]
L = ma.filled(D, 0.)
Rmax = 0
for PCT in QuartileArray:
IMG = L >= PCT
Labels = label(IMG,
output='int',
structure=[[1, 1, 1], [1, 1, 1], [1, 1, 1]])[0]
Sizes = np.sort(np.bincount(Labels.flatten()))
if len(Sizes) > 2:
Rmax = max(Rmax, Sizes[-3]**2 / float(Sizes[-2]))
if graph and (Rmax == Sizes[-3]**2 / float(Sizes[-2])):
plt.figure()
plt.imshow(D.data, cmap='gray_r', interpolation='nearest')
plt.imshow(Labels,
interpolation='nearest',
alpha=0.3,
cmap='cubehelix_r')
plt.xlim(20, Nx - 20)
plt.ylim(20, Ny - 20)
plt.yticks([])
plt.xticks([])
plt.title('Multimode')
plt.savefig('MultimodeExample.png', format='png', dpi=300)
plt.close()
return Rmax
def maskClouds1000m(self, filename):
b1_img = Image(self.b1)
b2_img = Image(self.b2)
b3_img = Image(self.b3)
b4_img = Image(self.b4)
b5_img = Image(self.b5)
b6_img = Image(self.b6)
b7_img = Image(self.b7)
bandsPaths = [
self.b1, self.b2, self.b3, self.b4, self.b5, self.b6, self.b7
]
newBandsPaths = []
for band in bandsPaths:
image = Image(band)
band_masked = image.getArray(masked=True,
lower_valid_range=0,
upper_valid_range=10,
cloud_overlay_filename=filename,
data_source=self.src)
band_masked = ma.filled(band_masked, np.nan)
image.save_band(
band_masked,
image.filename.replace("masked30m_reproject", "masked1000m"))
#newBandsPaths.append(image.filename.replace("masked30m", "masked1000m"))
newBandsPaths.append(
image.filename.replace("masked30m_reproject", "masked1000m"))
self.b1 = newBandsPaths[0]
self.b2 = newBandsPaths[1]
self.b3 = newBandsPaths[2]
self.b4 = newBandsPaths[3]
self.b5 = newBandsPaths[4]
self.b6 = newBandsPaths[5]
self.b7 = newBandsPaths[6]
def f(x):
"""Evaluate a cone around the given centre with the given gradient, i.e. a function whose value increases
linearly with distance from the centre.
Given an array of shape (num_points, num_dimensions), returns an array of shape (num_points) containing the
function values and an array of shape (num_points, num_dimensions) containing the function jacobians.
Given an array of shape (num_dimensions), returns a 0D array containing the function value and an array of shape
(num_dimensions) containing the function jacobian.
"""
distance = np.linalg.norm(x - centre, axis=-1)
value = distance * gradient
distance = np.expand_dims(distance, -1)
distance = ma.masked_equal(distance, 0) # Avoid division by zero
jacobian = (x - centre) * gradient / distance
# The jacobian isn't defined at the centre of the cone but we return a value to keep the optimiser happy.
jacobian = ma.filled(jacobian, x)
return value, jacobian
def get_xy_values(self, order="C", asmasked=False):
"""Get X Y coordinate values as numpy 2D arrays."""
nno = self.ncol * self.nrow
ier, xvals, yvals = _cxtgeo.surf_xy_as_values(
self.xori,
self.xinc,
self.yori,
self.yinc * self.yflip,
self.ncol,
self.nrow,
self.rotation,
nno,
nno,
0,
XTGDEBUG,
)
if ier != 0:
logger.critical("Error code %s, contact the author", ier)
# reshape
xvals = xvals.reshape((self.ncol, self.nrow))
yvals = yvals.reshape((self.ncol, self.nrow))
if order == "F":
xvals = np.array(xvals, order="F")
yvals = np.array(yvals, order="F")
if asmasked:
tmpv = ma.filled(self.values, fill_value=np.nan)
tmpv = np.array(tmpv, order=order)
tmpv = ma.masked_invalid(tmpv)
mymask = ma.getmaskarray(tmpv)
xvals = ma.array(xvals, mask=mymask, order=order)
yvals = ma.array(yvals, mask=mymask, order=order)
return xvals, yvals
def get_bestfit(arr, also_floats=False):
'''return smallest compatible format code for given array
>>> x = as_num_array([1234,45,888])
>>> print get_format(x), get_bestfit(x)
int32 int16
>>> y = ma.array([12,45555555,88],mask=[0,1,0])
>>> print get_format(y), get_bestfit(y)
int32 int8
'''
format = get_format(arr)
if format[:3] == 'int':
check = ['bool_', 'int8', 'int16', 'int32']
elif format[:4] == 'uint':
check = ['bool_', 'uint8', 'uint16', 'uint32']
elif format[:5] == 'float':
#TODO: Floats, tricky - need to look for small decimals and fractions
#return 'Float32' - this breaks many tests by adding fuzz at the wee digits
if also_floats:
return 'float32'
return format
else:
#TODO: CharArray squeeze
return format
if is_masked_array(arr):
# masked arrays do not have min(), max()
arr = ma.filled(arr)
xmin = arr.min()
xmax = arr.max()
for nformat in check:
(minlim, maxlim) = _type_ranges[nformat]
if (xmin >= minlim) and (xmax <= maxlim):
return nformat
return format
def get_xy_values(self, order="C", asmasked=False):
"""Get X Y coordinate values as numpy 2D arrays."""
nno = self.ncol * self.nrow
ier, xvals, yvals = _cxtgeo.surf_xy_as_values(
self.xori,
self.xinc,
self.yori,
self.yinc * self.yflip,
self.ncol,
self.nrow,
self.rotation,
nno,
nno,
0,
)
if ier != 0:
raise XTGeoCLibError(f"Error in surf_xy_as_values, error code: {ier}")
# reshape
xvals = xvals.reshape((self.ncol, self.nrow))
yvals = yvals.reshape((self.ncol, self.nrow))
if order == "F":
xvals = np.array(xvals, order="F")
yvals = np.array(yvals, order="F")
if asmasked:
tmpv = ma.filled(self.values, fill_value=np.nan)
tmpv = np.array(tmpv, order=order)
tmpv = ma.masked_invalid(tmpv)
mymask = ma.getmaskarray(tmpv)
xvals = ma.array(xvals, mask=mymask, order=order)
yvals = ma.array(yvals, mask=mymask, order=order)
return xvals, yvals
def cp_self_consistent_Aiken_zvode(h1e, cp, nsteps, single_step, zvode_steps):
norm_diff = []
rho_list = []
rho_0 = gcp.rho.copy()
rho_list.append(rho_0.copy())
for i in range(nsteps):
prev_aitken_rho = rho_0.copy()
rho_1 = single_step(rho_0, cp, h1e, zvode_steps)
rho_2 = single_step(rho_1, cp, h1e, zvode_steps)
aitken_rho = rho_2 - (rho_2 - rho_1)**2 / ma.array(rho_2 - 2*rho_1 + rho_0)
aitken_rho = ma.filled(aitken_rho, fill_value=rho_2)
rho_0 = aitken_rho
rho_list.append(rho_0.copy())
norm_diff.append(linalg.norm(aitken_rho - prev_aitken_rho))
if np.allclose(aitken_rho, prev_aitken_rho):
print("Iterations converged!")
break
return norm_diff, rho_list
def annualyield(year, couna, counb):
if year <= 2005:
bb = year - 1901
aa = year - 1901
else:
bb = 104
aa = year - 1901
region1 = NetCDFFile(
'/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/HistoricalGLM_crop_150901.nc',
'r')
lonlon = region1.variables['lon'][:]
maitrop = region1.variables['soy_trop'][bb, :, :]
maitemp = region1.variables['soy_temp'][bb, :, :]
maitropi = region1.variables['soy_trop_irrig'][bb, :, :]
maitempi = region1.variables['soy_temp_irrig'][bb, :, :]
gridarea = region1.variables['area'][:, :]
maitrop = ma.masked_where(maitrop <= 0, maitrop)
maitrop = ma.filled(maitrop, fill_value=0.)
maitemp = ma.masked_where(maitemp <= 0, maitemp)
maitemp = ma.filled(maitemp, fill_value=0.)
maitropi = ma.masked_where(maitropi <= 0, maitropi)
maitropi = ma.filled(maitropi, fill_value=0.)
maitempi = ma.masked_where(maitempi <= 0, maitempi)
maitempi = ma.filled(maitempi, fill_value=0.)
maizetor = maitrop + maitemp
maizetoi = maitropi + maitempi
maizeto = maitrop + maitemp + maitropi + maitempi
maizeto, lon1 = shiftgrid(0.5, maizeto, lonlon, start=True)
# print lon1
clm = NetCDFFile(
'/scratch2/scratchdirs/tslin2/isam/cheyenne/yieldout/isamhistorical_cru/heat/fertfao/fixedirr_new1/soytemp_historical_co2_rf_fert_0.5x0.5.nc',
'r')
clmtropf = clm.variables['totalyield'][bb, :, :]
clmtropf = ma.masked_where(clmtropf <= 0, clmtropf)
clmtropf = ma.filled(clmtropf, fill_value=0.)
clm2 = NetCDFFile(
'/scratch2/scratchdirs/tslin2/isam/cheyenne/yieldout/isamhistorical_cru/heat/fertfao/fixedirr_new1/soytemp_historical_co2_irrig_fert_0.5x0.5.nc',
'r')
clmtropf1 = clm2.variables['totalyield'][bb, :, :]
clmtropf1 = ma.masked_where(clmtropf1 <= 0, clmtropf1)
clmtropf1 = ma.filled(clmtropf1, fill_value=0.)
clm3n = NetCDFFile(
'/scratch2/scratchdirs/tslin2/isam/cheyenne/yieldout/isamhistorical_cru/heat/fertfao/fixedirr_new1/soytemp_historical_co2_rf_nofert_0.5x0.5.nc',
'r')
isammai = clm3n.variables['totalyield'][bb, :, :]
isammai = ma.masked_where(isammai <= 0, isammai)
isammai = ma.filled(isammai, fill_value=0.)
clm3n1 = NetCDFFile(
'/scratch2/scratchdirs/tslin2/isam/cheyenne/plot/finalyield/isam/heat/isamhis_maiscaleifyield_heat.nc',
'r')
isamcrumai = clm3n1.variables['yield'][bb, :, :]
isamcrumai = ma.masked_where(isamcrumai <= 0, isamcrumai)
isamcrumai = ma.filled(isamcrumai, fill_value=0.)
clm3n2 = NetCDFFile(
'/scratch2/scratchdirs/tslin2/isam/cheyenne/yieldout/isamhistorical_cru/heat/fertfao/fixedirr_new1/soytemp_historical_co2_irrig_nofert_0.5x0.5.nc',
'r')
isamcrumai2 = clm3n2.variables['totalyield'][aa, :, :]
isamcrumai2 = ma.masked_where(isamcrumai2 <= 0, isamcrumai2)
isamcrumai2 = ma.filled(isamcrumai2, fill_value=0.)
clm3n3 = NetCDFFile(
'/scratch2/scratchdirs/tslin2/isam/cheyenne/plot/finalyield/isam/heat/fertall/isamhis_maiscaleiyield_heat_fertall.nc',
'r')
isamcrumai3 = clm3n3.variables['yield'][bb, :, :]
isamcrumai3 = ma.masked_where(isamcrumai3 <= 0, isamcrumai3)
isamcrumai3 = ma.filled(isamcrumai3, fill_value=0.)
yieldclm1 = 0
yieldclm = 0.
yieldisam = 0.
yieldisamc = 0.
yieldisamc2 = 0.
yieldisamc3 = 0.
a = 0
harea = 0
yieldc1 = 0
yieldc = 0.
yieldi = 0.
yieldic = 0.
yieldic2 = 0.
yieldic3 = 0.
#USA 11501~11550
for xx in range(0, 360):
for yy in range(0, 720):
if coun[xx, yy] >= couna and coun[xx, yy] <= counb:
harea = maizeto[xx, yy] * gridarea[xx, yy] + harea
yieldclm1 = (clmtropf1[xx, yy] * maizeto[xx, yy] *
gridarea[xx, yy]) + yieldclm1
yieldclm = (clmtropf[xx, yy] * maizeto[xx, yy] *
gridarea[xx, yy]) + yieldclm
yieldisam = (isammai[xx, yy] * maizeto[xx, yy] *
gridarea[xx, yy]) + yieldisam
yieldisamc = (isamcrumai[xx, yy] * maizeto[xx, yy] *
gridarea[xx, yy]) + yieldisamc
yieldisamc2 = (isamcrumai2[xx, yy] * maizeto[xx, yy] *
gridarea[xx, yy]) + yieldisamc2
yieldisamc3 = (isamcrumai3[xx, yy] * maizeto[xx, yy] *
gridarea[xx, yy]) + yieldisamc3
a = a + 1
yieldc1 = N.average(clmtropf1, weights=maizeto)
yieldc = N.average(clmtropf, weights=maizeto)
yieldi = N.average(isammai, weights=maizeto)
yieldic = yieldisamc / harea
yieldic2 = N.average(isamcrumai2, weights=maizeto)
yieldic3 = yieldisamc3 / harea
return harea, yieldc, yieldi, yieldic, yieldclm, yieldisam, yieldisamc, yieldic2, yieldic3, yieldisamc2, yieldisamc3, yieldc1
import numpy.ma as ma
from scipy.interpolate import griddata
from scipy.stats import ttest_ind
region1 = NetCDFFile(
'/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/RCP45_crop_150901.nc',
'r')
maitrop = region1.variables['maize_trop'][4, :, :]
maitemp = region1.variables['maize_temp'][4, :, :]
maitropi = region1.variables['maize_trop_irrig'][4, :, :]
maitempi = region1.variables['maize_temp_irrig'][4, :, :]
maitrop = ma.masked_where(maitrop <= 0, maitrop)
maitropi = ma.masked_where(maitropi <= 0, maitropi)
maitemp = ma.masked_where(maitemp <= 0, maitemp)
maitempi = ma.masked_where(maitempi <= 0, maitempi)
maitrop = ma.filled(maitrop, fill_value=0.)
maitropi = ma.filled(maitropi, fill_value=0.)
maitemp = ma.filled(maitemp, fill_value=0.)
maitempi = ma.filled(maitempi, fill_value=0.)
maizeto = maitrop + maitemp
maizetoi = maitropi + maitempi
maitoatemp = maitemp + maitempi
maitoatrop = maitrop + maitropi
maizetotal = maizeto + maizetoi
clm = NetCDFFile(
'/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/clm45rcp45/maizetrop_rcp45_constco2_rf_nofert_0.5x0.5.nc',
'r')
clma = NetCDFFile(
'/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/clm45rcp45/maizetrop_rcp45_co2_rf_nofert_0.5x0.5.nc',
'r')
#clm=NetCDFFile('/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/clm45rcp45/maizetrop_rcp45_co2_rf_fert_0.5x0.5.nc','r')
def netcdf_preloader(crop_size=50,
tmp_array_size=200,
admin_config_file_path=None,
path_output='.',
dataframe=None,
target_datetimes=None,
stations=None):
"""
Preprocess netcdf files which are related to target_datetimes and stations
The resulting preprocessed netcdf files will be stored in path_output
If admin_config_file_path is not specified, the following have to be specified:
* dataframe
* target_datetimes
* stations
If admin_config_file is specified, it overwrites the parameters specified above.
:param crop_size: The crop size around each station in pixels
:param tmp_array_size: nb of timestamps to fit in temporary array
:param admin_config_file_path: The admin configuration file path
:param path_output: The folder where the outputted preprocessed netcdf files will be
:param dataframe: a pandas dataframe that provides the netCDF file path (or HDF5 file path and offset) for all
relevant timestamp values over the test period.
:param target_datetimes: a list of timestamps that your data loader should use to provide imagery for your model.
The ordering of this list is important, as each element corresponds to a sequence of GHI values
to predict. By definition, the GHI values must be provided for the offsets given by ``target_time_offsets``
which are added to each timestamp (T=0) in this datetimes list.
:param stations: a map of station names of interest paired with their coordinates (latitude, longitude, elevation).
"""
if admin_config_file_path:
admin_config_dict = helpers.load_dict(admin_config_file_path)
dataframe_path = admin_config_dict['dataframe_path']
with open(dataframe_path, 'rb') as df_file_handler:
dataframe = pickle.load(df_file_handler)
target_datetimes = [
datetime.datetime.strptime(date_time, '%Y-%m-%dT%H:%M:%S')
for date_time in admin_config_dict['target_datetimes']
]
stations = admin_config_dict['stations']
# hard coded values for now
n_channels = 5
n_timestep = 5
dc = crop_size // 2
ddt = datetime.timedelta(minutes=15)
# number of sample points
n_sample = len(target_datetimes)
# Generate all datetimes including prior timesteps from targets
all_dt = []
for dt0 in target_datetimes:
for i in range(4, 0, -1):
all_dt.append(dt0 - i * ddt)
all_dt.append(dt0)
chunksizes = min(256, n_sample)
# Initialize output netcdf files (one for each station)
nc_outs = {}
os.makedirs(path_output, exist_ok=True)
for station, coord in stations.items():
nc_outs[station] = netCDF4.Dataset(
os.path.join(path_output, f'{station}.nc'), 'w')
nc_outs[station].createDimension('time', n_sample)
nc_outs[station].createDimension('lat', crop_size)
nc_outs[station].createDimension('lon', crop_size)
nc_outs[station].createDimension('channel', n_channels)
nc_outs[station].createDimension('timestep', n_timestep)
time = nc_outs[station].createVariable('time', 'f8', ('time', ))
time.calendar = 'standard'
time.units = 'days since 1970-01-01 00:00:00'
nc_outs[station].createVariable('lat', 'f4', ('lat', ))
nc_outs[station].createVariable('lon', 'f4', ('lon', ))
nc_outs[station].createVariable(
f'data',
'f4', ('time', 'timestep', 'channel', 'lat', 'lon'),
zlib=True,
chunksizes=(chunksizes, n_timestep, n_channels, crop_size,
crop_size))
# Initialize temporary arrays to store data to limit constantly writing to disk
init = True
tmp_arrays = {}
coord_ij = {}
for station, coord in stations.items():
tmp_arrays[f"{station}"] = ma.masked_all(
(tmp_array_size, n_timestep, n_channels, crop_size, crop_size))
tmp_arrays["time"] = ma.masked_all((tmp_array_size, ))
# Loop through all timesteps and load images. The sequence moves through
# T-60min, T-45min, T-30min, T-15min, T0 and store those in the timestep
# dimension in the netcdf files. A temporary array is used in case the number
# of timestamps to process would be to big for memory.
for t, dt in enumerate(tqdm.tqdm(all_dt)):
at_t0 = not ((t + 1) % n_timestep)
t_sample = t // n_timestep
t_sample_tmp = t_sample % tmp_array_size
timestep_id = t % n_timestep
k = dataframe.index.get_loc(dt)
nc_path = dataframe['ncdf_path'][k]
try:
nc_loop = netCDF4.Dataset(nc_path, 'r')
except OSError:
# What to do when the netCDF4 file is not available.
# Currently filling with 0 later in the code...
if at_t0:
tmp_arrays["time"][t_sample_tmp] = netCDF4.date2num(
dt, time.units, time.calendar)
else:
if init:
lat_loop = nc_loop['lat'][:]
lon_loop = nc_loop['lon'][:]
for station, coord in stations.items():
lat_diff = np.abs(lat_loop - coord[0])
i = np.where(lat_diff == lat_diff.min())[0][0]
lon_diff = np.abs(lon_loop - coord[1])
j = np.where(lon_diff == lon_diff.min())[0][0]
nc_outs[station].variables['lat'][:] = lat_loop[i - dc:i +
dc]
nc_outs[station].variables['lon'][:] = lon_loop[j - dc:j +
dc]
coord_ij[station] = (i, j)
init = False
for d, c in enumerate([1, 2, 3, 4, 6]):
channel_data = nc_loop.variables[f'ch{c}'][0, :, :]
for station, coord in stations.items():
i, j = coord_ij[station]
tmp_arrays[f"{station}"][t_sample_tmp, timestep_id, d, :, :] = \
channel_data[i - dc:i + dc, j - dc:j + dc]
if at_t0:
tmp_arrays["time"][t_sample_tmp] = nc_loop.variables['time'][0]
nc_loop.close()
if ((t_sample_tmp == (tmp_array_size - 1)) and
(timestep_id == n_timestep - 1)) or (t == (len(all_dt) - 1)):
t0 = t_sample - t_sample_tmp
for station, coord in stations.items():
# Here we fill missing values with 0
nc_outs[station]['data'][t0:t_sample + 1, :, :, :, :] = \
ma.filled(tmp_arrays[f"{station}"][:t_sample_tmp + 1, :, :, :, :], 0)
tmp_arrays[f"{station}"] = \
ma.masked_all((tmp_array_size, n_timestep, n_channels, crop_size, crop_size))
nc_outs[station]['time'][t0:t_sample + 1] = \
tmp_arrays['time'][:t_sample_tmp + 1]
tmp_arrays["time"] = ma.masked_all((tmp_array_size, ))
for station, coord in stations.items():
nc_outs[station].close()
def _zone_averaging(
xprop, yprop, zoneprop, zone_minmax, coarsen, zone_avg, dzprop, mprop, summing=False
):
# General preprocessing, and...
# Change the 3D numpy array so they get layers by
# averaging across zones. This may speed up a lot,
# but will reduce the resolution.
# The x y coordinates shall be averaged (ideally
# with thickness weighting...) while e.g. hcpfzprop
# must be summed.
# Somewhat different processing whether this is a hc thickness
# or an average.
xpr = xprop
ypr = yprop
zpr = zoneprop
dpr = dzprop
mpr = mprop
if coarsen > 1:
xpr = xprop[::coarsen, ::coarsen, ::].copy(order="C")
ypr = yprop[::coarsen, ::coarsen, ::].copy(order="C")
zpr = zoneprop[::coarsen, ::coarsen, ::].copy(order="C")
dpr = dzprop[::coarsen, ::coarsen, ::].copy(order="C")
mpr = mprop[::coarsen, ::coarsen, ::].copy(order="C")
zpr.astype(np.int32)
if zone_avg:
zmin = int(zone_minmax[0])
zmax = int(zone_minmax[1])
if zpr.min() > zmin:
zmin = zpr.min()
if zpr.max() < zmax:
zmax = zpr.max()
newx = []
newy = []
newz = []
newm = []
newd = []
for izv in range(zmin, zmax + 1):
logger.info("Averaging for zone %s ...", izv)
xpr2 = ma.masked_where(zpr != izv, xpr)
ypr2 = ma.masked_where(zpr != izv, ypr)
zpr2 = ma.masked_where(zpr != izv, zpr)
dpr2 = ma.masked_where(zpr != izv, dpr)
mpr2 = ma.masked_where(zpr != izv, mpr)
# get the thickness and normalize along axis 2 (vertical)
# to get normalized thickness weights
lay_sums = dpr2.sum(axis=2)
normed_dz = dpr2 / lay_sums[:, :, np.newaxis]
# assume that coordinates have equal weights within a zone
xpr2 = ma.average(xpr2, axis=2)
ypr2 = ma.average(ypr2, axis=2)
zpr2 = ma.average(zpr2, axis=2) # avg zone
dpr2 = ma.sum(dpr2, axis=2)
if summing:
mpr2 = ma.sum(mpr2, axis=2)
else:
mpr2 = ma.average(mpr2, weights=normed_dz, axis=2) # avg zone
newx.append(xpr2)
newy.append(ypr2)
newz.append(zpr2)
newd.append(dpr2)
newm.append(mpr2)
xpr = ma.dstack(newx)
ypr = ma.dstack(newy)
zpr = ma.dstack(newz)
dpr = ma.dstack(newd)
mpr = ma.dstack(newm)
zpr.astype(np.int32)
xpr = ma.filled(xpr, fill_value=xtgeo.UNDEF)
ypr = ma.filled(ypr, fill_value=xtgeo.UNDEF)
zpr = ma.filled(zpr, fill_value=0)
dpr = ma.filled(dpr, fill_value=0.0)
mpr = ma.filled(mpr, fill_value=0.0)
return xpr, ypr, zpr, mpr, dpr
def deriv(*positional_inputs, **keyword_inputs):
"""Calculate the derivative along a single dimension.
Calling Sequence:
Result = deriv([x_in,] y_in, missing=1e+20, algorithm='default')
Positional Input Arguments:
* x_in: Abscissa values of y_in to take with respect to. If
not set, the derivative of y_in is take with respect to unit
abscissa intervals. Numeric array of same shape and size as
y_in. Must be monotonic and with no duplicate values.
Optional. First positional argument out of two, if present.
* y_in: Ordinate values, to take the derivative with respect
to. Numeric array vector of rank 1. Required. Second posi-
tional argument, if x_in is present; only positional argument
if x_in is absent.
Keyword Input Arguments:
* missing: If y_in and/or x_in has missing values, this is the
missing value value. Scalar. Default is 1e+20.
* algorithm: Name of the algorithm to use. String scalar.
Default is 'default'. Possible values include:
+ 'default': Default method (currently set to 'order1').
+ 'order1': First-order finite-differencing (backward and
forward differencing used at the endpoints, and centered
differencing used everywhere else). If abscissa intervals
are irregular, differencing will be correspondingly asym-
metric.
Output Result:
* Derivative of y_in with respect to x_in (or unit interval
abscissa, if x_in is not given). Numeric array of same shape
and size as y_in. If there are missing values, those elements
in the output are set to the value in |missing|. For instance,
if y_in is only one element, a one-element vector is returned
as the derivative with the value of |missing|. If there are
missing values in the output due to math errors and |missing|
is set to None, output will fill those missing values with the
MA default value of 1e+20.
References:
* Press, W. H., et al. (1992): Numerical Recipes in Fortran
77: The Art of Scientific Computing. New York, NY: Cambridge
University Press, pp. 180-184.
* Wang, Y. (1999): "Numerical Differentiation," Introduction to
MHD Numerical Simulation in Space, ESS265: Instrumentation,
Data Processing and Data Analysis in Space Physics (UCLA).
URL: http://www-ssc.igpp.ucla.edu/personnel/russell/ESS265/
Ch10/ylwang/node21.html.
Example with one argument, no missing values, using the default
method:
>>> from deriv import deriv
>>> import Numeric as N
>>> y = N.sin(N.arange(8))
>>> dydx = deriv(y)
>>> ['%.7g' % dydx[i] for i in range(4)]
['0.841471', '0.4546487', '-0.3501755', '-0.83305']
>>> true = N.cos(N.arange(8)) #- Compare with exact solution
>>> ['%.7g' % true[i] for i in range(4)]
['1', '0.5403023', '-0.4161468', '-0.9899925']
Example with two arguments with missing values, using first-
order differencing:
>>> x = N.arange(8)/(2.*N.pi)
>>> y = N.sin(x)
>>> y[3] = 1e20 #- Set an element to missing value
>>> dydx = deriv(x, y, missing=1e20, algorithm='order1')
>>> ['%.7g' % dydx[i] for i in range(5)]
['0.9957836', '0.9831985', '1e+20', '0.8844179', '1e+20']
>>> true = N.cos(x) #- Compare with exact solution
>>> ['%.7g' % true[i] for i in range(5)]
['1', '0.9873616', '0.9497657', '0.8881628', '0.8041098']
"""
import numpy.ma as MA
import numpy as N
#- Establish y_in and x_in from *positional_inputs:
if len(positional_inputs) == 1:
y_in = positional_inputs[0]
x_in = N.arange(len(y_in))
elif len(positional_inputs) == 2:
x_in = positional_inputs[0]
y_in = positional_inputs[1]
else:
raise ValueError, "deriv: Bad inputs"
#- Establish missing and algorithm from *keyword_inputs:
if keyword_inputs.has_key('missing') == 1:
missing = keyword_inputs['missing']
else:
missing = 1e+20
if keyword_inputs.has_key('algorithm') == 1:
algorithm = keyword_inputs['algorithm']
else:
algorithm = 'default'
#- Check positional and keyword inputs for possible errors:
if (len(y_in.shape) != 1) or (len(x_in.shape) != 1):
raise ValueError, "deriv: Inputs not a vector"
if type(algorithm) != type(''):
raise ValueError, "deriv: algorithm not str"
#- Set algorithm_to_use variable, based on the algorithm keyword.
# The algorithm_to_use tells which algorithm below to actually
# use (so here is where we set what algorithm to use for default):
if algorithm == 'default':
algorithm_to_use = 'order1'
else:
algorithm_to_use = algorithm
#- Change input to MA: just set to input value unless there are
# missing values, in which case add mask:
if missing == None:
x = MA.masked_array(x_in)
y = MA.masked_array(y_in)
else:
x = MA.masked_values(x_in, missing, copy=0)
y = MA.masked_values(y_in, missing, copy=0)
#- Calculate and return derivative:
# * Create working arrays that are consistent with a 3-point
# stencil in the interior and 2-point stencil on the ends:
# *im1 means the point before index i, *ip1 means the point
# after index i, and the i index array is just plain x or
# y; the endpadded arrays replicate the ends of x and y.
# I use an MA array filled approach instead of concatentation
# because the MA concatenation routine doesn't work right
# when the endpoint element is a missing value:
x_endpadded = MA.zeros(len(x) + 2)
x_endpadded[0] = x[0]
x_endpadded[1:-1] = x
x_endpadded[-1] = x[-1]
y_endpadded = MA.zeros(len(y) + 2)
y_endpadded[0] = y[0]
y_endpadded[1:-1] = y
y_endpadded[-1] = y[-1]
y_im1 = y_endpadded[:-2]
y_ip1 = y_endpadded[2:]
x_im1 = x_endpadded[:-2]
x_ip1 = x_endpadded[2:]
# * Option 1: First-order differencing (interior points use
# centered differencing, and end points use forward or back-
# ward differencing, as applicable):
if algorithm_to_use == 'order1':
dydx = (y_ip1 - y_im1) / (x_ip1 - x_im1)
# * Option 2: Bad algorithm specified:
else:
raise ValueError, "deriv: bad algorithm"
#- Return derivative as Numeric array:
return MA.filled(dydx, missing)
mask=NetCDFFile('india_mask.nc','r')
ind = mask.variables['MASK'][:,:]
ind= ma.masked_where(ind<=0.0,ind)
area=NetCDFFile('/scratch2/scratchdirs/tslin2/plot/globalcrop/data/gridareahalf.nc','r')
gridarea = area.variables['cell_area']
isam=NetCDFFile('/scratch2/scratchdirs/tslin2/isam/rice_cheyenne/his_cru/egu_new/ric_irr_fert/output/ric_irr_fert.nc','r')
lonisam1=isam.variables['lon'][:]
ind,lona11=shiftgrid(180.5,ind,lonisam1,start=False)
ric_i_f=isam.variables['totalyield'][79:109,:,:]
ric_i_f,lona11=shiftgrid(180.5,ric_i_f,lonisam1,start=False)
ric_i_f=ma.masked_where(ric_i_f<=0.0,ric_i_f)
ric_i_f=ma.filled(ric_i_f, fill_value=0.)
isam=NetCDFFile('/scratch2/scratchdirs/tslin2/isam/rice_cheyenne/his_cru/egu_new/ric_fert/output/ric_fert.nc','r')
ric_f=isam.variables['totalyield'][79:109,:,:]
ric_f,lona11=shiftgrid(180.5,ric_f,lonisam1,start=False)
ric_f=ma.masked_where(ric_f<=0.0,ric_f)
ric_f=ma.filled(ric_f, fill_value=0.)
isam=NetCDFFile('/scratch2/scratchdirs/tslin2/isam/rice_cheyenne/his_cru/egu_new/ric_irrfixed_fert/output/ric_irrfixed_fert.nc','r')
#isam=NetCDFFile('/scratch2/scratchdirs/tslin2/isam/rice_cheyenne/his_cru/egu_new/ric_fert_constcli/output/ric_fert_constcli.nc','r')
ric_fc=isam.variables['totalyield'][79:109,:,:]
ric_fc,lona11=shiftgrid(180.5,ric_fc,lonisam1,start=False)
gridarea1 = area.variables['cell_area'][:, :]
gridlon = area.variables['lon'][:]
gridlat = area.variables['lat'][:]
#print gridlon
nclu = NetCDFFile(
'/scratch2/scratchdirs/tslin2/plot/globalcrop/data/m3yield_isam.nc', 'r')
ncvar_maize = nclu.variables['soyy'][0, :, :]
marea = nclu.variables['soy_area'][0, :, :]
region1 = NetCDFFile(
'/scratch2/scratchdirs/tslin2/isam/maisoy_cheyenne/code/isamhiscru_soy_luh2.nc',
'r')
maitrop = region1.variables['area'][95:105, :, :]
lonisam1 = region1.variables['lon'][:]
maitrop = ma.masked_where(maitrop <= 0, maitrop)
maitrop = ma.filled(maitrop, fill_value=0.)
maizeto = maitrop
edat = NetCDFFile(
'/scratch2/scratchdirs/tslin2/isam/maisoy_cheyenne/his_cru/equili/soyequilibrium/output/soyequilibrium.nc',
'r')
eiyield1ynew = edat.variables['totalyield'][95:105, :, :]
edat2 = NetCDFFile(
'/scratch2/scratchdirs/tslin2/isam/maisoy_cheyenne/his_cru/equili/soyequilibrium_irr/output/soyequilibrium_irr.nc',
'r')
eiyield2ynew = edat2.variables['totalyield'][95:105, :, :]
edat3 = NetCDFFile(
'/scratch2/scratchdirs/tslin2/isam/maisoy_cheyenne/his_cru/fixedirr/soy_irr/output/soy_irr.nc',
def post_check(self, values, dataset_pool):
self.do_check("0 <= x and x <= 100", ma.filled(values,0))
def compute(self, dataset_pool):
hh_wwd = self.get_dataset().get_attribute(self.number_of_households_within_walking_distance)
return 100.0*ma.filled(self.get_dataset().get_attribute(self.number_of_high_income_households_within_walking_distance)/
ma.masked_where(hh_wwd == 0, hh_wwd.astype(float32)), 0.0)
region1 = NetCDFFile(
'/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/HistoricalGLM_crop_150901.nc',
'r')
#maitrop = region1.variables['soy_trop'][99,:,:]
#maitemp = region1.variables['soy_temp'][99,:,:]
#maitropi=region1.variables['soy_trop_irrig'][99,:,:]
#maitempi=region1.variables['soy_temp_irrig'][99,:,:]
maitrop = N.average(region1.variables['soy_trop'][95:105, :, :], axis=0)
maitemp = N.average(region1.variables['soy_temp'][95:105, :, :], axis=0)
maitropi = N.average(region1.variables['soy_trop_irrig'][95:105, :, :], axis=0)
maitempi = N.average(region1.variables['soy_temp_irrig'][95:105, :, :], axis=0)
gridarea = region1.variables['area'][:, :]
maitrop = ma.masked_where(maitrop <= 0, maitrop)
maitrop = ma.filled(maitrop, fill_value=0.)
maitemp = ma.masked_where(maitemp <= 0, maitemp)
maitemp = ma.filled(maitemp, fill_value=0.)
maitropi = ma.masked_where(maitropi <= 0, maitropi)
maitropi = ma.filled(maitropi, fill_value=0.)
maitempi = ma.masked_where(maitempi <= 0, maitempi)
maitempi = ma.filled(maitempi, fill_value=0.)
maizetor = maitrop + maitemp
maizetoi = maitropi + maitempi
maizeto = maitrop + maitemp + maitropi + maitempi
dat = NetCDFFile(
'/scratch2/scratchdirs/tslin2/isam/cheyenne/yieldout/isamhistorical_cru/heat/fertfao/fixedirr_new1/soytemp_historical_co2_irrig_fert_0.5x0.5.nc',
'r')
iyield1y = N.average(dat.variables['totalyield'][95:105, :, :], axis=0)
meareaisam = ma.masked_where(ind1 != 8, meareaisam)
area = NetCDFFile(
'/scratch2/scratchdirs/tslin2/plot/globalcrop/data/gridareahalf_isam.nc',
'r')
gridarea = area.variables['cell_area']
isam = NetCDFFile(
'/scratch2/scratchdirs/tslin2/isam/rice_cheyenne/his_cru/aogs/experiments/ric_irr_fert/output/ric_irr_fert.nc',
'r')
lonisam1 = isam.variables['lon'][:]
tric_i_f = isam.variables['totalyield'][79:109, :, :]
tric_i_f = ma.masked_where(tric_i_f <= 0.0, tric_i_f)
tric_i_f = ma.filled(tric_i_f, fill_value=0.)
tric_i_f = ma.masked_where(ind1 != 8, tric_i_f)
isam = NetCDFFile(
'/scratch2/scratchdirs/tslin2/isam/rice_cheyenne/his_cru/aogs/experiments/ric_fert/output/ric_fert.nc',
'r')
tric_f = isam.variables['totalyield'][79:109, :, :]
tric_f = ma.masked_where(tric_f <= 0.0, tric_f)
tric_f = ma.filled(tric_f, fill_value=0.)
tric_f = ma.masked_where(ind1 != 8, tric_f)
isam = NetCDFFile(
'/scratch2/scratchdirs/tslin2/isam/rice_cheyenne/his_cru/aogs/experiments/ric_cli/output/ric_cli.nc',
'r')
tric_fc = isam.variables['totalyield'][79:109, :, :]
tric_fc = ma.masked_where(tric_fc <= 0.0, tric_fc)
def yieldout(year):
bb = year - 1900
bb1 = year - 850
isam1 = NetCDFFile(
'/scratch2/scratchdirs/tslin2/plot/globalcrop/data/mirca_isam.nc', 'r')
maitrop1 = isam1.variables['asoy_rf'][:, :] #mirca2000
maitropi1 = isam1.variables['asoy_irr'][:, :] #mirca2000
maitrop1 = ma.masked_where(maitrop1 <= 0, maitrop1)
maitrop1 = ma.filled(maitrop1, fill_value=0.)
maitropi1 = ma.masked_where(maitropi1 <= 0, maitropi1)
maitropi1 = ma.filled(maitropi1, fill_value=0.)
lonisam = isam1.variables['lon'][:]
maitrop, lonisam1 = shiftgrid(180.5, maitrop1, lonisam, start=False)
maitropi, lonisam1 = shiftgrid(180.5, maitropi1, lonisam, start=False)
maizetor = maitrop
maizetoi = maitropi
maizetrop = maitrop + maitropi
maizeto = maitrop + maitropi
isam = NetCDFFile(
'/scratch2/scratchdirs/tslin2/isam/maisoy_cheyenne/his_cru/new/soy_fert/output/soy_fert.nc',
'r')
clmtropf = isam.variables['g_ET'][bb - 1, 1, :, :]
clm2 = NetCDFFile(
'/scratch2/scratchdirs/tslin2/isam/maisoy_cheyenne/his_cru/new/soy_irr_fert/output/soy_irr_fert.nc',
'r')
clmtropfi = clm2.variables['g_ET'][bb - 1, 1, :, :]
lonisam = clm2.variables['lon'][:]
clmtropf, lonisam1 = shiftgrid(180.5, clmtropf, lonisam, start=False)
clmtropfi, lonisam1 = shiftgrid(180.5, clmtropfi, lonisam, start=False)
print lonisam1
clmtropf = ma.masked_where(maitrop <= 0, clmtropf)
clmtropf = ma.filled(clmtropf, fill_value=0.)
clmtropfi = ma.masked_where(maitropi <= 0, clmtropfi)
clmtropfi = ma.filled(clmtropfi, fill_value=0.)
yield_clmtf = clmtropf
yield_clmtf = ma.masked_where(yield_clmtf <= 0, yield_clmtf)
yield_clmtf = ma.filled(yield_clmtf, fill_value=0.)
yield_clmtfi = clmtropfi
yield_clmtfi = ma.masked_where(yield_clmtfi <= 0, yield_clmtfi)
yield_clmtfi = ma.filled(yield_clmtfi, fill_value=0.)
area = NetCDFFile(
'/scratch2/scratchdirs/tslin2/plot/globalcrop/data/gridareahalf_isam.nc',
'r')
gridarea = area.variables['cell_area'][:, :]
gridlon = area.variables['lon'][:]
gridlat = area.variables['lat'][:]
gridarea, gridlon = shiftgrid(180.5, gridarea, gridlon, start=False)
lon2, lat2 = N.meshgrid(gridlon, gridlat)
map = Basemap(projection='cyl',
llcrnrlat=-65,
urcrnrlat=90,
llcrnrlon=-180,
urcrnrlon=180,
resolution='c')
x, y = map(lon2, lat2)
yield_clmtf = maskoceans(x, y, yield_clmtf)
yield_clmtf = ma.masked_where(maizeto <= 0, yield_clmtf)
yield_clmtfi = maskoceans(x, y, yield_clmtfi)
yield_clmtfi = ma.masked_where(maizeto <= 0, yield_clmtfi)
clmy = ((yield_clmtf * maizetor) +
(yield_clmtfi * maizetoi)) / ((maizetoi) + (maizetor))
areall = (maizetoi) + (maizetor)
clmall = ((yield_clmtf * maizetor) + (yield_clmtfi * maizetoi))
return clmy, areall, clmall
