def compute(self, dataset_pool): constants = dataset_pool.get_dataset('constants') footprint = constants["FOOTPRINT"] covertypes_of_interest = self._cover_type_translation(constants) lct = ma.filled(self.get_dataset().get_2d_attribute(self.land_cover_type), 0) is_lct_of_interest = reduce(lambda prev_answer, lct_num: logical_or(prev_answer, lct==lct_num), covertypes_of_interest, zeros(shape=lct.shape, dtype=int32)) # maxg: maximum number of possible like-adjacencies within a window # a_i: # of cells of target covertypes within moving window a_i = correlate(ma.filled(is_lct_of_interest.astype(int32), 0), footprint, mode="reflect", cval=0.0) n = floor(sqrt(a_i)) m = a_i - power(n, 2) maxg = 2*n*(n-1) maxg += where(m==0, 0, where(m<=n, (2*m-1), where(m>n, (2*m-2), 0))) # detect all adjacencies by systematically checking each possibility # within the 5x5 footprint window gii = ma.masked_array(zeros(shape=lct.shape, dtype=int32)) for ri in range(0, footprint.shape[0]): for ci in range(0, footprint.shape[1]-1): gii += self._find_adjacency(is_lct_of_interest, (ri,ci), (ri,ci+1), footprint) for ri in range(0, footprint.shape[0]-1): for ci in range(0, footprint.shape[1]): gii += self._find_adjacency(is_lct_of_interest, (ri,ci), (ri+1,ci), footprint) #assert ma.alltrue(ravel(gii<=maxg)) return self.get_dataset().flatten_by_id(arcsin(sqrt(ma.filled(gii/maxg, 0))))
def compute(self, dataset_pool): constants = dataset_pool.get_dataset('constants') footprint = constants["FOOTPRINT"] lct = ma.filled(self.get_dataset().get_2d_attribute(self.land_cover_types), 0) temp = equal(lct, constants[self.lct_type.upper()]) values = correlate(temp.astype(int32), footprint, mode="reflect") return self.get_dataset().flatten_by_id(ma.filled(values, 0))
def ncf2ffi1001(f, outpath, mode = 'w'): outfile = open(outpath, mode) header_keys = "PI_CONTACT_INFO PLATFORM LOCATION ASSOCIATED_DATA INSTRUMENT_INFO DATA_INFO UNCERTAINTY ULOD_FLAG ULOD_VALUE LLOD_FLAG LLOD_VALUE DM_CONTACT_INFO PROJECT_INFO STIPULATIONS_ON_USE OTHER_COMMENTS REVISION".split() print('%d, %d' % (len(f.ncattrs()) + len(f.variables), 1001), file = outfile) print(getattr(f, 'PI_NAME', 'Unknown'), file = outfile) print(getattr(f, 'ORGANIZATION_NAME', 'Unknown'), file = outfile) print(getattr(f, 'SOURCE_DESCRIPTION', 'Unknown'), file = outfile) print(getattr(f, 'VOLUME_INFO', 'Unknown'), file = outfile) print(f.SDATE, f.WDATE, file = outfile) print(f.TIME_INTERVAL, file = outfile) print(f.INDEPENDENT_VARIABLE, file = outfile) print('%d' % len(f.variables), file = outfile) for key, var in f.variables.items(): print('%s, %s' % (key, getattr(var, 'units', 'unknown')), file = outfile) print(len(f.ncattrs()), file = outfile) for key in f.ncattrs(): print('%s: %s' % (key, getattr(f, key, '')), file = outfile) vals = [filled(f.variables[f.INDEPENDENT_VARIABLE][:]).ravel()] keys = [f.INDEPENDENT_VARIABLE] for key, var in f.variables.items(): if key == f.INDEPENDENT_VARIABLE: continue keys.append(key) vals.append(filled(var[:]).ravel()) print(', '.join(keys), file = outfile) for row in array(vals).T: row.tofile(outfile, format = '%.6e', sep = ', ') print('', file = outfile)
def extract(self,PSF): #Placeholder pixsig = 1. sh = self.sh npsf = PSF.size flux = np.zeros(sh) uflux = np.zeros(sh) sky1d = np.zeros(sh) usky1d = np.zeros(sh) im = self.Order.image_rect uim = self.Order.uimage_rect sky = self.Order.sky_rect usky = self.Order.usky_rect for i in np.arange(sh): # flux[i] = (PSF*im[i,:]).sum() / (PSF**2).sum() flux[i] = (PSF*im[i,:]/uim[i,:]**2).sum() / (PSF**2/uim[i,:]**2).sum() uflux[i] = np.sqrt(1.0/(PSF**2/uim[i,:]**2.).sum()) sky1d[i] = (np.abs(PSF)*sky[i,:]).sum() / (PSF**2).sum() usky1d[i] = np.sqrt(1.0/(PSF**2/usky[i,:]**2.).sum()) flux = ma.masked_invalid(flux) flux = ma.filled(flux,1.) uflux = ma.masked_invalid(uflux) uflux = ma.filled(uflux,1000.) sky1d = ma.masked_invalid(sky1d) sky1d = ma.filled(sky1d,1.) usky1d = ma.masked_invalid(usky1d) usky1d = ma.filled(usky1d,1000.) sky_cont = self._fitCont(self.wave,sky1d) return flux,uflux,sky1d-sky_cont,usky1d
def _compute_shannon_evenness_index(self, lct, footprint, lct_mask): """Compute Shannon's evenness index""" cover_types_list = self._get_cover_types(lct) numerator_sum = ma.masked_array(zeros(shape=lct.shape, dtype=float32), mask=lct_mask) for cover_type in cover_types_list: pi = self._compute_pct_cover_type_within_footprint(lct, cover_type, footprint, lct_mask) numerator_sum += pi*ma.filled(ma.log(pi), 0) m = self._count_covertypes_within_window(lct, cover_types_list, footprint) return ma.filled(-numerator_sum / ma.log(m), 0).astype(float32)
def compute(self, dataset_pool): ds = self.get_dataset() age = ds.get_attribute(self.building_age) result = ma.filled(age, -1) mask = ma.getmask(age) if mask is not ma.nomask and mask.any(): # If there are bad ages then mask won't be ma.nomask and will have some non-False values. In this case compute # the average age for all buildings, then find the ages of buildings within walking distance, and use those # to fill in the bad values. whole_area_average_age = int(age.sum()/float(ds.size() - age.mask.sum())) age_wwd_values = ma.filled(ds.get_attribute(self.age_wwd), whole_area_average_age) result[mask] = age_wwd_values[mask] return result
def reduce (self, target, axis=0): """Reduce target along the given axis.""" axes, attributes, id, grid = _extractMetadata(target, omit=axis) a = _makeMaskedArg(target) m = getmask(a) if m is nomask: t = filled(a) return masked_array (numpy.maximum.reduce (t, axis)) else: t = numpy.maximum.reduce(filled(a, numpy.ma.maximum_fill_value(a)), axis) m = numpy.logical_and.reduce(m, axis) return TransientVariable(t, mask=m, fill_value=fill_value(a), axes = axes, grid=grid, id=id)
def compute(self, dataset_pool): constants = dataset_pool.get_dataset('constants') fs = ma.filled(self.get_dataset().get_2d_attribute(self.footprint_size).astype(float32), 0) lct = ma.filled(self.get_dataset().get_2d_attribute(self.land_cover_type), 0) x = zeros(shape=lct.shape, dtype=float32) max_type = int(maximum.reduce(ravel(lct))) for itype in range(1, max_type+1): temp = equal(lct, itype).astype(int32) summed = correlate(ma.filled(temp, 0.0), constants['FOOTPRINT'], mode="reflect") x += temp * ma.filled(summed / ma.masked_where(fs==0, fs), 0.0) return self.get_dataset().flatten_by_id(arcsin(sqrt(x)))
def calcDeltaMags(): ''' Creates an array with the delta magnitudes of WDS objects. ''' # Calculating magnitude difference for each object # Make nice formats... Pri_mag_g = ma.filled(wdsInteresting[priMag],[-999]) Sec_mag_g = ma.filled(wdsInteresting[secMag],[-999]) Pri_mag_gg = np.asarray(Pri_mag_g, dtype = 'float_' ) Sec_mag_gg = np.asarray(Sec_mag_g, dtype = 'float_' ) # Then calculate the actual delta mags Delta_mag = np.subtract(Pri_mag_gg, Sec_mag_gg) return Delta_mag
def outer (self, a, b): "Return the function applied to the outer product of a and b." a = _makeMaskedArg(a) b = _makeMaskedArg(b) ma = getmask(a) mb = getmask(b) if ma is nomask and mb is nomask: m = None else: ma = getmaskarray(a) mb = getmaskarray(b) m = logical_or.outer(ma, mb) d = numpy.maximum.outer(filled(a), filled(b)) return TransientVariable(d, mask=m)
def test_testMasked(self): # Test of masked element xx = arange(6) xx[1] = masked self.assertTrue(str(masked) == '--') self.assertTrue(xx[1] is masked) self.assertEqual(filled(xx[1], 0), 0)
def compute_lambda_for_vacancy(grouping_location_set, units_variable, vacant_units_variable, movers_variable, secondary_residence_variable=None, multiplicator=1.0, resources=None): grouping_location_set.compute_variables([units_variable, vacant_units_variable, movers_variable], resources=resources) if secondary_residence_variable is not None: grouping_location_set.compute_variables([secondary_residence_variable], resources=resources) unitssecondary = grouping_location_set.get_attribute(secondary_residence_variable) else: unitssecondary = zeros(grouping_location_set.size()) unitsvacant = grouping_location_set.get_attribute(vacant_units_variable) units = grouping_location_set.get_attribute(units_variable) movers = grouping_location_set.get_attribute(movers_variable) tsv = units - unitssecondary - unitsvacant lambda_value = ma.filled((units - unitssecondary).astype(float32)/ ma.masked_where(tsv==0, tsv),0) \ - ma.filled(unitsvacant.astype(float32) / ma.masked_where(movers==0, movers),0) lambda_value = lambda_value * multiplicator return lambda_value
def compute(self, dataset_pool): hh_wwd = self.get_dataset().get_attribute(self.number_of_households_within_walking_distance) return 100.0 * ma.filled( self.get_dataset().get_attribute(self.number_of_young_households_within_walking_distance) / ma.masked_where(hh_wwd == 0, hh_wwd.astype(float32)), 0.0, )
def compute(self, dataset_pool): constants = dataset_pool.get_dataset('constants') footprint = constants["FOOTPRINT"] lct = ma.filled(self.get_dataset().get_2d_attribute(self.land_cover_type), 0) lct_mask = self.get_dataset().get_mask(is_2d_version=True) shei = self._compute_shannon_evenness_index(lct, footprint, lct_mask) return self.get_dataset().flatten_by_id(arcsin(sqrt(shei)))
def __setslice__(self, i, j, value): "Sets the slice described by [i,j] to `value`." _localdict = self.__dict__ d = self._data m = _localdict['_fieldmask'] names = self.dtype.names if value is masked: for n in names: m[i:j][n] = True elif not self._hardmask: fval = filled(value) mval = getmaskarray(value) for n in names: d[n][i:j] = fval m[n][i:j] = mval else: mindx = getmaskarray(self)[i:j] dval = np.asarray(value) valmask = getmask(value) if valmask is nomask: for n in names: mval = mask_or(m[n][i:j], valmask) d[n][i:j][~mval] = value elif valmask.size > 1: for n in names: mval = mask_or(m[n][i:j], valmask) d[n][i:j][~mval] = dval[~mval] m[n][i:j] = mask_or(m[n][i:j], mval) self._fieldmask = m
def __init__(self, MetricTable, opts): epsilon = 0.0 if opts.cpu: epsilon = 0.01 # Create empty ratio table nprobs = MetricTable.nprobs nsolvs = MetricTable.nsolvs self.ratios = ma.zeros((nprobs, nsolvs), dtype=numpy.float) # Compute best relative performance ratios across # solvers for each problem for prob in range(nprobs): metrics = MetricTable.prob_mets(prob) + epsilon best_met = ma.minimum(metrics) self.ratios[prob,:] = metrics * (1.0 / best_met) # Sort each solvers performance ratios for solv in range(nsolvs): self.ratios[:,solv] = ma.sort(self.ratios[:,solv]) # Compute largest ratio and use to replace failure entries self.maxrat = ma.maximum(self.ratios) self.ratios = ma.filled(self.ratios, 10 * self.maxrat)
def run_chunk(self, agents_index, agent_set, specification, coefficients): # unplaced agents in agents_index location_id_name = self.choice_set.get_id_name()[0] agent_set.set_values_of_one_attribute(location_id_name, resize(array([-1]), agents_index.size), agents_index) ## capacity may need to be re-computed for every chunk if self.compute_capacity_flag: self.capacity = ma.filled(self.determine_capacity(capacity_string=self.run_config.get("capacity_string", None), agent_set=agent_set, agents_index=agents_index), 0.0) if self.capacity is not None: logger.log_status("Available capacity: %s units." % self.capacity.sum()) self.run_config.merge({"capacity":self.capacity}) if self.run_config.get("agent_units_string", None): self.run_config["agent_units_all"] = agent_set.get_attribute_by_index(self.run_config["agent_units_string"], agents_index) choices = ChoiceModel.run_chunk(self, agents_index, agent_set, specification, coefficients) ## this is done in choice_model #modify locations #agent_set.set_values_of_one_attribute(location_id_name, choices, agents_index) if self.run_config.has_key("capacity"): del self.run_config["capacity"] return choices
def compute(self, dataset_pool): cellsize = dataset_pool.get_dataset('constants')['CELLSIZE'] fpdimension = int(self.footprint_width / cellsize) fp = ones((fpdimension, fpdimension), dtype="int32") summed = correlate( ma.filled( self.get_dataset().get_2d_attribute( self.comm_add ), 0.0 ), \ fp, mode="reflect" ) return ln(self.get_dataset().flatten_by_id(summed)+1)/10.0
def mag_to_flux(mag, limMag=False): """ Converts magnitudes to fluxes using the following law (from Kessler+2010): flux = 10^(-0.4 * m + 11) If the magnitude is above the limit of the instrument, returns the corresponding limiting flux. INPUT: mag: numpy array of magnitude limMag: specifies the limiting magnitude OUTPUT: flux: flux-converted magnitudes """ if limMag: exponent = (-0.4 * limMag) + 11 limFlux = np.power(10, exponent) # masked arrays if magnitude is grater then limit maMag = ma.masked_where(mag > limMag, mag) maExponent = (-0.4 * maMag) + 11 maFlux = np.power(10, exponent) maFlux.set_fill_value(limFlux) flux = ma.filled(maFlux) else: exponent = (-0.4 * mag) + 11 flux = 10**exponent return flux
def compute(self, dataset_pool): recent_years = dataset_pool.get_dataset('urbansim_constant')["recent_years"] buildings = dataset_pool.get_dataset('building') age = buildings.get_attribute("building_age") values = buildings.get_attribute(self.units) values = where(ma.filled(age, recent_years+1) <= recent_years, values, 0) return self.get_dataset().sum_over_ids(buildings.get_attribute("grid_id"), values)
def create_gap_mask(self, wave): """ Use the gap configuration and a wavelength vector, wave, to build a masked array that masks out the location of the gap. Wavelengths between and including both gap endpoints will be masked out. Parameters ---------- wave: numpy.ndarray Wavelength vector to construct mask from Returns ------- mask: numpy.ma 1D masked array 1D array masked at the locations within the configured detector gap and 1.0 elsewhere """ disperser = self.instrument['disperser'] aperture = self.instrument['aperture'] filt = self.instrument['filter'] gap = self.aperture_config[aperture]['gap'][disperser] if filt in gap: gap_start = gap[filt]['gap_start'] gap_end = gap[filt]['gap_end'] else: gap_start = gap['gap_start'] gap_end = gap['gap_end'] if gap_start is not None and gap_end is not None: masked_wave = ma.masked_inside(wave, gap_start, gap_end) mask = masked_wave / wave mask = ma.filled(mask, 0.0) else: mask = 1.0 return mask
def run(self, utilities, resources=None): """ Compute probabilities of a discrete choice model from the given utitlities. 'utilities' is a 2D array (nobservations x nequations). The return value is a 2D array (same shape as utilities). """ if utilities.ndim < 2: raise StandardError, "Argument 'utilities' must be a 2D numpy array." util_min = utilities.min() util_max = utilities.max() if (util_min < self.computable_range[0]) or (util_max > self.computable_range[1]): # shift utilities to zero (maximum is at zero) to_be_transformed=where((utilities < self.computable_range[0]) + (utilities > self.computable_range[1])) to_be_transformed=unique(to_be_transformed[0]) for idx in arange(to_be_transformed.size): i = to_be_transformed[idx] this_max = utilities[i,:].max() utilities[i,:]=utilities[i,:]-this_max availability = resources.get('availability', None) if availability is None: exponentiated_utility = exp(utilities) else: exponentiated_utility = exp(utilities) * (availability).astype('b') sum_exponentiated_utility = sum(exponentiated_utility, axis=1, dtype="float64") sum_exponentiated_utility = ma.masked_where(sum_exponentiated_utility==0, sum_exponentiated_utility) return ma.filled(exponentiated_utility/reshape(sum_exponentiated_utility,(utilities.shape[0], 1)), 0)
def __init__(self, MetricTable): # Create empty ratio table nprobs = MetricTable.nprobs nsolvs = MetricTable.nsolvs self.ratios = ma.masked_array(1.0 * ma.zeros((nprobs+1, nsolvs))) # Compute best relative performance ratios across # solvers for each problem for prob in range(nprobs): metrics = MetricTable.prob_mets(prob) best_met = ma.minimum(metrics) if (ma.count(metrics)==nsolvs and ma.maximum(metrics)<=opts.minlimit): self.ratios[prob+1,:] = 1.0; else: self.ratios[prob+1,:] = metrics * (1.0 / best_met) # Sort each solvers performance ratios for solv in range(nsolvs): self.ratios[:,solv] = ma.sort(self.ratios[:,solv]) # Compute largest ratio and use to replace failures entries self.maxrat = ma.maximum(self.ratios) self.ratios = ma.filled(self.ratios, 1.01 * self.maxrat)
def compute(self, dataset_pool): sqft = self.get_dataset().get_attribute(self.sqft) num_of_job_spaces = ma.filled(self.get_dataset().get_attribute(self.commercial_sqft)/ ma.masked_where(sqft == 0, sqft.astype(float32)), 0.0).astype(int32) return clip_to_zero_if_needed(num_of_job_spaces - self.get_dataset().get_attribute(self.number_of_commercial_jobs), 'vacant_commercial_job_space')
def get_average_omega(self, omega, probability, index, nsupply, demand): omega_prob = omega[:, newaxis]*probability omega_prob_sum_over_i = array(ndimage_sum(omega_prob, labels=index+1, index=arange(nsupply)+1)) prob_sum_over_i = array(ndimage_sum(probability, labels=index+1, index=arange(nsupply)+1)) average_omega = ma.filled(omega_prob_sum_over_i/ ma.masked_where(prob_sum_over_i==0, prob_sum_over_i), 0.0) return average_omega
def identify(cls, variables, ignore=None, target=None, warn=True, monotonic=False): result = {} ignore, target = cls._identify_common(variables, ignore, target) # Identify all CF coordinate variables. for nc_var_name, nc_var in target.iteritems(): if nc_var_name in ignore: continue # String variables can't be coordinates if np.issubdtype(nc_var.dtype, np.str): continue # Restrict to one-dimensional with name as dimension OR zero-dimensional scalar if not ((nc_var.ndim == 1 and nc_var_name in nc_var.dimensions) or (nc_var.ndim == 0)): continue # Restrict to monotonic? if monotonic: data = nc_var[:] # Gracefully fill a masked coordinate. if ma.isMaskedArray(data): data = ma.filled(data) if nc_var.shape == () or nc_var.shape == (1,) or iris.util.monotonic(data): result[nc_var_name] = CFCoordinateVariable(nc_var_name, nc_var) else: result[nc_var_name] = CFCoordinateVariable(nc_var_name, nc_var) return result
def compute(self, dataset_pool): buildings = self.get_dataset() non_residential_sqft = buildings.get_attribute("non_residential_sqft") building_sqft_per_job = buildings.get_attribute("building_sqft_per_job") remaining_space = clip(non_residential_sqft - buildings.get_attribute("occupied_building_sqft_by_jobs"), 0, non_residential_sqft) return ma.filled(ma.masked_where(building_sqft_per_job==0, remaining_space / building_sqft_per_job), 0)
def flux_to_mag(flux, limFlux=False): """ Converts fluxes to magnitudes using the following law (from Kessler+2010): flux = 10^(-0.4 * m + 11) => m = -2.5 * (log_10(flux) - 11) If the flux is below the limit of the instrument, returns the corresponding limiting magnitude. INPUT: flux: numpy array of fluxes limFlux: specifies the limiting flux OUTPUT: mag: magnitude-converted fluxes """ if limFlux: limMag = -2.5 * (-11 + np.log10(limFlux)) # applying the mask to detection below the limiting flux maFlux = ma.masked_where(flux < limFlux, flux) # to avoid warnings due to values passed to np.log10 # fluxMask = maFlux.mask # maMag = -2.5 * (-11.0 + np.log10(ma.filled(maFlux,1))) maMag = -2.5 * (-11.0 + np.log10(maFlux)) mag = ma.filled(maMag, limMag) else: if flux > 0: mag = -2.5 * (-11 + np.log10(flux)) else: mag = None return mag
def test_testMasked(self): # Test of masked element xx = arange(6) xx[1] = masked assert_(str(masked) == '--') assert_(xx[1] is masked) assert_equal(filled(xx[1], 0), 0)
def compute(self, dataset_pool): nj = self.get_dataset().get_attribute(self.number_of_industrial_jobs_wwd) sqft = self.get_dataset().get_attribute(self.industrial_sqft_within_walking_distance) regional_average = self.get_dataset().get_attribute(self.industrial_sqft).sum() / float( self.get_dataset().get_attribute(self.number_of_industrial_jobs).sum() ) return where(sqft < 5000, regional_average, ma.filled(sqft / ma.masked_where(nj == 0, nj.astype(float32)), 0))
def from_surface(self, surf, zname="Z_TVDSS"): """Get points as X Y Value from a surface object nodes. Note that undefined surface nodes will not be included. Args: surf (RegularSurface): A XTGeo RegularSurface object instance. zname (str): Name of value column (3'rd column) Example:: topx = RegularSurface('topx.gri') topx_aspoints = Points() topx_aspoints.from_surface(topx) # alternative shortform: topx_aspoints = Points(topx) # get an instance directly topx_aspoints.to_file('mypoints.poi') # export as XYZ file """ # check if surf is instance from RegularSurface if not isinstance(surf, xtgeo.surface.RegularSurface): raise ValueError("Given surf is not a RegularSurface object") val = surf.values xc, yc = surf.get_xy_values() coor = [] for vv in [xc, yc, val]: vv = ma.filled(vv.flatten(order="C"), fill_value=np.nan) vv = vv[~np.isnan(vv)] coor.append(vv) # now populate the dataframe: xc, yc, val = coor # pylint: disable=unbalanced-tuple-unpacking ddatas = OrderedDict() ddatas[self._xname] = xc ddatas[self._yname] = yc ddatas[self._zname] = val self._df = pd.DataFrame(ddatas) self.zname = zname
def extract_snow_accum(self): """ Retrieves snow accumulation. Returns ------- list """ nan = float('nan') if self.snow_accum == None: if self.tmt_site_index != ([],): snow_accum_var = self.tmt_nc.variables["snow_depth_inches"] snow_accum = ma.filled(snow_accum_var[self.tmt_site_index], nan).flatten() snow_accum[snow_accum==9.9692099683868690e+36] = nan snow_accum = snow_accum[self.current_utc_hour:] self.snow_accum = snow_accum else: self.snow_accum = [] return self.snow_accum
def test_testBasic2d(self): # Test of basic array creation and properties in 2 dimensions. for s in [(4, 3), (6, 2)]: (x, y, a10, m1, m2, xm, ym, z, zm, xf, s) = self.d x.shape = s y.shape = s xm.shape = s ym.shape = s xf.shape = s assert_(not isMaskedArray(x)) assert_(isMaskedArray(xm)) assert_equal(shape(xm), s) assert_equal(xm.shape, s) assert_equal(xm.size, reduce(lambda x, y: x * y, s)) assert_equal(count(xm), len(m1) - reduce(lambda x, y: x + y, m1)) assert_(eq(xm, xf)) assert_(eq(filled(xm, 1.e20), xf)) assert_(eq(x, xm)) self.setup()
def extract_road_temps(self): """ Retrieves road temperatures. Returns ------- list """ nan = float('nan') if self.road_temps == None: if self.tmt_site_index != ([],): road_temp_var = self.tmt_nc.variables[backend_sys_path.Met_vars.tmt_road_temp_var] road_temps = ma.filled(road_temp_var[self.tmt_site_index], nan).flatten() road_temps[road_temps==9.9692099683868690e+36] = nan road_temps = road_temps[self.current_utc_hour:] self.road_temps = road_temps else: self.road_temps = [] return self.road_temps
def _compute_patch_size_of_cover_types(self, lct, footprint): """Computes the mean size of all patches of covertypes of interest that are (partially) within each cell's footprint""" eightway_structure = ones( (3, 3), dtype="int32") # put this to class variable will result in error patchsizes = zeros(shape=lct.shape, dtype=float32) patchcount = zeros(shape=lct.shape, dtype=float32) labels, n = label(lct, eightway_structure) slices = find_objects(labels) # Summing the 0/1 mask gives the patch size for ip in range(n): locmask = where(labels[slices[ip]] == (ip + 1), lct[slices[ip]], 0) patchcount[slices[ip]] += locmask patchsizes[slices[ip]] += locmask.sum() * locmask pcount_corr = ma.masked_array( correlate(patchcount, footprint, mode="reflect", cval=0.0)) psize_corr = correlate(patchsizes, footprint, mode="reflect", cval=0.0) return ma.filled(psize_corr / pcount_corr, 0)
def get_mask_of_attribute(self, attribute, is_2d_version=False): """Get mask for lct.""" if attribute is None: raise StandardError, "No non-attributes found. Cannot create mask." if is_2d_version: values = self.get_2d_attribute(attribute) else: values = self.get_attribute(attribute) nodata_val = self.get_nodata_value() missingdata_val = self.get_missing_data_value() self.resources["NODATA_value"] = nodata_val self.resources["MISSING_value"] = missingdata_val values = ma.filled(values, nodata_val) nodata_filter = where(values == nodata_val, True, False) return logical_or(where(values == missingdata_val, True, False), nodata_filter)
def nowyield(year, couna, counb): bb = year - 1901 region1 = NetCDFFile( '/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/HistoricalGLM_crop_150901.nc', 'r') maitrop = region1.variables['maize_trop'][99, :, :] maitemp = region1.variables['maize_temp'][99, :, :] maitropi = region1.variables['maize_trop_irrig'][99, :, :] maitempi = region1.variables['maize_temp_irrig'][99, :, :] gridarea = region1.variables['area'][:, :] maitrop = ma.masked_where(maitrop <= 0, maitrop) maitrop = ma.filled(maitrop, fill_value=0.) maitemp = ma.masked_where(maitemp <= 0, maitemp) maitemp = ma.filled(maitemp, fill_value=0.) maitropi = ma.masked_where(maitropi <= 0, maitropi) maitropi = ma.filled(maitropi, fill_value=0.) maitempi = ma.masked_where(maitempi <= 0, maitempi) maitempi = ma.filled(maitempi, fill_value=0.) maizetro = maitrop + maitropi maizetem = maitemp + maitempi maizetor = maitrop + maitemp maizetoi = maitropi + maitempi maizeto = maitrop + maitemp + maitropi + maitempi clm = NetCDFFile( '/scratch2/scratchdirs/tslin2/isam/cheyenne/plot/finalyield/clm/clm45his_maiscaleifyield_luc2000.nc', 'r') clmyield = clm.variables['yield'][bb, :, :] isam = NetCDFFile( '/scratch2/scratchdirs/tslin2/isam/cheyenne/plot/finalyield/isam/heat/isamhis_maiscaleifyield_heat_luc2000.nc', 'r') isamyield = isam.variables['yield'][bb, :, :] clmyield[N.isnan(clmyield)] = -9999 clmyield = ma.masked_where(clmyield <= 0, clmyield) clmyield = ma.masked_where(maizeto <= 0, clmyield) clmyield = ma.filled(clmyield, fill_value=0.) isamyield[N.isnan(isamyield)] = -9999 isamyield = ma.masked_where(isamyield <= 0, isamyield) isamyield = ma.masked_where(maizeto <= 0, isamyield) isamyield = ma.filled(isamyield, fill_value=0.) return clmyield, isamyield, maizeto
def normalize(weight_array): """Normalize a weight_array of either 1 or 2 dimensions""" prob_array = None if weight_array is None: return prob_array # dimensions of weight array try: d = weight_array.ndim except: d = 0 lt0_index = weight_array < 0 if lt0_index.sum() > 0: raise ValueError, "some values of rows %s in weight_array are less than 0" % where(lt0_index)[0] if d > 2: raise ValueError, "Can't handle a weight_array with more than 2d" elif d == 2: weight_array_sum = sum(weight_array, axis=1, dtype=float32)[:, newaxis] zero_index = weight_array_sum<=0 if zero_index.sum() > 0: logger.log_warning("Rows %s of weight_array sum to 0 or less" % where(zero_index)[0]) prob_array = ma.filled( weight_array/ma.masked_where(weight_array_sum<=0, weight_array_sum), 0.0) else: prob_array = weight_array/weight_array_sum elif d == 1: weight_array_sum = weight_array.sum() if weight_array_sum > 0: prob_array = weight_array / float(weight_array_sum) else: logger.log_warning("weight_array sums to 0 or less") prob_array = weight_array * 0 else: raise ValueError, "Can't handle this weight_array" return prob_array
def Main(ZRef_File, Depth_File, N_File, Background_N): # Define constants Cu = 4.5 a = 1. g = 9.8 # m/s^2 # Load in values from rasters as arrays zref = LoadZRef(ZRef_File) depth = LoadDepth(Depth_File) depth = depth[:-1, :-1] print(str(depth.max()) + ' ' + str(depth.mean()) + ' ' + str(depth.min())) # Produced masked array of roughness eps = Epsilon(zref, depth) print(str(zref.max()) + ' ' + str(zref.mean()) + ' ' + str(zref.min())) print(str(eps.max()) + ' ' + str(eps.mean()) + ' ' + str(eps.min())) func = Func(a, eps) n = NCalc(Cu, g, depth, func) print(str(n.max()) + ' ' + str(n.mean()) + ' ' + str(n.min())) # Fill masked values with default of 0.04 Background_N = float(Background_N) n = ma.filled(n, fill_value=Background_N) print(str(n.max()) + ' ' + str(n.mean()) + ' ' + str(n.min())) ReadWriteRaster(ZRef_File, N_File, n)
def write(Maps, file_name, feedback=2): """Write a map to fits file. Map should be a map_data.MapData object. """ # If only a single Map was passed, make it iterable. if not hasattr(Maps, '__iter__'): Maps = (Maps, ) # First create a primary with the history and such: prihdu = pyfits.PrimaryHDU() # Add history to the primary. fname_abbr = ku.abbreviate_file_path(file_name) # Add final history entry and store in the primary header. history = base_data.merge_histories(*Maps) history.add('Written to file.', 'File name: ' + fname_abbr) bf.write_history_header(prihdu.header, history) list_of_hdus = [prihdu] for ii, Map in enumerate(Maps): # Creat an image HDU. map = sp.array(ma.filled(Map.data, float('nan'))) imhdu = pyfits.ImageHDU(sp.swapaxes(map, 0, 2), name='MAP%d' % ii) # Add extra data to the HDU for key in Map.field.iterkeys(): if Map.field_axes[key] != (): raise ce.DataError('Only 0D data can be written to a Fits Map ' 'Header.') card = pyfits.Card(key, Map.field[key].item()) imhdu.header.ascardlist().append(card) list_of_hdus.append(imhdu) # Creat the HDU list and write to file. hdulist = pyfits.HDUList(list_of_hdus) hdulist.writeto(file_name, clobber=True) if feedback > 0: print 'Wrote data to file: ' + fname_abbr
def test_testCopySize(self): # Tests of some subtle points of copying and sizing. n = [0, 0, 1, 0, 0] m = make_mask(n) m2 = make_mask(m) self.assertTrue(m is m2) m3 = make_mask(m, copy=1) self.assertTrue(m is not m3) x1 = np.arange(5) y1 = array(x1, mask=m) self.assertTrue(y1._data is not x1) self.assertTrue(allequal(x1, y1._data)) self.assertTrue(y1.mask is m) y1a = array(y1, copy=0) self.assertTrue(y1a.mask is y1.mask) y2 = array(x1, mask=m, copy=0) self.assertTrue(y2.mask is m) self.assertTrue(y2[2] is masked) y2[2] = 9 self.assertTrue(y2[2] is not masked) self.assertTrue(y2.mask is not m) self.assertTrue(allequal(y2.mask, 0)) y3 = array(x1 * 1.0, mask=m) self.assertTrue(filled(y3).dtype is (x1 * 1.0).dtype) x4 = arange(4) x4[2] = masked y4 = resize(x4, (8, )) self.assertTrue(eq(concatenate([x4, x4]), y4)) self.assertTrue(eq(getmask(y4), [0, 0, 1, 0, 0, 0, 1, 0])) y5 = repeat(x4, (2, 2, 2, 2), axis=0) self.assertTrue(eq(y5, [0, 0, 1, 1, 2, 2, 3, 3])) y6 = repeat(x4, 2, axis=0) self.assertTrue(eq(y5, y6))
def Multimode(D, graph=False): """ Calculates the multimode of the masked array D. graph plots the image with the differently colored regions on top of it. It's not super optimized, it makes a new plot every time it finds a new maximum multimode value. """ N = D.count() Ny, Nx = D.shape RawData = ma.compressed(D) QuartileArray = [ np.percentile(RawData, p) for p in np.linspace(1, 100, 201) ] L = ma.filled(D, 0.) Rmax = 0 for PCT in QuartileArray: IMG = L >= PCT Labels = label(IMG, output='int', structure=[[1, 1, 1], [1, 1, 1], [1, 1, 1]])[0] Sizes = np.sort(np.bincount(Labels.flatten())) if len(Sizes) > 2: Rmax = max(Rmax, Sizes[-3]**2 / float(Sizes[-2])) if graph and (Rmax == Sizes[-3]**2 / float(Sizes[-2])): plt.figure() plt.imshow(D.data, cmap='gray_r', interpolation='nearest') plt.imshow(Labels, interpolation='nearest', alpha=0.3, cmap='cubehelix_r') plt.xlim(20, Nx - 20) plt.ylim(20, Ny - 20) plt.yticks([]) plt.xticks([]) plt.title('Multimode') plt.savefig('MultimodeExample.png', format='png', dpi=300) plt.close() return Rmax
def maskClouds1000m(self, filename): b1_img = Image(self.b1) b2_img = Image(self.b2) b3_img = Image(self.b3) b4_img = Image(self.b4) b5_img = Image(self.b5) b6_img = Image(self.b6) b7_img = Image(self.b7) bandsPaths = [ self.b1, self.b2, self.b3, self.b4, self.b5, self.b6, self.b7 ] newBandsPaths = [] for band in bandsPaths: image = Image(band) band_masked = image.getArray(masked=True, lower_valid_range=0, upper_valid_range=10, cloud_overlay_filename=filename, data_source=self.src) band_masked = ma.filled(band_masked, np.nan) image.save_band( band_masked, image.filename.replace("masked30m_reproject", "masked1000m")) #newBandsPaths.append(image.filename.replace("masked30m", "masked1000m")) newBandsPaths.append( image.filename.replace("masked30m_reproject", "masked1000m")) self.b1 = newBandsPaths[0] self.b2 = newBandsPaths[1] self.b3 = newBandsPaths[2] self.b4 = newBandsPaths[3] self.b5 = newBandsPaths[4] self.b6 = newBandsPaths[5] self.b7 = newBandsPaths[6]
def f(x): """Evaluate a cone around the given centre with the given gradient, i.e. a function whose value increases linearly with distance from the centre. Given an array of shape (num_points, num_dimensions), returns an array of shape (num_points) containing the function values and an array of shape (num_points, num_dimensions) containing the function jacobians. Given an array of shape (num_dimensions), returns a 0D array containing the function value and an array of shape (num_dimensions) containing the function jacobian. """ distance = np.linalg.norm(x - centre, axis=-1) value = distance * gradient distance = np.expand_dims(distance, -1) distance = ma.masked_equal(distance, 0) # Avoid division by zero jacobian = (x - centre) * gradient / distance # The jacobian isn't defined at the centre of the cone but we return a value to keep the optimiser happy. jacobian = ma.filled(jacobian, x) return value, jacobian
def get_xy_values(self, order="C", asmasked=False): """Get X Y coordinate values as numpy 2D arrays.""" nno = self.ncol * self.nrow ier, xvals, yvals = _cxtgeo.surf_xy_as_values( self.xori, self.xinc, self.yori, self.yinc * self.yflip, self.ncol, self.nrow, self.rotation, nno, nno, 0, XTGDEBUG, ) if ier != 0: logger.critical("Error code %s, contact the author", ier) # reshape xvals = xvals.reshape((self.ncol, self.nrow)) yvals = yvals.reshape((self.ncol, self.nrow)) if order == "F": xvals = np.array(xvals, order="F") yvals = np.array(yvals, order="F") if asmasked: tmpv = ma.filled(self.values, fill_value=np.nan) tmpv = np.array(tmpv, order=order) tmpv = ma.masked_invalid(tmpv) mymask = ma.getmaskarray(tmpv) xvals = ma.array(xvals, mask=mymask, order=order) yvals = ma.array(yvals, mask=mymask, order=order) return xvals, yvals
def get_bestfit(arr, also_floats=False): '''return smallest compatible format code for given array >>> x = as_num_array([1234,45,888]) >>> print get_format(x), get_bestfit(x) int32 int16 >>> y = ma.array([12,45555555,88],mask=[0,1,0]) >>> print get_format(y), get_bestfit(y) int32 int8 ''' format = get_format(arr) if format[:3] == 'int': check = ['bool_', 'int8', 'int16', 'int32'] elif format[:4] == 'uint': check = ['bool_', 'uint8', 'uint16', 'uint32'] elif format[:5] == 'float': #TODO: Floats, tricky - need to look for small decimals and fractions #return 'Float32' - this breaks many tests by adding fuzz at the wee digits if also_floats: return 'float32' return format else: #TODO: CharArray squeeze return format if is_masked_array(arr): # masked arrays do not have min(), max() arr = ma.filled(arr) xmin = arr.min() xmax = arr.max() for nformat in check: (minlim, maxlim) = _type_ranges[nformat] if (xmin >= minlim) and (xmax <= maxlim): return nformat return format
def get_xy_values(self, order="C", asmasked=False): """Get X Y coordinate values as numpy 2D arrays.""" nno = self.ncol * self.nrow ier, xvals, yvals = _cxtgeo.surf_xy_as_values( self.xori, self.xinc, self.yori, self.yinc * self.yflip, self.ncol, self.nrow, self.rotation, nno, nno, 0, ) if ier != 0: raise XTGeoCLibError(f"Error in surf_xy_as_values, error code: {ier}") # reshape xvals = xvals.reshape((self.ncol, self.nrow)) yvals = yvals.reshape((self.ncol, self.nrow)) if order == "F": xvals = np.array(xvals, order="F") yvals = np.array(yvals, order="F") if asmasked: tmpv = ma.filled(self.values, fill_value=np.nan) tmpv = np.array(tmpv, order=order) tmpv = ma.masked_invalid(tmpv) mymask = ma.getmaskarray(tmpv) xvals = ma.array(xvals, mask=mymask, order=order) yvals = ma.array(yvals, mask=mymask, order=order) return xvals, yvals
def cp_self_consistent_Aiken_zvode(h1e, cp, nsteps, single_step, zvode_steps): norm_diff = [] rho_list = [] rho_0 = gcp.rho.copy() rho_list.append(rho_0.copy()) for i in range(nsteps): prev_aitken_rho = rho_0.copy() rho_1 = single_step(rho_0, cp, h1e, zvode_steps) rho_2 = single_step(rho_1, cp, h1e, zvode_steps) aitken_rho = rho_2 - (rho_2 - rho_1)**2 / ma.array(rho_2 - 2*rho_1 + rho_0) aitken_rho = ma.filled(aitken_rho, fill_value=rho_2) rho_0 = aitken_rho rho_list.append(rho_0.copy()) norm_diff.append(linalg.norm(aitken_rho - prev_aitken_rho)) if np.allclose(aitken_rho, prev_aitken_rho): print("Iterations converged!") break return norm_diff, rho_list
def annualyield(year, couna, counb): if year <= 2005: bb = year - 1901 aa = year - 1901 else: bb = 104 aa = year - 1901 region1 = NetCDFFile( '/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/HistoricalGLM_crop_150901.nc', 'r') lonlon = region1.variables['lon'][:] maitrop = region1.variables['soy_trop'][bb, :, :] maitemp = region1.variables['soy_temp'][bb, :, :] maitropi = region1.variables['soy_trop_irrig'][bb, :, :] maitempi = region1.variables['soy_temp_irrig'][bb, :, :] gridarea = region1.variables['area'][:, :] maitrop = ma.masked_where(maitrop <= 0, maitrop) maitrop = ma.filled(maitrop, fill_value=0.) maitemp = ma.masked_where(maitemp <= 0, maitemp) maitemp = ma.filled(maitemp, fill_value=0.) maitropi = ma.masked_where(maitropi <= 0, maitropi) maitropi = ma.filled(maitropi, fill_value=0.) maitempi = ma.masked_where(maitempi <= 0, maitempi) maitempi = ma.filled(maitempi, fill_value=0.) maizetor = maitrop + maitemp maizetoi = maitropi + maitempi maizeto = maitrop + maitemp + maitropi + maitempi maizeto, lon1 = shiftgrid(0.5, maizeto, lonlon, start=True) # print lon1 clm = NetCDFFile( '/scratch2/scratchdirs/tslin2/isam/cheyenne/yieldout/isamhistorical_cru/heat/fertfao/fixedirr_new1/soytemp_historical_co2_rf_fert_0.5x0.5.nc', 'r') clmtropf = clm.variables['totalyield'][bb, :, :] clmtropf = ma.masked_where(clmtropf <= 0, clmtropf) clmtropf = ma.filled(clmtropf, fill_value=0.) clm2 = NetCDFFile( '/scratch2/scratchdirs/tslin2/isam/cheyenne/yieldout/isamhistorical_cru/heat/fertfao/fixedirr_new1/soytemp_historical_co2_irrig_fert_0.5x0.5.nc', 'r') clmtropf1 = clm2.variables['totalyield'][bb, :, :] clmtropf1 = ma.masked_where(clmtropf1 <= 0, clmtropf1) clmtropf1 = ma.filled(clmtropf1, fill_value=0.) clm3n = NetCDFFile( '/scratch2/scratchdirs/tslin2/isam/cheyenne/yieldout/isamhistorical_cru/heat/fertfao/fixedirr_new1/soytemp_historical_co2_rf_nofert_0.5x0.5.nc', 'r') isammai = clm3n.variables['totalyield'][bb, :, :] isammai = ma.masked_where(isammai <= 0, isammai) isammai = ma.filled(isammai, fill_value=0.) clm3n1 = NetCDFFile( '/scratch2/scratchdirs/tslin2/isam/cheyenne/plot/finalyield/isam/heat/isamhis_maiscaleifyield_heat.nc', 'r') isamcrumai = clm3n1.variables['yield'][bb, :, :] isamcrumai = ma.masked_where(isamcrumai <= 0, isamcrumai) isamcrumai = ma.filled(isamcrumai, fill_value=0.) clm3n2 = NetCDFFile( '/scratch2/scratchdirs/tslin2/isam/cheyenne/yieldout/isamhistorical_cru/heat/fertfao/fixedirr_new1/soytemp_historical_co2_irrig_nofert_0.5x0.5.nc', 'r') isamcrumai2 = clm3n2.variables['totalyield'][aa, :, :] isamcrumai2 = ma.masked_where(isamcrumai2 <= 0, isamcrumai2) isamcrumai2 = ma.filled(isamcrumai2, fill_value=0.) clm3n3 = NetCDFFile( '/scratch2/scratchdirs/tslin2/isam/cheyenne/plot/finalyield/isam/heat/fertall/isamhis_maiscaleiyield_heat_fertall.nc', 'r') isamcrumai3 = clm3n3.variables['yield'][bb, :, :] isamcrumai3 = ma.masked_where(isamcrumai3 <= 0, isamcrumai3) isamcrumai3 = ma.filled(isamcrumai3, fill_value=0.) yieldclm1 = 0 yieldclm = 0. yieldisam = 0. yieldisamc = 0. yieldisamc2 = 0. yieldisamc3 = 0. a = 0 harea = 0 yieldc1 = 0 yieldc = 0. yieldi = 0. yieldic = 0. yieldic2 = 0. yieldic3 = 0. #USA 11501~11550 for xx in range(0, 360): for yy in range(0, 720): if coun[xx, yy] >= couna and coun[xx, yy] <= counb: harea = maizeto[xx, yy] * gridarea[xx, yy] + harea yieldclm1 = (clmtropf1[xx, yy] * maizeto[xx, yy] * gridarea[xx, yy]) + yieldclm1 yieldclm = (clmtropf[xx, yy] * maizeto[xx, yy] * gridarea[xx, yy]) + yieldclm yieldisam = (isammai[xx, yy] * maizeto[xx, yy] * gridarea[xx, yy]) + yieldisam yieldisamc = (isamcrumai[xx, yy] * maizeto[xx, yy] * gridarea[xx, yy]) + yieldisamc yieldisamc2 = (isamcrumai2[xx, yy] * maizeto[xx, yy] * gridarea[xx, yy]) + yieldisamc2 yieldisamc3 = (isamcrumai3[xx, yy] * maizeto[xx, yy] * gridarea[xx, yy]) + yieldisamc3 a = a + 1 yieldc1 = N.average(clmtropf1, weights=maizeto) yieldc = N.average(clmtropf, weights=maizeto) yieldi = N.average(isammai, weights=maizeto) yieldic = yieldisamc / harea yieldic2 = N.average(isamcrumai2, weights=maizeto) yieldic3 = yieldisamc3 / harea return harea, yieldc, yieldi, yieldic, yieldclm, yieldisam, yieldisamc, yieldic2, yieldic3, yieldisamc2, yieldisamc3, yieldc1
import numpy.ma as ma from scipy.interpolate import griddata from scipy.stats import ttest_ind region1 = NetCDFFile( '/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/RCP45_crop_150901.nc', 'r') maitrop = region1.variables['maize_trop'][4, :, :] maitemp = region1.variables['maize_temp'][4, :, :] maitropi = region1.variables['maize_trop_irrig'][4, :, :] maitempi = region1.variables['maize_temp_irrig'][4, :, :] maitrop = ma.masked_where(maitrop <= 0, maitrop) maitropi = ma.masked_where(maitropi <= 0, maitropi) maitemp = ma.masked_where(maitemp <= 0, maitemp) maitempi = ma.masked_where(maitempi <= 0, maitempi) maitrop = ma.filled(maitrop, fill_value=0.) maitropi = ma.filled(maitropi, fill_value=0.) maitemp = ma.filled(maitemp, fill_value=0.) maitempi = ma.filled(maitempi, fill_value=0.) maizeto = maitrop + maitemp maizetoi = maitropi + maitempi maitoatemp = maitemp + maitempi maitoatrop = maitrop + maitropi maizetotal = maizeto + maizetoi clm = NetCDFFile( '/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/clm45rcp45/maizetrop_rcp45_constco2_rf_nofert_0.5x0.5.nc', 'r') clma = NetCDFFile( '/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/clm45rcp45/maizetrop_rcp45_co2_rf_nofert_0.5x0.5.nc', 'r') #clm=NetCDFFile('/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/clm45rcp45/maizetrop_rcp45_co2_rf_fert_0.5x0.5.nc','r')
def netcdf_preloader(crop_size=50, tmp_array_size=200, admin_config_file_path=None, path_output='.', dataframe=None, target_datetimes=None, stations=None): """ Preprocess netcdf files which are related to target_datetimes and stations The resulting preprocessed netcdf files will be stored in path_output If admin_config_file_path is not specified, the following have to be specified: * dataframe * target_datetimes * stations If admin_config_file is specified, it overwrites the parameters specified above. :param crop_size: The crop size around each station in pixels :param tmp_array_size: nb of timestamps to fit in temporary array :param admin_config_file_path: The admin configuration file path :param path_output: The folder where the outputted preprocessed netcdf files will be :param dataframe: a pandas dataframe that provides the netCDF file path (or HDF5 file path and offset) for all relevant timestamp values over the test period. :param target_datetimes: a list of timestamps that your data loader should use to provide imagery for your model. The ordering of this list is important, as each element corresponds to a sequence of GHI values to predict. By definition, the GHI values must be provided for the offsets given by ``target_time_offsets`` which are added to each timestamp (T=0) in this datetimes list. :param stations: a map of station names of interest paired with their coordinates (latitude, longitude, elevation). """ if admin_config_file_path: admin_config_dict = helpers.load_dict(admin_config_file_path) dataframe_path = admin_config_dict['dataframe_path'] with open(dataframe_path, 'rb') as df_file_handler: dataframe = pickle.load(df_file_handler) target_datetimes = [ datetime.datetime.strptime(date_time, '%Y-%m-%dT%H:%M:%S') for date_time in admin_config_dict['target_datetimes'] ] stations = admin_config_dict['stations'] # hard coded values for now n_channels = 5 n_timestep = 5 dc = crop_size // 2 ddt = datetime.timedelta(minutes=15) # number of sample points n_sample = len(target_datetimes) # Generate all datetimes including prior timesteps from targets all_dt = [] for dt0 in target_datetimes: for i in range(4, 0, -1): all_dt.append(dt0 - i * ddt) all_dt.append(dt0) chunksizes = min(256, n_sample) # Initialize output netcdf files (one for each station) nc_outs = {} os.makedirs(path_output, exist_ok=True) for station, coord in stations.items(): nc_outs[station] = netCDF4.Dataset( os.path.join(path_output, f'{station}.nc'), 'w') nc_outs[station].createDimension('time', n_sample) nc_outs[station].createDimension('lat', crop_size) nc_outs[station].createDimension('lon', crop_size) nc_outs[station].createDimension('channel', n_channels) nc_outs[station].createDimension('timestep', n_timestep) time = nc_outs[station].createVariable('time', 'f8', ('time', )) time.calendar = 'standard' time.units = 'days since 1970-01-01 00:00:00' nc_outs[station].createVariable('lat', 'f4', ('lat', )) nc_outs[station].createVariable('lon', 'f4', ('lon', )) nc_outs[station].createVariable( f'data', 'f4', ('time', 'timestep', 'channel', 'lat', 'lon'), zlib=True, chunksizes=(chunksizes, n_timestep, n_channels, crop_size, crop_size)) # Initialize temporary arrays to store data to limit constantly writing to disk init = True tmp_arrays = {} coord_ij = {} for station, coord in stations.items(): tmp_arrays[f"{station}"] = ma.masked_all( (tmp_array_size, n_timestep, n_channels, crop_size, crop_size)) tmp_arrays["time"] = ma.masked_all((tmp_array_size, )) # Loop through all timesteps and load images. The sequence moves through # T-60min, T-45min, T-30min, T-15min, T0 and store those in the timestep # dimension in the netcdf files. A temporary array is used in case the number # of timestamps to process would be to big for memory. for t, dt in enumerate(tqdm.tqdm(all_dt)): at_t0 = not ((t + 1) % n_timestep) t_sample = t // n_timestep t_sample_tmp = t_sample % tmp_array_size timestep_id = t % n_timestep k = dataframe.index.get_loc(dt) nc_path = dataframe['ncdf_path'][k] try: nc_loop = netCDF4.Dataset(nc_path, 'r') except OSError: # What to do when the netCDF4 file is not available. # Currently filling with 0 later in the code... if at_t0: tmp_arrays["time"][t_sample_tmp] = netCDF4.date2num( dt, time.units, time.calendar) else: if init: lat_loop = nc_loop['lat'][:] lon_loop = nc_loop['lon'][:] for station, coord in stations.items(): lat_diff = np.abs(lat_loop - coord[0]) i = np.where(lat_diff == lat_diff.min())[0][0] lon_diff = np.abs(lon_loop - coord[1]) j = np.where(lon_diff == lon_diff.min())[0][0] nc_outs[station].variables['lat'][:] = lat_loop[i - dc:i + dc] nc_outs[station].variables['lon'][:] = lon_loop[j - dc:j + dc] coord_ij[station] = (i, j) init = False for d, c in enumerate([1, 2, 3, 4, 6]): channel_data = nc_loop.variables[f'ch{c}'][0, :, :] for station, coord in stations.items(): i, j = coord_ij[station] tmp_arrays[f"{station}"][t_sample_tmp, timestep_id, d, :, :] = \ channel_data[i - dc:i + dc, j - dc:j + dc] if at_t0: tmp_arrays["time"][t_sample_tmp] = nc_loop.variables['time'][0] nc_loop.close() if ((t_sample_tmp == (tmp_array_size - 1)) and (timestep_id == n_timestep - 1)) or (t == (len(all_dt) - 1)): t0 = t_sample - t_sample_tmp for station, coord in stations.items(): # Here we fill missing values with 0 nc_outs[station]['data'][t0:t_sample + 1, :, :, :, :] = \ ma.filled(tmp_arrays[f"{station}"][:t_sample_tmp + 1, :, :, :, :], 0) tmp_arrays[f"{station}"] = \ ma.masked_all((tmp_array_size, n_timestep, n_channels, crop_size, crop_size)) nc_outs[station]['time'][t0:t_sample + 1] = \ tmp_arrays['time'][:t_sample_tmp + 1] tmp_arrays["time"] = ma.masked_all((tmp_array_size, )) for station, coord in stations.items(): nc_outs[station].close()
def _zone_averaging( xprop, yprop, zoneprop, zone_minmax, coarsen, zone_avg, dzprop, mprop, summing=False ): # General preprocessing, and... # Change the 3D numpy array so they get layers by # averaging across zones. This may speed up a lot, # but will reduce the resolution. # The x y coordinates shall be averaged (ideally # with thickness weighting...) while e.g. hcpfzprop # must be summed. # Somewhat different processing whether this is a hc thickness # or an average. xpr = xprop ypr = yprop zpr = zoneprop dpr = dzprop mpr = mprop if coarsen > 1: xpr = xprop[::coarsen, ::coarsen, ::].copy(order="C") ypr = yprop[::coarsen, ::coarsen, ::].copy(order="C") zpr = zoneprop[::coarsen, ::coarsen, ::].copy(order="C") dpr = dzprop[::coarsen, ::coarsen, ::].copy(order="C") mpr = mprop[::coarsen, ::coarsen, ::].copy(order="C") zpr.astype(np.int32) if zone_avg: zmin = int(zone_minmax[0]) zmax = int(zone_minmax[1]) if zpr.min() > zmin: zmin = zpr.min() if zpr.max() < zmax: zmax = zpr.max() newx = [] newy = [] newz = [] newm = [] newd = [] for izv in range(zmin, zmax + 1): logger.info("Averaging for zone %s ...", izv) xpr2 = ma.masked_where(zpr != izv, xpr) ypr2 = ma.masked_where(zpr != izv, ypr) zpr2 = ma.masked_where(zpr != izv, zpr) dpr2 = ma.masked_where(zpr != izv, dpr) mpr2 = ma.masked_where(zpr != izv, mpr) # get the thickness and normalize along axis 2 (vertical) # to get normalized thickness weights lay_sums = dpr2.sum(axis=2) normed_dz = dpr2 / lay_sums[:, :, np.newaxis] # assume that coordinates have equal weights within a zone xpr2 = ma.average(xpr2, axis=2) ypr2 = ma.average(ypr2, axis=2) zpr2 = ma.average(zpr2, axis=2) # avg zone dpr2 = ma.sum(dpr2, axis=2) if summing: mpr2 = ma.sum(mpr2, axis=2) else: mpr2 = ma.average(mpr2, weights=normed_dz, axis=2) # avg zone newx.append(xpr2) newy.append(ypr2) newz.append(zpr2) newd.append(dpr2) newm.append(mpr2) xpr = ma.dstack(newx) ypr = ma.dstack(newy) zpr = ma.dstack(newz) dpr = ma.dstack(newd) mpr = ma.dstack(newm) zpr.astype(np.int32) xpr = ma.filled(xpr, fill_value=xtgeo.UNDEF) ypr = ma.filled(ypr, fill_value=xtgeo.UNDEF) zpr = ma.filled(zpr, fill_value=0) dpr = ma.filled(dpr, fill_value=0.0) mpr = ma.filled(mpr, fill_value=0.0) return xpr, ypr, zpr, mpr, dpr
def deriv(*positional_inputs, **keyword_inputs): """Calculate the derivative along a single dimension. Calling Sequence: Result = deriv([x_in,] y_in, missing=1e+20, algorithm='default') Positional Input Arguments: * x_in: Abscissa values of y_in to take with respect to. If not set, the derivative of y_in is take with respect to unit abscissa intervals. Numeric array of same shape and size as y_in. Must be monotonic and with no duplicate values. Optional. First positional argument out of two, if present. * y_in: Ordinate values, to take the derivative with respect to. Numeric array vector of rank 1. Required. Second posi- tional argument, if x_in is present; only positional argument if x_in is absent. Keyword Input Arguments: * missing: If y_in and/or x_in has missing values, this is the missing value value. Scalar. Default is 1e+20. * algorithm: Name of the algorithm to use. String scalar. Default is 'default'. Possible values include: + 'default': Default method (currently set to 'order1'). + 'order1': First-order finite-differencing (backward and forward differencing used at the endpoints, and centered differencing used everywhere else). If abscissa intervals are irregular, differencing will be correspondingly asym- metric. Output Result: * Derivative of y_in with respect to x_in (or unit interval abscissa, if x_in is not given). Numeric array of same shape and size as y_in. If there are missing values, those elements in the output are set to the value in |missing|. For instance, if y_in is only one element, a one-element vector is returned as the derivative with the value of |missing|. If there are missing values in the output due to math errors and |missing| is set to None, output will fill those missing values with the MA default value of 1e+20. References: * Press, W. H., et al. (1992): Numerical Recipes in Fortran 77: The Art of Scientific Computing. New York, NY: Cambridge University Press, pp. 180-184. * Wang, Y. (1999): "Numerical Differentiation," Introduction to MHD Numerical Simulation in Space, ESS265: Instrumentation, Data Processing and Data Analysis in Space Physics (UCLA). URL: http://www-ssc.igpp.ucla.edu/personnel/russell/ESS265/ Ch10/ylwang/node21.html. Example with one argument, no missing values, using the default method: >>> from deriv import deriv >>> import Numeric as N >>> y = N.sin(N.arange(8)) >>> dydx = deriv(y) >>> ['%.7g' % dydx[i] for i in range(4)] ['0.841471', '0.4546487', '-0.3501755', '-0.83305'] >>> true = N.cos(N.arange(8)) #- Compare with exact solution >>> ['%.7g' % true[i] for i in range(4)] ['1', '0.5403023', '-0.4161468', '-0.9899925'] Example with two arguments with missing values, using first- order differencing: >>> x = N.arange(8)/(2.*N.pi) >>> y = N.sin(x) >>> y[3] = 1e20 #- Set an element to missing value >>> dydx = deriv(x, y, missing=1e20, algorithm='order1') >>> ['%.7g' % dydx[i] for i in range(5)] ['0.9957836', '0.9831985', '1e+20', '0.8844179', '1e+20'] >>> true = N.cos(x) #- Compare with exact solution >>> ['%.7g' % true[i] for i in range(5)] ['1', '0.9873616', '0.9497657', '0.8881628', '0.8041098'] """ import numpy.ma as MA import numpy as N #- Establish y_in and x_in from *positional_inputs: if len(positional_inputs) == 1: y_in = positional_inputs[0] x_in = N.arange(len(y_in)) elif len(positional_inputs) == 2: x_in = positional_inputs[0] y_in = positional_inputs[1] else: raise ValueError, "deriv: Bad inputs" #- Establish missing and algorithm from *keyword_inputs: if keyword_inputs.has_key('missing') == 1: missing = keyword_inputs['missing'] else: missing = 1e+20 if keyword_inputs.has_key('algorithm') == 1: algorithm = keyword_inputs['algorithm'] else: algorithm = 'default' #- Check positional and keyword inputs for possible errors: if (len(y_in.shape) != 1) or (len(x_in.shape) != 1): raise ValueError, "deriv: Inputs not a vector" if type(algorithm) != type(''): raise ValueError, "deriv: algorithm not str" #- Set algorithm_to_use variable, based on the algorithm keyword. # The algorithm_to_use tells which algorithm below to actually # use (so here is where we set what algorithm to use for default): if algorithm == 'default': algorithm_to_use = 'order1' else: algorithm_to_use = algorithm #- Change input to MA: just set to input value unless there are # missing values, in which case add mask: if missing == None: x = MA.masked_array(x_in) y = MA.masked_array(y_in) else: x = MA.masked_values(x_in, missing, copy=0) y = MA.masked_values(y_in, missing, copy=0) #- Calculate and return derivative: # * Create working arrays that are consistent with a 3-point # stencil in the interior and 2-point stencil on the ends: # *im1 means the point before index i, *ip1 means the point # after index i, and the i index array is just plain x or # y; the endpadded arrays replicate the ends of x and y. # I use an MA array filled approach instead of concatentation # because the MA concatenation routine doesn't work right # when the endpoint element is a missing value: x_endpadded = MA.zeros(len(x) + 2) x_endpadded[0] = x[0] x_endpadded[1:-1] = x x_endpadded[-1] = x[-1] y_endpadded = MA.zeros(len(y) + 2) y_endpadded[0] = y[0] y_endpadded[1:-1] = y y_endpadded[-1] = y[-1] y_im1 = y_endpadded[:-2] y_ip1 = y_endpadded[2:] x_im1 = x_endpadded[:-2] x_ip1 = x_endpadded[2:] # * Option 1: First-order differencing (interior points use # centered differencing, and end points use forward or back- # ward differencing, as applicable): if algorithm_to_use == 'order1': dydx = (y_ip1 - y_im1) / (x_ip1 - x_im1) # * Option 2: Bad algorithm specified: else: raise ValueError, "deriv: bad algorithm" #- Return derivative as Numeric array: return MA.filled(dydx, missing)
mask=NetCDFFile('india_mask.nc','r') ind = mask.variables['MASK'][:,:] ind= ma.masked_where(ind<=0.0,ind) area=NetCDFFile('/scratch2/scratchdirs/tslin2/plot/globalcrop/data/gridareahalf.nc','r') gridarea = area.variables['cell_area'] isam=NetCDFFile('/scratch2/scratchdirs/tslin2/isam/rice_cheyenne/his_cru/egu_new/ric_irr_fert/output/ric_irr_fert.nc','r') lonisam1=isam.variables['lon'][:] ind,lona11=shiftgrid(180.5,ind,lonisam1,start=False) ric_i_f=isam.variables['totalyield'][79:109,:,:] ric_i_f,lona11=shiftgrid(180.5,ric_i_f,lonisam1,start=False) ric_i_f=ma.masked_where(ric_i_f<=0.0,ric_i_f) ric_i_f=ma.filled(ric_i_f, fill_value=0.) isam=NetCDFFile('/scratch2/scratchdirs/tslin2/isam/rice_cheyenne/his_cru/egu_new/ric_fert/output/ric_fert.nc','r') ric_f=isam.variables['totalyield'][79:109,:,:] ric_f,lona11=shiftgrid(180.5,ric_f,lonisam1,start=False) ric_f=ma.masked_where(ric_f<=0.0,ric_f) ric_f=ma.filled(ric_f, fill_value=0.) isam=NetCDFFile('/scratch2/scratchdirs/tslin2/isam/rice_cheyenne/his_cru/egu_new/ric_irrfixed_fert/output/ric_irrfixed_fert.nc','r') #isam=NetCDFFile('/scratch2/scratchdirs/tslin2/isam/rice_cheyenne/his_cru/egu_new/ric_fert_constcli/output/ric_fert_constcli.nc','r') ric_fc=isam.variables['totalyield'][79:109,:,:] ric_fc,lona11=shiftgrid(180.5,ric_fc,lonisam1,start=False)
gridarea1 = area.variables['cell_area'][:, :] gridlon = area.variables['lon'][:] gridlat = area.variables['lat'][:] #print gridlon nclu = NetCDFFile( '/scratch2/scratchdirs/tslin2/plot/globalcrop/data/m3yield_isam.nc', 'r') ncvar_maize = nclu.variables['soyy'][0, :, :] marea = nclu.variables['soy_area'][0, :, :] region1 = NetCDFFile( '/scratch2/scratchdirs/tslin2/isam/maisoy_cheyenne/code/isamhiscru_soy_luh2.nc', 'r') maitrop = region1.variables['area'][95:105, :, :] lonisam1 = region1.variables['lon'][:] maitrop = ma.masked_where(maitrop <= 0, maitrop) maitrop = ma.filled(maitrop, fill_value=0.) maizeto = maitrop edat = NetCDFFile( '/scratch2/scratchdirs/tslin2/isam/maisoy_cheyenne/his_cru/equili/soyequilibrium/output/soyequilibrium.nc', 'r') eiyield1ynew = edat.variables['totalyield'][95:105, :, :] edat2 = NetCDFFile( '/scratch2/scratchdirs/tslin2/isam/maisoy_cheyenne/his_cru/equili/soyequilibrium_irr/output/soyequilibrium_irr.nc', 'r') eiyield2ynew = edat2.variables['totalyield'][95:105, :, :] edat3 = NetCDFFile( '/scratch2/scratchdirs/tslin2/isam/maisoy_cheyenne/his_cru/fixedirr/soy_irr/output/soy_irr.nc',
def post_check(self, values, dataset_pool): self.do_check("0 <= x and x <= 100", ma.filled(values,0))
def compute(self, dataset_pool): hh_wwd = self.get_dataset().get_attribute(self.number_of_households_within_walking_distance) return 100.0*ma.filled(self.get_dataset().get_attribute(self.number_of_high_income_households_within_walking_distance)/ ma.masked_where(hh_wwd == 0, hh_wwd.astype(float32)), 0.0)
region1 = NetCDFFile( '/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/HistoricalGLM_crop_150901.nc', 'r') #maitrop = region1.variables['soy_trop'][99,:,:] #maitemp = region1.variables['soy_temp'][99,:,:] #maitropi=region1.variables['soy_trop_irrig'][99,:,:] #maitempi=region1.variables['soy_temp_irrig'][99,:,:] maitrop = N.average(region1.variables['soy_trop'][95:105, :, :], axis=0) maitemp = N.average(region1.variables['soy_temp'][95:105, :, :], axis=0) maitropi = N.average(region1.variables['soy_trop_irrig'][95:105, :, :], axis=0) maitempi = N.average(region1.variables['soy_temp_irrig'][95:105, :, :], axis=0) gridarea = region1.variables['area'][:, :] maitrop = ma.masked_where(maitrop <= 0, maitrop) maitrop = ma.filled(maitrop, fill_value=0.) maitemp = ma.masked_where(maitemp <= 0, maitemp) maitemp = ma.filled(maitemp, fill_value=0.) maitropi = ma.masked_where(maitropi <= 0, maitropi) maitropi = ma.filled(maitropi, fill_value=0.) maitempi = ma.masked_where(maitempi <= 0, maitempi) maitempi = ma.filled(maitempi, fill_value=0.) maizetor = maitrop + maitemp maizetoi = maitropi + maitempi maizeto = maitrop + maitemp + maitropi + maitempi dat = NetCDFFile( '/scratch2/scratchdirs/tslin2/isam/cheyenne/yieldout/isamhistorical_cru/heat/fertfao/fixedirr_new1/soytemp_historical_co2_irrig_fert_0.5x0.5.nc', 'r') iyield1y = N.average(dat.variables['totalyield'][95:105, :, :], axis=0)
meareaisam = ma.masked_where(ind1 != 8, meareaisam) area = NetCDFFile( '/scratch2/scratchdirs/tslin2/plot/globalcrop/data/gridareahalf_isam.nc', 'r') gridarea = area.variables['cell_area'] isam = NetCDFFile( '/scratch2/scratchdirs/tslin2/isam/rice_cheyenne/his_cru/aogs/experiments/ric_irr_fert/output/ric_irr_fert.nc', 'r') lonisam1 = isam.variables['lon'][:] tric_i_f = isam.variables['totalyield'][79:109, :, :] tric_i_f = ma.masked_where(tric_i_f <= 0.0, tric_i_f) tric_i_f = ma.filled(tric_i_f, fill_value=0.) tric_i_f = ma.masked_where(ind1 != 8, tric_i_f) isam = NetCDFFile( '/scratch2/scratchdirs/tslin2/isam/rice_cheyenne/his_cru/aogs/experiments/ric_fert/output/ric_fert.nc', 'r') tric_f = isam.variables['totalyield'][79:109, :, :] tric_f = ma.masked_where(tric_f <= 0.0, tric_f) tric_f = ma.filled(tric_f, fill_value=0.) tric_f = ma.masked_where(ind1 != 8, tric_f) isam = NetCDFFile( '/scratch2/scratchdirs/tslin2/isam/rice_cheyenne/his_cru/aogs/experiments/ric_cli/output/ric_cli.nc', 'r') tric_fc = isam.variables['totalyield'][79:109, :, :] tric_fc = ma.masked_where(tric_fc <= 0.0, tric_fc)
def yieldout(year): bb = year - 1900 bb1 = year - 850 isam1 = NetCDFFile( '/scratch2/scratchdirs/tslin2/plot/globalcrop/data/mirca_isam.nc', 'r') maitrop1 = isam1.variables['asoy_rf'][:, :] #mirca2000 maitropi1 = isam1.variables['asoy_irr'][:, :] #mirca2000 maitrop1 = ma.masked_where(maitrop1 <= 0, maitrop1) maitrop1 = ma.filled(maitrop1, fill_value=0.) maitropi1 = ma.masked_where(maitropi1 <= 0, maitropi1) maitropi1 = ma.filled(maitropi1, fill_value=0.) lonisam = isam1.variables['lon'][:] maitrop, lonisam1 = shiftgrid(180.5, maitrop1, lonisam, start=False) maitropi, lonisam1 = shiftgrid(180.5, maitropi1, lonisam, start=False) maizetor = maitrop maizetoi = maitropi maizetrop = maitrop + maitropi maizeto = maitrop + maitropi isam = NetCDFFile( '/scratch2/scratchdirs/tslin2/isam/maisoy_cheyenne/his_cru/new/soy_fert/output/soy_fert.nc', 'r') clmtropf = isam.variables['g_ET'][bb - 1, 1, :, :] clm2 = NetCDFFile( '/scratch2/scratchdirs/tslin2/isam/maisoy_cheyenne/his_cru/new/soy_irr_fert/output/soy_irr_fert.nc', 'r') clmtropfi = clm2.variables['g_ET'][bb - 1, 1, :, :] lonisam = clm2.variables['lon'][:] clmtropf, lonisam1 = shiftgrid(180.5, clmtropf, lonisam, start=False) clmtropfi, lonisam1 = shiftgrid(180.5, clmtropfi, lonisam, start=False) print lonisam1 clmtropf = ma.masked_where(maitrop <= 0, clmtropf) clmtropf = ma.filled(clmtropf, fill_value=0.) clmtropfi = ma.masked_where(maitropi <= 0, clmtropfi) clmtropfi = ma.filled(clmtropfi, fill_value=0.) yield_clmtf = clmtropf yield_clmtf = ma.masked_where(yield_clmtf <= 0, yield_clmtf) yield_clmtf = ma.filled(yield_clmtf, fill_value=0.) yield_clmtfi = clmtropfi yield_clmtfi = ma.masked_where(yield_clmtfi <= 0, yield_clmtfi) yield_clmtfi = ma.filled(yield_clmtfi, fill_value=0.) area = NetCDFFile( '/scratch2/scratchdirs/tslin2/plot/globalcrop/data/gridareahalf_isam.nc', 'r') gridarea = area.variables['cell_area'][:, :] gridlon = area.variables['lon'][:] gridlat = area.variables['lat'][:] gridarea, gridlon = shiftgrid(180.5, gridarea, gridlon, start=False) lon2, lat2 = N.meshgrid(gridlon, gridlat) map = Basemap(projection='cyl', llcrnrlat=-65, urcrnrlat=90, llcrnrlon=-180, urcrnrlon=180, resolution='c') x, y = map(lon2, lat2) yield_clmtf = maskoceans(x, y, yield_clmtf) yield_clmtf = ma.masked_where(maizeto <= 0, yield_clmtf) yield_clmtfi = maskoceans(x, y, yield_clmtfi) yield_clmtfi = ma.masked_where(maizeto <= 0, yield_clmtfi) clmy = ((yield_clmtf * maizetor) + (yield_clmtfi * maizetoi)) / ((maizetoi) + (maizetor)) areall = (maizetoi) + (maizetor) clmall = ((yield_clmtf * maizetor) + (yield_clmtfi * maizetoi)) return clmy, areall, clmall