Exemplo n.º 1
0
def kdtree_sample2d(xin, yin, z2d, xout, yout, distance=2.,method='linear'):
    """ bin random data points to grids
    loc_points: 2 x dpoints, lon, lat
    loc_grids: 2 x dgrids, x.ravel, y.ravel
    """
    from scipy import spatial, ma
    from numpy import meshgrid, exp, array,c_, where
    xip,xin  = find_overlap(xin, xout)
    yip,yin = find_overlap(yin, yout)
    z2ds = z2d[where(yip==True)[0],:][:,where(xip==True)[0]]
        
    ismask = ma.is_masked(z2ds)
    
    xin2d,yin2d = meshgrid(xin, yin)
    
    if ismask:
        xin1d, yin1d = xin2d[z2ds.mask==False].ravel(), yin2d[z2ds.mask==False].ravel()
        z1d = z2ds[z2ds.mask==False]
        locs = c_[xin1d, yin1d]
    else:
        xin1d, yin1d = xin2d.ravel(), yin2d.ravel()
        z1d = z2ds.ravel()
        locs = c_[xin1d, yin1d]
    
    tree = spatial.cKDTree(locs)
    
    grids = zip(xout, yout)

    index = tree.query_ball_point(grids, distance)

    Tmis=[]
    sample_size=[]
    
    for i in range(xout.size):
        ip = index[i]
        if len(ip) == 0:
            Tmis.append(999999)
            sample_size.append(0)
        else:
            dis = ((xin1d[ip]-xout[i])**2+(yin1d[ip]-yout[i])**2)
            if method=='linear':
                dis = ma.masked_greater(dis**0.5, distance)
                weight = distance - dis**0.5
            else:
                weight = exp(-(dis/distance**2))
            weight = weight/weight.sum()
            Tmis.append((weight*z1d[ip]).sum())
            sample_size.append(len(ip))
            
    zout = ma.masked_greater(array(Tmis),1e5)
    sample_size = ma.masked_equal(array(sample_size),0)
    return zout, sample_size
Exemplo n.º 2
0
def read(grb_name):
    """Read the MPE data from a grib file into an array.

    This function uses wgrib2 to write and then read the MPE
    data via an intermediate binary file, which is later
    deleted. The non-raining regions are masked and the data
    converted into mm/hr.

    """
    
    rows = 3712 # fixed for MPE grid
    cols = 3712 # fixed for MPE grid

    bin_name = grb_name[0:-4] + '.bin'

    exec_str = 'wgrib2 -no_f77 ' + '-bin ' + bin_name + ' ' + grb_name # spaces are N.B.

    execute(exec_str)
    a = read_bin_data(bin_name)
    os.remove(bin_name)

    a = a.reshape(rows, cols)

    # data in grib file are oriented EW:SN
    # need to check that this is correct, the plots look fine...
    a = np.fliplr(a)
##    a = np.flipud(a)

    # mask
    a = ma.masked_greater(a, 9E20)

    # convert from mm/s to mm/hr (I think...)
    a = 3600*a

    return a
Exemplo n.º 3
0
    def _breakList(self, inList, index, parameter):
        par = float(parameter)

        array = N.empty(shape=[len(inList),],dtype=N.float64)
        i = 0
        for parameters in inList:
            array[i] = parameters[index]
            i = i + 1 

        greater = MA.masked_less(array, par)
        less = MA.masked_greater(array, par)

        upper = MA.minimum(greater)
        lower = MA.maximum(less)

        upperArray = MA.masked_inside(array, par, upper)
        lowerArray = MA.masked_inside(array, lower, par)

        upperList = []
        lowerList = []
        i = 0
        for parameters in inList:
            if upperArray.mask[i]:
                upperList.append(parameters)
            if lowerArray.mask[i]:
                lowerList.append(parameters)
            i = i + 1

        return upperList, lowerList
Exemplo n.º 4
0
    def action(self, state):
        mesh = state.mesh.copy().apply_transform(state.T_obj_world.matrix)

        mesh.fix_normals()
        vertices, face_ind = sample.sample_surface_even(mesh, self.num_samples)
        normals = mesh.face_normals[face_ind]
        
        angles = normals.dot(up)
        valid_pushes = np.logical_and(ma.masked_greater(angles, 1.39626), ma.masked_less(angles, 1.74533)) # within 10 degrees of horizontal
        if not np.any(valid_pushes):
            return LinearPushAction(None, None, metadata={'vertex': np.array([0,0,0]), 'normal': np.array([1,0,0])})
        best_valid_ind = np.argmax(normals[valid_pushes][:,2]) # index of best 
        best_ind = np.arange(self.num_samples)[valid_pushes][best_valid_ind]
        
        start_position = vertices[best_ind] + normals[best_ind] * .015
        end_position = vertices[best_ind] - normals[best_ind] * .04
        
        start_pose, end_pose = self.get_hand_pose(start_position, end_position)
        return LinearPushAction(
            start_pose,
            end_pose,
            metadata={
                'vertex': vertices[best_ind],
                'normals': normals[best_ind],
            }
        )
Exemplo n.º 5
0
def import_ijxyz_ascii_tmpl(self, mfile, template):
    """Import OW/DSG IJXYZ ascii format, with a Cube or RegularSurface
    instance as template."""

    fin = xtgeo._XTGeoCFile(mfile)

    if isinstance(template, (xtgeo.cube.Cube, xtgeo.surface.RegularSurface)):
        logger.info("OK template")
    else:
        raise ValueError("Template is of wrong type: {}".format(
            type(template)))

    nxy = template.ncol * template.nrow
    _iok, val = _cxtgeo.surf_import_ijxyz_tmpl(fin.fhandle, template.ilines,
                                               template.xlines, nxy, 0)

    val = ma.masked_greater(val, _cxtgeo.UNDEF_LIMIT)

    self._xori = template.xori
    self._xinc = template.xinc
    self._yori = template.yori
    self._yinc = template.yinc
    self._ncol = template.ncol
    self._nrow = template.nrow
    self._rotation = template.rotation
    self._yflip = template.yflip
    self._values = val.reshape((self._ncol, self._nrow))
    self._filesrc = mfile

    self._ilines = template._ilines.copy()
    self._xlines = template._xlines.copy()

    fin.close()
Exemplo n.º 6
0
def masked_cube_simple(mycube, slicevar, v1, v2, threshold):
    """

    Mask function 1 -- simple cube cropping
    masking for a specific variable slicevar (string)
    arguments: cube, variable, min value, max value, threshold

    """
    import numpy.ma as ma
    coord_names = [coord.name() for coord in mycube.coords()]
    if slicevar in coord_names:
        coord = mycube.coord(slicevar)
        print('Masking on variable: %s' % coord.standard_name)
        cubeslice = mycube.extract(
            iris.Constraint(
                coord_values={
                    coord.standard_name: lambda cell: v1 <= cell.point <= v2
                }))
        if cubeslice is not None:
            masked_cubeslice = cubeslice.copy()
            masked_cubeslice.data = ma.masked_greater(cubeslice.data,
                                                      threshold)
            print('Masking cube keeping only what is in between %f and %f' %
                  (v1, v2))
            return masked_cubeslice
        else:
            print('NOT masking the cube')
            return mycube
    else:
        print('Variable is not a cube dimension, leaving cube untouched')
        return mycube
def extract_lai(nc_file):
    fh_in = Dataset('../../raw_data/lai/' + nc_file, 'r')

    for index, n_days in enumerate(fh_in.variables['time'][:]):
        date = (datetime.datetime(2000, 1, 1, 0, 0) + datetime.timedelta(int(n_days))).strftime('%Y%m%d')
        print(date)
        fh_out = Dataset('../../processed_data/lai/500m/{}.nc'.format(date), 'w')

        for name, dim in fh_in.dimensions.items():
            if name != 'time':
                fh_out.createDimension(name, len(dim) if not dim.isunlimited() else None)

        ignore_features = ["time", "crs", "FparExtra_QC", "FparLai_QC"]
        mask_value_dic = {'Lai_500m': 10, 'LaiStdDev_500m': 10, 'Fpar_500m': 1, 'FparStdDev_500m': 1}
        for v_name, varin in fh_in.variables.items():
            if v_name not in ignore_features:
                dimensions = varin.dimensions if v_name in ['lat', 'lon'] else ('lat', 'lon')
                outVar = fh_out.createVariable(v_name, varin.datatype, dimensions)
                if v_name == "lat":
                    outVar.setncatts({"units": "degree_north"})
                    outVar[:] = varin[:]
                elif v_name == "lon":
                    outVar.setncatts({"units": "degree_east"})
                    outVar[:] = varin[:]
                else:
                    outVar.setncatts({k: varin.getncattr(k) for k in varin.ncattrs()})
                    vin = varin[index, :, :]
                    vin = ma.masked_greater(vin, mask_value_dic[v_name])
                    vin = ma.masked_less(vin, 0)
                    outVar[:] = vin[:]
        fh_out.close()
    fh_in.close()
def extract_ndvi(nc_file):
    fh_in = Dataset('../../raw_data/ndvi/' + nc_file, 'r')

    for index, n_days in enumerate(fh_in.variables['time'][:]):
        date = (datetime.datetime(2000, 1, 1, 0, 0) + datetime.timedelta(int(n_days))).strftime('%Y%m%d')
        print(date)
        fh_out = Dataset('../../processed_data/ndvi/1km/{}.nc'.format(date[:-2]), 'w')

        for name, dim in fh_in.dimensions.items():
            if name != 'time':
                fh_out.createDimension(name, len(dim) if not dim.isunlimited() else None)

        ignore_features = ["time", "crs", "_1_km_monthly_VI_Quality"]
        for v_name, varin in fh_in.variables.items():
            if v_name not in ignore_features:
                dimensions = varin.dimensions if v_name in ['lat', 'lon'] else ('lat', 'lon')
                v_name = v_name if v_name in ['lat', 'lon'] else v_name.split('_')[-1].lower()
                outVar = fh_out.createVariable(v_name, varin.datatype, dimensions)
                if v_name == "lat":
                    outVar.setncatts({"units": "degree_north"})
                    outVar[:] = varin[:]
                elif v_name == "lon":
                    outVar.setncatts({"units": "degree_east"})
                    outVar[:] = varin[:]
                else:
                    outVar.setncatts({k: varin.getncattr(k) for k in varin.ncattrs()})
                    vin = varin[index, :, :]
                    vin = ma.masked_greater(vin, 1.0)
                    vin = ma.masked_less(vin, -0.2)
                    outVar[:] = vin[:]
        fh_out.close()
    fh_in.close()
Exemplo n.º 9
0
def get_fence(self, xyfence, sampling="bilinear"):
    """Get surface values along fence."""

    cxarr = xyfence[:, 0]
    cyarr = xyfence[:, 1]
    czarr = xyfence[:, 2].copy()

    sampleoptions = {"bilinear": 0, "nearest": 2}

    # czarr will be updated "inplace":
    istat = _cxtgeo.surf_get_zv_from_xyv(
        cxarr,
        cyarr,
        czarr,
        self.ncol,
        self.nrow,
        self.xori,
        self.yori,
        self.xinc,
        self.yinc,
        self.yflip,
        self.rotation,
        self.get_values1d(),
        sampleoptions.get(sampling, 0),
    )

    if istat != 0:
        logger.warning("Seem to be rotten")

    xyfence[:, 2] = czarr
    xyfence = ma.masked_greater(xyfence, xtgeo.UNDEF_LIMIT)
    xyfence = ma.mask_rows(xyfence)

    return xyfence
Exemplo n.º 10
0
def _import_ijxyz(self, mfile):  # pylint: disable=too-many-locals
    """Import OW/DSG IJXYZ ascii format."""
    # import of seismic column system on the form:
    # 2588	1179	476782.2897888889	6564025.6954	1000.0
    # 2588	1180	476776.7181777778	6564014.5058	1000.0
    logger.debug("Read data from file... (scan for dimensions)")

    cfhandle = mfile.get_cfhandle()

    xlist = _cxtgeo.surf_import_ijxyz(cfhandle, 0, 1, 1, 1, 0)

    (
        ier,
        ncol,
        nrow,
        _,
        xori,
        yori,
        xinc,
        yinc,
        rot,
        iln,
        xln,
        val,
        yflip,
    ) = xlist

    if ier != 0:
        mfile.cfclose()
        raise RuntimeError("Import from C is wrong...")

    # now real read mode
    xlist = _cxtgeo.surf_import_ijxyz(cfhandle, 1, ncol, nrow, ncol * nrow, 0)

    ier, ncol, nrow, _, xori, yori, xinc, yinc, rot, iln, xln, val, yflip = xlist

    if ier != 0:
        raise RuntimeError("Import from C is wrong...")

    logger.info(xlist)

    val = ma.masked_greater(val, xtgeo.UNDEF_LIMIT)

    self._xori = xori
    self._xinc = xinc
    self._yori = yori
    self._yinc = yinc
    self._ncol = ncol
    self._nrow = nrow
    self._rotation = rot
    self._yflip = yflip

    self.values = val.reshape((self._ncol, self._nrow))

    self._ilines = iln
    self._xlines = xln

    mfile.cfclose()
    self._metadata.required = self
Exemplo n.º 11
0
def import_irap_binary(self, mfile, values=True):
    """Import Irap binary format."""

    ifile = xtgeo._XTGeoCFile(mfile)

    logger.debug("Enter function...")
    # read with mode 0, to get mx my and other metadata
    (
        ier,
        self._ncol,
        self._nrow,
        _ndef,
        self._xori,
        self._yori,
        self._xinc,
        self._yinc,
        self._rotation,
        val,
    ) = _cxtgeo.surf_import_irap_bin(ifile.fhandle, 0, 1, 0)

    if ier != 0:
        ifile.close()
        raise RuntimeError("Error in reading Irap binary file")

    self._yflip = 1
    if self._yinc < 0.0:
        self._yinc *= -1
        self._yflip = -1

    self._filesrc = mfile

    self._ilines = np.array(range(1, self._ncol + 1), dtype=np.int32)
    self._xlines = np.array(range(1, self._nrow + 1), dtype=np.int32)

    # lazy loading, not reading the arrays
    if not values:
        self._values = None
        ifile.close()
        return

    nval = self._ncol * self._nrow
    xlist = _cxtgeo.surf_import_irap_bin(ifile.fhandle, 1, nval, 0)
    if xlist[0] != 0:
        ifile.close()
        raise RuntimeError("Problem in {}, code {}".format(__name__, ier))

    val = xlist[-1]

    val = np.reshape(val, (self._ncol, self._nrow), order="C")

    val = ma.masked_greater(val, _cxtgeo.UNDEF_LIMIT)

    if np.isnan(val).any():
        logger.info("NaN values are found, will mask...")
        val = ma.masked_invalid(val)

    self._values = val

    ifile.close()
Exemplo n.º 12
0
 def load(fn):
     ff = Dataset(fn, "r")
     f = ff.variables
     ssh = f["Grid_0001"][:].T
     ssh = ma.masked_greater(ssh, 500)
     ssh = (ssh[:-1, :] + ssh[1:, :]) / 2.0
     ff.close()
     return ssh
Exemplo n.º 13
0
 def load(fn):
     ff = Dataset(fn, 'r')
     f = ff.variables
     ssh = f['Grid_0001'][:].T
     ssh = ma.masked_greater(ssh, 500)
     ssh = (ssh[:-1, :] + ssh[1:, :]) / 2.
     ff.close()
     return ssh
Exemplo n.º 14
0
def autocorrelation_length_estimate(series, acf=None, M=5, axis=0):
    r"""Returns an estimate of the autocorrelation length of the given
    series:

    .. math::

      L = \int_{-\infty}^\infty \rho(t) dt

    The estimate is the smallest :math:`L` such that 

    .. math::

      L = \rho(0) + 2 \sum_{j = 1}^{M L} \rho(j)

    In words: the ACL is estimated over a window that is at least
    :math:`M` ACLs long, with the constraint that :math:`ML < N/2`.

    Defined in this way, the ACL gives the reduction factor between
    the number of samples and the "effective" number of samples.  In
    particular, the variance of the estimated mean of the series is
    given by

    .. math::

      \left\langle \left( \frac{1}{N} \sum_{i=0}^{N-1} x_i - \mu
      \right)^2 \right\rangle = \frac{\left\langle \left(x_i -
      \mu\right)^2 \right\rangle}{N/L}

    Returns ``nan`` if there is no such estimate possible (because
    the series is too short to fit :math:`2M` ACLs).

    For an N-dimensional array, returns an array of ACLs of the same
    shape as ``series``, but with the dimension along ``axis``
    removed.

    """
    if acf is None:
        acf = autocorrelation_function(series, axis=axis)
    m = [slice(None)] * len(acf.shape)
    nmax = acf.shape[axis] / 2

    # Generate ACL candidates.
    m[axis] = slice(0, nmax)
    acl_ests = 2.0 * np.cumsum(acf[m], axis=axis) - 1.0

    # Build array of lags (like arange, but N-dimensional).
    shape = acf.shape[:axis] + (nmax, ) + acf.shape[axis + 1:]
    lags = np.cumsum(np.ones(shape), axis=axis) - 1.0

    # Mask out unwanted lags and set corresponding ACLs to nan.
    mask = M * acl_ests >= lags
    acl_ests[mask] = np.nan
    i = ma.masked_greater(mask, lags, copy=False)

    # Now get index of smallest unmasked lag -- if all are masked, this will be 0.
    j = i.argmin(axis=axis)
    k = tuple(np.indices(j.shape))
    return acl_ests[k[:axis] + (j, ) + k[axis:]]
Exemplo n.º 15
0
def plotOneMonth(dataset, dataVar, month, minval, maxval, cbarTicks = None, cmap = 'viridis'): 
    """Plots map of the arctic on North Pole Stereo projection with one month of data overlayed, along with the sea ice edge for each month.
   
    Args:
        dataset (xr Dataset): dataset from google bucket
        dataVar (str): variable of interest
        month (str): month and year of interest, i.e. 'Dec 2019' (does not need to be in any particular format)
        minval, maxval (int): minimum and maximum values for the data variable 
        cbarTicks (list or np array of length 2): ticks to use on colorbar (default to [minval + 1, maxval +1])
        cmap (str, optional): color map (default to viridis)
        
    Returns:
        Figure displayed in notebook 
    
    """
    
    #define projection and transform
    proj = ccrs.NorthPolarStereo(central_longitude = -45)
    transform = ccrs.PlateCarree()
    
    #initialize the figure and axes 
    plt.figure(figsize=(6, 6))
    ax = plt.axes(projection = proj)
    
    #define arguments if not inputted 
    cbarTicks = np.arange(minval, maxval + 1, 1) if cbarTicks is None else cbarTicks
    
    #plot sea ice concentraion 
    SICarray = dataset['seaice_conc_monthly_cdr'].sel(time = month).where(dataset['region_mask']!=21) #dont plot contour along coastlines
    
    #stackexchange workaround for plotting on a rotated grid
    lonGreater = ma.masked_greater(SICarray.longitude.values, -0.01)
    lonLesser = ma.masked_less(SICarray.longitude.values, 0)
    latGreater = ma.MaskedArray(SICarray.latitude.values, mask = lonGreater.mask)
    latLesser = ma.MaskedArray(SICarray.latitude.values, mask = lonLesser.mask)
    dataGreater = ma.MaskedArray(SICarray.values[0], mask = lonGreater.mask)
    dataLesser = ma.MaskedArray(SICarray.values[0], mask = lonLesser.mask)
    
    #plot contour using each part of the 2 masked data sets
    im2a = ax.contour(lonGreater, latGreater, dataGreater, levels = [0.5], transform = transform, colors = 'magenta', linewidths = 0.9, zorder=5, alpha=1)
    im2b = ax.contour(lonLesser, latLesser, dataLesser, levels = [0.5], transform = transform, colors = 'magenta', linewidths = 0.9, zorder=5, alpha=1)
    #im = ax.contour(SICarray.longitude.values, SICarray.latitude.values, SICarray.values[0], levels = [0.15], transform = transform, colors = 'magenta', linewidths = 0.8, zorder=15, alpha=1)
    
    #plot the data
    dataset[dataVar].where(dataset['seaice_conc_monthly_cdr'] > 0.5).sel(time = month).plot(x = 'longitude', y = 'latitude', vmin = minval, vmax = maxval, extend = 'both', 
                    ax = ax, add_colorbar = True, transform = transform, zorder = 2, cmap = cmap, 
                    cbar_kwargs = {'label': "\n".join(wrap(dataset[dataVar].attrs['long_name'] + ' (' + dataset[dataVar].attrs['units'] + ')', 50)), 'orientation': 'horizontal', 'shrink': 0.75, 'pad': 0.025})
    
    #add features to the map
    ax.coastlines(linewidth=0.15, color = 'black', zorder = 10) #add coastlines 
    ax.add_feature(cfeature.LAND, color ='0.95', zorder = 5) #add land 
    ax.add_feature(cfeature.LAKES, color = 'grey', zorder = 5) #add lakes 
    ax.gridlines(draw_labels = False, linewidth = 0.25, color = 'gray', alpha = 0.7, linestyle = '--', zorder = 6) #add gridlines
    ax.set_extent([-179, 179, 55, 90], crs = transform) #zoom in so map only displays the Arctic
    ax.set_title("\n".join(wrap(month + ": " + dataset[dataVar].attrs['long_name'], 38)), fontsize = 'x-large')
    
    #display figure in notebook 
    plt.show()
Exemplo n.º 16
0
def autocorrelation_length_estimate(series, acf=None, M=5, axis=0):
    r"""Returns an estimate of the autocorrelation length of the given
    series:

    .. math::

      L = \int_{-\infty}^\infty \rho(t) dt

    The estimate is the smallest :math:`L` such that

    .. math::

      L = \rho(0) + 2 \sum_{j = 1}^{M L} \rho(j)

    In words: the ACL is estimated over a window that is at least
    :math:`M` ACLs long, with the constraint that :math:`ML < N/2`.

    Defined in this way, the ACL gives the reduction factor between
    the number of samples and the "effective" number of samples.  In
    particular, the variance of the estimated mean of the series is
    given by

    .. math::

      \left\langle \left( \frac{1}{N} \sum_{i=0}^{N-1} x_i - \mu
      \right)^2 \right\rangle = \frac{\left\langle \left(x_i -
      \mu\right)^2 \right\rangle}{N/L}

    Returns ``nan`` if there is no such estimate possible (because
    the series is too short to fit :math:`2M` ACLs).

    For an N-dimensional array, returns an array of ACLs of the same
    shape as ``series``, but with the dimension along ``axis``
    removed.

    """
    if acf is None:
        acf = autocorrelation_function(series, axis=axis)
    m = [slice(None), ] * len(acf.shape)
    nmax = acf.shape[axis]//2

    # Generate ACL candidates.
    m[axis] = slice(0, nmax)
    acl_ests = 2.0*np.cumsum(acf[m], axis=axis) - 1.0

    # Build array of lags (like arange, but N-dimensional).
    shape = acf.shape[:axis] + (nmax,) + acf.shape[axis+1:]
    lags = np.cumsum(np.ones(shape), axis=axis) - 1.0

    # Mask out unwanted lags and set corresponding ACLs to nan.
    mask = M*acl_ests >= lags
    acl_ests[mask] = np.nan
    i = ma.masked_greater(mask, lags, copy=False)

    # Now get index of smallest unmasked lag -- if all are masked, this will be 0.
    j = i.argmin(axis=axis)
    k = tuple(np.indices(j.shape))
    return acl_ests[k[:axis] + (j,) + k[axis:]]
Exemplo n.º 17
0
def _convert_to_xtgeo_prop(self, rox, pname, roxgrid, roxprop, realisation,
                           faciescodes):  # pragma: no cover
    """Collect numpy array and convert to XTGeo fmt"""
    indexer = roxgrid.get_grid().grid_indexer
    self._ncol, self._nrow, self._nlay = indexer.dimensions

    if rox.version_required("1.3"):
        logger.info(indexer.ijk_handedness)
    else:
        logger.info(indexer.handedness)

    pvalues = roxprop.get_values(realisation=realisation)

    if str(roxprop.type) == "body_facies" and faciescodes:
        fmap = roxprop.get_facies_map(realisation=realisation)
        pvalues = fmap[pvalues]  # numpy magics

    self._roxar_dtype = pvalues.dtype

    if self._isdiscrete:
        mybuffer = np.ndarray(indexer.dimensions, dtype=np.int32)
        mybuffer.fill(xtgeo.UNDEF_INT)
    else:
        mybuffer = np.ndarray(indexer.dimensions, dtype=np.float64)
        mybuffer.fill(xtgeo.UNDEF)

    cellno = indexer.get_cell_numbers_in_range((0, 0, 0), indexer.dimensions)

    ijk = indexer.get_indices(cellno)

    iind = ijk[:, 0]
    jind = ijk[:, 1]
    kind = ijk[:, 2]

    mybuffer[iind, jind, kind] = pvalues[cellno]

    if self._isdiscrete:
        mybuffer = ma.masked_greater(mybuffer, xtgeo.UNDEF_INT_LIMIT)
        self.codes = _fix_codes(roxprop.code_names)
        logger.info("Fixed codes: %s", self.codes)
    else:
        mybuffer = ma.masked_greater(mybuffer, xtgeo.UNDEF_LIMIT)

    self._values = mybuffer
    self._name = pname
Exemplo n.º 18
0
    def create_anolamy_map(self, snowmap):
        '''displays snow cover anomaly data'''
        snowmap = ma.masked_greater(snowmap, 2)
        snowmap = snowmap.reshape(610, 450)
        snowmap = snowmap * 100

        fig_anom, ax = plt.subplots(figsize=(8, 10))
        cmap_anom = plt.cm.RdBu
        cmap_anom.set_bad('black', 0.8)
        ax.clear()

        ax.text(0.82,
                0.98,
                'Map: Nico Sun',
                fontsize=10,
                color='black',
                transform=ax.transAxes)

        ax.set_title(
            'NOAA / NSIDC IMS Snow & Ice Extent Anomaly      {}-{}-{}'.format(
                self.year, self.month, self.day),
            x=0.5)
        ax.set_xlabel('Data source: https://nsidc.org/data/g02156', x=0.22)
        ax.set_ylabel(
            'https://sites.google.com/site/cryospherecomputing/snow-cover',
            y=0.25)
        ax.text(1.02,
                0.22,
                'Snow cover anomaly in percent',
                transform=ax.transAxes,
                rotation='vertical',
                color='black',
                fontsize=9)
        axins1 = inset_axes(ax, width="5%", height="25%", loc=4)
        im1 = ax.imshow(snowmap,
                        interpolation='nearest',
                        vmin=-100,
                        vmax=100,
                        cmap=cmap_anom)

        plt.colorbar(im1,
                     cax=axins1,
                     orientation='vertical',
                     ticks=[-100, 0, +100])
        axins1.yaxis.set_ticks_position("left")

        ax.axes.get_yaxis().set_ticks([])
        ax.axes.get_xaxis().set_ticks([])
        plt.tight_layout(pad=1)

        fig_anom.savefig('X:/Upload/Snow_Cover_Data/NOAA_Snowmap_anomaly.png')
        fig_anom.savefig(
            'X:/SnowCover/Images/NOAA_Snowmap_anomaly_{}.png'.format(
                str(self.day_of_year).zfill(3)))

        #
        plt.pause(0.01)
Exemplo n.º 19
0
def get_mask_for_unphysical(U, cutoffU=2000., fill_value=np.nan):
    """
    Returns a mask for masking module. if absolute value of value is greater than cutoff, the value is masked.
    Parameters
    ----------
    U: array-like
    cutoffU: float
        if |value| > cutoff, this method considers those values unphysical.
    fill_value:


    Returns
    -------
    mask: multidimensional boolean array

    """
    print 'number of invalid values (nan and inf) in the array: ' + str(
        np.isnan(U).sum() + np.isinf(U).sum())
    print 'number of nan values in U: ' + str(np.isnan(U).sum())
    print 'number of inf values in U: ' + str(np.isinf(U).sum()) + '\n'

    # a=ma.masked_invalid(U)
    # print 'number of masked elements by masked_invalid: '+ str(ma.count_masked(a))

    # Replace all nan and inf values with fill_value.
    # fix_invalid still enforces a mask on elements with originally invalid values
    U_fixed = ma.fix_invalid(U, fill_value=99999)
    n_invalid = ma.count_masked(U_fixed)
    print 'number of masked elements by masked_invalid: ' + str(n_invalid)
    # Update the mask to False (no masking)
    U_fixed.mask = False

    # Mask unreasonable values of U_fixed
    b = ma.masked_greater(U_fixed, cutoffU)
    c = ma.masked_less(U_fixed, -cutoffU)
    n_greater = ma.count_masked(b) - n_invalid
    n_less = ma.count_masked(c)
    print 'number of masked elements greater than cutoff: ' + str(n_greater)
    print 'number of masked elements less than -cutoff: ' + str(n_less)

    # Generate a mask for all nonsense values in the array U
    mask = ~(~b.mask * ~c.mask)

    d = ma.array(U_fixed, mask=mask)
    n_total = ma.count_masked(d)
    # U_filled = ma.filled(d, fill_value)

    #Total number of elements in U
    N = 1
    for i in range(len(U.shape)):
        N *= U.shape[i]

    print 'total number of unphysical values: ' + str(
        ma.count_masked(d)) + '  (' + str((float(n_total) / N * 100)) + '%)\n'

    return mask
Exemplo n.º 20
0
    def concentrationshow(self, icemap, Areavalue, extentvalue, outlooktype):
        icemap = icemap / 250
        icemap = ma.masked_greater(icemap, 1)
        icemap = icemap.reshape(448, 304)
        icemap = icemap[60:410, 30:260]

        #		areavalue = int(areavalue*1e6)
        #		extentvalue = int(extentvalue*1e6)
        Areavalue = '{:,}'.format(Areavalue) + ' ' r'$km^2$'
        extentvalue = '{:,}'.format(extentvalue) + ' ' r'$km^2$'

        cmap = plt.cm.jet
        cmap.set_bad('black', 0.6)

        self.ax2.clear()
        self.ax2.set_title('{}_Forecast , Date: {}-{}-{}'.format(
            outlooktype, self.year, self.stringmonth, self.stringday))
        #self.ax.set_title('Average Forecast')
        self.ax2.set_xlabel('Area: {} / Extent: {}'.format(
            Areavalue, extentvalue),
                            fontsize=14)
        self.cax2 = self.ax2.imshow(icemap,
                                    interpolation='nearest',
                                    vmin=0,
                                    vmax=1,
                                    cmap=cmap)

        self.ax2.axes.get_yaxis().set_ticks([])
        self.ax2.axes.get_xaxis().set_ticks([])
        self.ax2.text(2,
                      8,
                      r'Data: NSIDC',
                      fontsize=10,
                      color='white',
                      fontweight='bold')
        self.ax2.text(2,
                      18,
                      r'Map: Nico Sun',
                      fontsize=10,
                      color='white',
                      fontweight='bold')
        self.ax2.text(
            -0.04,
            0.48,
            'https://sites.google.com/site/cryospherecomputing/forecast',
            transform=self.ax2.transAxes,
            rotation='vertical',
            color='grey',
            fontsize=10)
        self.fig2.tight_layout(pad=1)
        self.fig2.subplots_adjust(left=0.05)
        if self.loopday.month > 9:
            self.fig2.savefig(
                'X:/Forecast/August_test/SIPN_SIC_{}{}{}.png'.format(
                    self.year, self.stringmonth, self.stringday))
        plt.pause(0.01)
Exemplo n.º 21
0
def F(points):
    rsq = sd.pdist(points, 'sqeuclidean')
    rsq = ma.masked_greater(rsq, 2**(1/3)*sigma**2)
    q = 4*epsilon*(-12*sigma**12/rsq**7 + 6*sigma**6/rsq**4)
    xs = points[:,0]
    ys = points[:,1]
    sf = sd.squareform(q.filled(0))
    xout = np.subtract.outer(xs, xs)
    yout = np.subtract.outer(ys, ys)
    return np.array([np.sum(xout*sf, axis=1), np.sum(yout*sf, axis=1)]).T
Exemplo n.º 22
0
def get_mask_for_unphysical_using_cutoff(U, cutoff=None, mode='less'):
    if mode == 'less' or mode == 'l':
        U_masked = ma.masked_less(U, cutoff)
    elif mode == 'lesseqal' or mode == 'leq':
        U_masked = ma.masked_less_equal(U, cutoff)
    elif mode == 'greater' or mode == 'g':
        U_masked = ma.masked_greater(U, cutoff)
    elif mode == 'greaterequal' or mode == 'geq':
        U_masked = ma.masked_greater_equal(U, cutoff)
    return U_masked.mask
Exemplo n.º 23
0
def projectToLatLong(ifile, outfile, img_bnd_coords, proj4string):

    ####################################
    # Inputs
    # For details see http://fcm7/projects/SPS/browser/SPS/trunk/data/products/MSG_Products.nl
    ulx_ll, uly_ll, lrx_ll, lry_ll = img_bnd_coords  # Bounding coordinates in latlon, although the the image is in mercator projection!!!
    # epsg_code = 3395 # Mercator projection code of the imagery
    ####################################

    # For testing
    # ifile = '/data/users/hadhy/HyVic/Obs/OLR_noborders/EIAM50_201901210230.png'

    # Open the image file
    ds = gdal.Open(ifile)
    band = ds.GetRasterBand(1)
    arr = band.ReadAsArray()
    # Mask out pixel values with no data
    arr = ma.masked_less(arr, 4)
    arr = ma.masked_greater(arr, 254)
    # Apply equation to retrieve brightness temperatures. See Chamberlain et al. 2013 https://rmets.onlinelibrary.wiley.com/doi/full/10.1002/met.1403
    arrbt = (-0.44 * (arr - 4.)) + 308.
    # Fill masked values with -9999 (because the Geotiff format doesn't accept masked arrays)
    ma.set_fill_value(arrbt, -9999)
    arrbt = arrbt.filled()

    # Set some image info ...
    ## Gets the bounding coordinates in Mercator projection
    ulx, uly = latlon2projected(ulx_ll, uly_ll, proj4string)
    lrx, lry = latlon2projected(lrx_ll, lry_ll, proj4string)
    ## Calculate geotransform object
    nx, ny = [ds.RasterXSize, ds.RasterYSize]

    ## NB: The coordinates start in the top left corner (usually), so
    ## xDist needs to be positive, and yDist negative
    xDist = (lrx - ulx) / nx if lrx > ulx else -1 * (lrx - ulx) / nx
    yDist = -1 * (uly - lry) / ny if uly > lry else (uly - lry) / ny
    rtnX, rtnY = [0, 0]
    gt = [ulx, xDist, rtnX, uly, rtnY, yDist]
    ## Projection information
    srs = osr.SpatialReference()
    srs.ImportFromProj4(proj4string)
    # srs.ImportFromEPSG(epsg_code) # Documentation says it is mercator

    # Create dataset in mercator projection
    memfile = ''
    ds_merc = makeGDALds(arrbt, nx, ny, gt, srs, 'MEM', memfile)

    # Regrid to latlon
    gdal.Warp(outfile, ds_merc, dstSRS='EPSG:4326', dstNodata=-9999)

    # Convert geotiff to a cube and save
    timestamp = os.path.basename(outfile).split('.')[0].split('_')[1]
    cube = sf.geotiff2cube(outfile, timestamp)

    return cube
Exemplo n.º 24
0
def make_paperfig():
    fig=figure(figsize=(8,8))
    gs = gridspec.GridSpec(2, 1, height_ratios=[2.75, 1])

    ax=subplot(gs[0],projection=ccrs.NorthPolarStereo(central_longitude=0))
    infile=files[9]
    with open(infile, 'rb') as fr:
        hdr = fr.read(300)
        ice = np.fromfile(fr, dtype=np.uint8)
    ice = ice.reshape(448, 304)
    ice = ice / 250.
    ice = ma.masked_greater(ice, 1.0)
    # ax = plt.axes(projection=ccrs.NorthPolarStereo(central_longitude=0))
    cs = ax.coastlines(resolution='10m', linewidth=0.5)
    # ax.gridlines()
    ax.set_extent([-90, 10, 60, 90], crs=ccrs.PlateCarree())
    kw = dict(central_latitude=90, central_longitude=-45, true_scale_latitude=70)
    cs = ax.contourf(x, y, ice,arange(0,1.1,0.1),cmap=plt.cm.Blues,
                       transform=ccrs.Stereographic(**kw))
    ax.contour(x, y, ice,0.2,colors='k',transform=ccrs.Stereographic(**kw))
    lon_start=-43#4
    lon_end=-42#2
    lat_start=60
    lat_end=61
    x_start,y_start=transform(outProj,inProj,lon_start,lat_start)
    x_end,y_end=transform(outProj,inProj,lon_end,lat_end)
    # x_start,y_start=transform(outProj,inProj,lon_start,lat_start)
    # ax.plot([x_start,x_start,x_end,x_end,x_start],[y_start,y_end,y_end,y_start,y_start],linewidth=2,color='r',transform=ccrs.Stereographic(**kw))
    lon_FS1=-18
    lon_FS2=0
    lat_FS=78.5
    x_FS1,y_FS=transform(outProj,inProj,lon_FS1,lat_FS)
    x_FS2,y_FS=transform(outProj,inProj,lon_FS2,lat_FS)
    ax.plot([x_FS1,x_FS2],[y_FS,y_FS],linewidth=4,color='k',transform=ccrs.Stereographic(**kw))
    lon_DS1=-26.5
    lon_DS2=-24
    lat_DS1=68.5
    lat_DS2=67
    x_DS1,y_DS1=transform(outProj,inProj,lon_DS1,lat_DS1)
    x_DS2,y_DS2=transform(outProj,inProj,lon_DS2,lat_DS2)
    ax.plot([x_DS1,x_DS2],[y_DS1,y_DS2],linewidth=4,color='k',transform=ccrs.Stereographic(**kw))
    ax.plot([x_start,x_end],[y_start,y_start],linewidth=4,color='k',transform=ccrs.Stereographic(**kw))
    x_basin,y_basin=transform(outProj,inProj,en4['lon_basin'],en4['lat_basin'])
    ax.plot(x_basin,y_basin,linewidth=2,color='k',transform=ccrs.Stereographic(**kw))
    yr=infile[18:22]
    mnth=infile[22:24]
    # ax.set_title(mnth+'/'+yr,fontsize=14)
    # cbaxes = ax.add_axes([0.95, 0.25, 0.04, 0.45])
    cb=colorbar(cs)#,cax=cbaxes)
    cb.set_label('sea ice concentration\n 05/2015')

    ax1=subplot(gs[1])
    compseries(daily.date,cc['trans filt'],'Coastal current transport [Sv]','k',
                datevec,CFice,'Sea ice concentration','blue',ax1)
    savefig('../figures/paperfigs/seaice_4poster.pdf',bbox_inches='tight')
Exemplo n.º 25
0
def get_mask_for_unphysical_using_median(U, cutoffratio=0.4, mode='less'):
    median = np.median(U)
    if mode == 'less' or mode == 'l':
        U_masked = ma.masked_less(U, median * cutoffratio)
    elif mode == 'lesseqal' or mode == 'leq':
        U_masked = ma.masked_less_equal(U, median * cutoffratio)
    elif mode == 'greater' or mode == 'g':
        U_masked = ma.masked_greater(U, median * cutoffratio)
    elif mode == 'greaterequal' or mode == 'geq':
        U_masked = ma.masked_greater_equal(U, median * cutoffratio)
    return U_masked.mask
Exemplo n.º 26
0
def rsxs2nparray(rsns, rsxs):
    max_n = np.max(rsns) + 2  # See note above.
    rsesxs_cube = np.full((len(rsxs), len(rsns[0]), max_n), -100.0)

    for ri, r_esxs in enumerate(rsxs2rsesxs(rsns, rsxs)):
        for ei, e_xs in enumerate(r_esxs):
            rsesxs_cube[ri, ei] = np.pad(e_xs, (0, max_n - len(e_xs)),
                                         'constant',
                                         constant_values=100.0)

    return ma.masked_greater(rsesxs_cube, 99.0)
Exemplo n.º 27
0
def test_2Dmasked_array(N=25):
    l = N/2

    # Ones array
    grid = np.linspace(-10, 10, N)
    X, Y = np.meshgrid(grid, grid)
    data = random((N, N))
    thr = np.percentile(data, 70)
    data = ma.masked_greater(data, thr)
    h = wmean_2D(X, Y, data, l=l)
    assert h.mask.any()
Exemplo n.º 28
0
def makeAM(vertex, howManyOnes
           ):  #how many vertex there are in the graph,  % of connections
    adjacencyM = np.random.rand(vertex, vertex)  # make random matrix
    zeros = ma.masked_greater(
        adjacencyM, howManyOnes).mask  #find where elements are greater than X
    ones = ma.masked_less_equal(
        adjacencyM, howManyOnes).mask  #find where elements are less than X

    adjacencyM[zeros] = 0  # mask = 0
    adjacencyM[ones] = 1  # mask = 1

    return adjacencyM
Exemplo n.º 29
0
def plotHistogramContour(X, Y, Z, xlabel="",ylabel=""):
    # Number of bins in each axis in the histogram
    bins = 40

    # Range of x and y-axis, respectively
    r = [[np.min(X), np.max(X)], [np.min(Y), np.max(Y)]]
    print('Plotting contour... valid data-points: ', len(Z))
    print('[X-range, Y-range] : ', r)

    # Weights are temperatures. H is the sum of all temperature at point (X, Y)
    H, xedges, yedges = np.histogram2d(X, Y, bins=(bins, bins*5), range=r, weights=Z)
    # N is the number of data-points in the bins at position (X, Y)
    N, xedges, yedges = np.histogram2d(X, Y, bins=(bins, bins*5), range=r, weights=None)
    # Some sanity check and finally taking the average of temperatures
    H = ma.masked_less(H, 100)
    S = ma.divide(H, N, where=N>0)      # S is average temperature as each (X,Y)
    S = ma.masked_greater(S, 300)
    #print('H: ', H)
    #print('N: ', N)
    #print('S: ', S)
    #print("Shape of S: ", np.shape(S), " max: ", np.max(S), " min: ", np.min(S))

    # We had 'buckets' in the histogram for x and y-axis,
    # but we need a 'single value' for each bucket. We take their midpoint.
    ## Using list comprehension :)
    xpoints = [(xedges[i]+xedges[i+1])/2.0 for i in range(len(xedges)-1)]
    ypoints = [(yedges[i]+yedges[i+1])/2.0 for i in range(len(yedges)-1)]

    # Standard for plotting contour, create a meshgrid
    (P,Q) = np.meshgrid(xpoints, ypoints)
    plt.figure(figsize=(9,9))
    # The contour lines at the following temperature will be labeled
    V = (140, 160, 180, 200, 220, 240, 260, 280)
    # Plot a line contour
    contours = plt.contour(P, Q, S.T, V, colors='0.20', corner_mask=True)
    # We use transpose of S as S.T because that's what histogram2d() returns
    # Plot a filled contour
    plt.contourf(P, Q, S.T, 128, cmap=plt.cm.jet)
    plt.clabel(contours, inline=True, fontsize=8)
    plt.xlabel(xlabel)
    plt.ylabel(ylabel)
    plt.title("Temperature Contour")
    plt.autoscale()
    plt.colorbar()
    print('Done plotting.')

    # Save the contour plot as a high resolution EPS file
    global figcount
    if(figcount != -1):
        savefile = 'contour-%d.eps' %(figcount)
        plt.savefig(savefile, format='eps')
        figcount = figcount + 1
        print("File '%s' saved in the current working directory." %(savefile))
    def thicknessshow(self, icemap, Volumevalue, extentvalue, outlooktype):
        icemap = ma.masked_greater(icemap, 5)
        icemap = icemap.reshape(332, 316)

        #		areavalue = int(areavalue*1e6)
        #		extentvalue = int(extentvalue*1e6)
        Volumevalue = '{:,}'.format(Volumevalue) + ' ' r'$km^3$'
        extentvalue = '{:,}'.format(extentvalue) + ' ' r'$km^2$'

        cmap = plt.cm.jet
        cmap.set_bad('black', 0.6)

        self.ax.clear()
        self.ax.set_title('{}_Forecast , Date: {}-{}-{}'.format(
            outlooktype, self.year, self.stringmonth, self.stringday))
        #self.ax.set_title('Average Forecast')
        self.ax.set_xlabel('Volume: {} / Extent: {}'.format(
            Volumevalue, extentvalue),
                           fontsize=14)
        self.cax = self.ax.imshow(icemap,
                                  interpolation='nearest',
                                  vmin=0,
                                  vmax=3,
                                  cmap=cmap)

        self.ax.axes.get_yaxis().set_ticks([])
        self.ax.axes.get_xaxis().set_ticks([])
        self.ax.text(2,
                     8,
                     r'Data: NSIDC',
                     fontsize=10,
                     color='white',
                     fontweight='bold')
        self.ax.text(2,
                     18,
                     r'Map: Nico Sun',
                     fontsize=10,
                     color='white',
                     fontweight='bold')
        self.ax.text(
            -0.04,
            0.48,
            'https://sites.google.com/site/cryospherecomputing/forecast',
            transform=self.ax.transAxes,
            rotation='vertical',
            color='grey',
            fontsize=10)
        self.fig.tight_layout(pad=1)
        self.fig.subplots_adjust(left=0.05)
        #		if self.loopday.month > 9:
        #			self.fig.savefig('X:/Sea_Ice_Forecast/SIPN_south/SIPN_{}{}{}.png'.format(self.year,self.stringmonth,self.stringday))
        plt.pause(0.01)
Exemplo n.º 31
0
def _convert_to_xtgeo_prop(self, rox, pname, roxgrid, roxprop):

    indexer = roxgrid.get_grid().grid_indexer
    self._ncol, self._nrow, self._nlay = indexer.dimensions

    if rox.version_required("1.3"):
        logger.info(indexer.ijk_handedness)
    else:
        logger.info(indexer.handedness)

    pvalues = roxprop.get_values()
    self._roxar_dtype = pvalues.dtype

    if self._isdiscrete:
        mybuffer = np.ndarray(indexer.dimensions, dtype=np.int32)
        mybuffer.fill(xtgeo.UNDEF_INT)
    else:
        mybuffer = np.ndarray(indexer.dimensions, dtype=np.float64)
        mybuffer.fill(xtgeo.UNDEF)

    cellno = indexer.get_cell_numbers_in_range((0, 0, 0), indexer.dimensions)

    ijk = indexer.get_indices(cellno)

    iind = ijk[:, 0]
    jind = ijk[:, 1]
    kind = ijk[:, 2]

    mybuffer[iind, jind, kind] = pvalues[cellno]

    if self._isdiscrete:
        mybuffer = ma.masked_greater(mybuffer, xtgeo.UNDEF_INT_LIMIT)
        self.codes = _fix_codes(roxprop.code_names)
        logger.info("Fixed codes: %s", self.codes)
    else:
        mybuffer = ma.masked_greater(mybuffer, xtgeo.UNDEF_LIMIT)

    self._values = mybuffer
    self._name = pname
Exemplo n.º 32
0
    def importFromXYZ(self, b, H, RB, alpha):
        RT = RB - H * (np.tan(alpha))
        x, y, z = b[:, 0], b[:, 1], b[:, 2]
        r, tht, z = rec2cyl(x, y, z)
        self.setGeometry(RB, H, alpha)
        rPerf = self._getRperf(z)
        if self.imp_type == 'thick':
            imp = b[:, 3]
        else:
            imp = getGeomImperfection(r, z, rPerf)

        tm1 = ma.masked_less(tht, 0.1 * np.pi).mask
        tm2 = ma.masked_greater(tht, 1.9 * np.pi).mask

        tht = np.hstack((tht, np.pi * 2.0 + tht[tm1], 0.0 + (-1.0) * tht[tm2]))
        r = np.hstack((r, r[tm1], r[tm2]))
        z = np.hstack((z, z[tm1], z[tm2]))
        imp = np.hstack((imp, imp[tm1], imp[tm2]))

        ft = np.linspace(0, 2.0 * np.pi, self.samplingRadial)
        fz = np.linspace(0, H, self.samplingAxial)
        IMPERF = getImperfectionArray(tht, z, imp, ft, fz)

        mf = []
        for row in IMPERF[0:len(IMPERF)]:
            mf.append(np.isnan(np.sum(row)))
        row1 = mf.index(False)
        mr = []
        for i in reversed(mf):
            mr.append(i)
        row2 = len(mf) - 1 - mr.index(False)

        row1 += 1
        row2 -= 2
        dr1 = row1
        rows1 = range(0, row1)
        rows1sym = range(row1, row1 + dr1)[::-1]

        dr2 = len(mf) - 1 - row2
        rows2 = range(len(mf) - 1, row2, -1)
        rows2sym = range(row2 - dr2, row2)[::-1]

        IMPERF[rows1] = IMPERF[rows1sym].copy()
        IMPERF[rows2] = IMPERF[rows2sym].copy()

        #EXTRUDE
        #IMPERF[0:row1]=IMPERF[row1]
        #IMPERF[row2::]=IMPERF[row2]

        self.addData(IMPERF, ft, fz)
Exemplo n.º 33
0
    def importFromXYZ(self,b,H,RB,alpha):
        RT=RB-H *( np.tan(alpha ) )
        x,y,z=b[:,0],b[:,1],b[:,2]
        r,tht,z=rec2cyl(x,y,z)
        self.setGeometry(RB,H,alpha)
        rPerf=self._getRperf(z)
        if self.imp_type == 'thick':
            imp=b[:,3]
        else:
            imp=getGeomImperfection(r,z,rPerf)

        tm1=ma.masked_less(tht,0.1*np.pi).mask
        tm2=ma.masked_greater(tht,1.9*np.pi).mask

        tht=np.hstack((tht, np.pi*2.0+tht[tm1],  0.0+(-1.0)*tht[tm2]))
        r=np.hstack((r,r[tm1],r[tm2] ))
        z=np.hstack((z,z[tm1],z[tm2]))
        imp=np.hstack((imp,imp[tm1],imp[tm2]))

        ft=np.linspace(0,2.0*np.pi,self.samplingRadial)
        fz=np.linspace(0,H,self.samplingAxial)
        IMPERF=getImperfectionArray(tht,z,imp,ft,fz)

        mf=[]
        for row in IMPERF[0:len(IMPERF)]:
            mf.append( np.isnan(np.sum(row))  )
        row1=mf.index(False)
        mr=[]
        for i in reversed(mf):
            mr.append(i)
        row2= len(mf)-1-mr.index(False)

        row1+=1
        row2-=2
        dr1=row1
        rows1=range(0,row1)
        rows1sym=range(row1,row1+dr1)[::-1]

        dr2=len(mf)-1-row2
        rows2=range(len(mf)-1,row2,-1)
        rows2sym=range(row2-dr2,row2)[::-1]

        IMPERF[rows1]=IMPERF[rows1sym].copy()
        IMPERF[rows2]=IMPERF[rows2sym].copy()

        #EXTRUDE
        #IMPERF[0:row1]=IMPERF[row1]
        #IMPERF[row2::]=IMPERF[row2]

        self.addData(IMPERF,ft,fz)
Exemplo n.º 34
0
def modis_lai_extract(dates_folder, nc_file):
    out_path = get_out_path(os.path.join("Data", "MCD15A3H", "500m"))

    fh_in = Dataset(
        os.path.join("n5eil01u.ecs.nsidc.org", "MCD15A3H", dates_folder,
                     nc_file + ".nc"), 'r')

    for index, n_days in enumerate(fh_in.variables['time'][:]):
        date = (datetime.datetime(2000, 1, 1, 0, 0) +
                datetime.timedelta(int(n_days))).strftime('%Y%m%d')
        print(date)
        fh_out = Dataset(os.path.join(out_path, date + '.nc'), 'w')

        for name, dim in fh_in.dimensions.items():
            if name != 'time':
                fh_out.createDimension(
                    name,
                    len(dim) if not dim.isunlimited() else None)

        ignore_features = ["time", "crs", "FparExtra_QC", "FparLai_QC"]
        mask_value_dic = {
            'Lai_500m': 10,
            'LaiStdDev_500m': 10,
            'Fpar_500m': 1,
            'FparStdDev_500m': 1
        }
        for v_name, varin in fh_in.variables.items():
            if v_name not in ignore_features:
                dimensions = varin.dimensions if v_name in ['lat', 'lon'
                                                            ] else ('lat',
                                                                    'lon')
                outVar = fh_out.createVariable(v_name, varin.datatype,
                                               dimensions)
                if v_name == "lat":
                    outVar.setncatts({"units": "degree_north"})
                    outVar[:] = varin[:]
                elif v_name == "lon":
                    outVar.setncatts({"units": "degree_east"})
                    outVar[:] = varin[:]
                else:
                    outVar.setncatts(
                        {k: varin.getncattr(k)
                         for k in varin.ncattrs()})
                    vin = varin[index, :, :]
                    vin = ma.masked_greater(vin, mask_value_dic[v_name])
                    vin = ma.masked_less(vin, 0)
                    outVar[:] = vin[:]
        fh_out.close()
    fh_in.close()
def update(val):

    # Get parameter
    vmin = slider_cutoff.val
    borderType = borderTypes[int(np.around(
        slider_bordertype.val))]  # Border type
    d = int(np.around(slider_diameter.val))  # Diameter
    sigma = slider_sigma.val  # Sigma
    sigma_color = slider_sigma_color.val  # Sigma color (Bilateral filter)
    sigma_space = slider_sigma_space.val  # Sigma space (Bilateral filter)
    nlm_h = slider_nlm.val

    # Apply filters
    box = cv2.blur(img, (d, d), borderType=borderType)  # Box filter
    gaussian = cv2.GaussianBlur(img, (d, d), sigma,
                                borderType=borderType)  # Gaussian
    median = cv2.medianBlur(img, d)  # Median
    bilateral = cv2.bilateralFilter(img,
                                    d,
                                    sigma_color,
                                    sigma_space,
                                    borderType=borderType)  # Bilateral
    kernel_opening = cv2.getStructuringElement(cv2.MORPH_CROSS, (d, d))
    opening = cv2.morphologyEx(img, cv2.MORPH_OPEN, kernel_opening)
    masked = ma.masked_greater(opening, 0)
    opening2 = np.copy(img)
    opening2[~masked.mask] = 0
    non_local_means = cv2.fastNlMeansDenoising(img,
                                               None,
                                               h=nlm_h,
                                               templateWindowSize=d,
                                               searchWindowSize=d)

    # Show results
    original.set_clim(vmin, 255.0)
    box_filter.set_data(box)
    box_filter.set_clim(vmin, 255.0)
    gaussian_filter.set_data(gaussian)
    gaussian_filter.set_clim(vmin, 255.0)
    median_filter.set_data(median)
    median_filter.set_clim(vmin, 255.0)
    bilateral_filter.set_data(bilateral)
    bilateral_filter.set_clim(vmin, 255.0)
    opening_filter.set_data(opening)
    opening_filter.set_clim(vmin, 255.0)
    opening_filter2.set_data(opening2)
    opening_filter2.set_clim(vmin, 255.0)
    nlm_filter.set_data(non_local_means)
    nlm_filter.set_clim(vmin, 255.0)
Exemplo n.º 36
0
    def normalshow(self, icemap, icesum):
        '''displays sea ice data'''
        icemap = ma.masked_greater(icemap, 1)
        icemap = icemap.reshape(448, 304)
        icemap = icemap[60:410, 30:260]
        icesum = round(icesum, 3)
        icesum = '{0:.3f}'.format(icesum)

        cmap = plt.cm.jet
        cmap.set_bad('black', 0.6)

        self.ax.clear()
        self.ax.set_title('Date: {}-{}-{}'.format(self.year, self.stringmonth,
                                                  self.stringday))
        #self.ax.set_title('Minimum of Minima')
        self.ax.set_xlabel('Area: ' + str(icesum) + ' million km2',
                           fontsize=14)
        self.cax = self.ax.imshow(icemap,
                                  interpolation='nearest',
                                  vmin=0,
                                  vmax=1,
                                  cmap=cmap)

        self.ax.axes.get_yaxis().set_ticks([])
        self.ax.axes.get_xaxis().set_ticks([])
        self.ax.text(2,
                     8,
                     r'Data: NSIDC NRT',
                     fontsize=10,
                     color='white',
                     fontweight='bold')
        self.ax.text(2,
                     18,
                     r'Map: Nico Sun',
                     fontsize=10,
                     color='white',
                     fontweight='bold')
        self.ax.text(
            -0.04,
            0.48,
            'https://sites.google.com/site/cryospherecomputing/daily-data',
            transform=self.ax.transAxes,
            rotation='vertical',
            color='grey',
            fontsize=10)
        self.fig.tight_layout(pad=1)
        self.fig.subplots_adjust(left=0.05)
        self.fig.savefig('X:/Upload/Arctic_yesterday.png')
        plt.pause(0.01)
def processVolume(threshold):
    print("Generating Volume Data")
    data = dfr.getVolumeData()
    print("Applying threshold")
    masked = ma.masked_greater(data, threshold)
    print("Performing binary opening")
    newmask = binary_opening(masked.mask, selem=np.ones((9,9,9)))
    seed = np.copy(newmask)
    seed[1:-1, 1:-1, 1:-1] = newmask.max()
    print("Performing reconstruction")
    newmask = reconstruction(seed, newmask, method='erosion').astype(np.int)
    #newmask = binary_erosion(newmask, selem=np.ones((29,29,29))).astype(np.int)
    masked2 = ma.array(data, mask=np.logical_not(newmask))
    
    return masked, masked2
Exemplo n.º 38
0
def contourValuesAbove(lons, lats, rawdata, contourAboveValue):
    """
    Contours a grid for lats and lons passed in having values above the given value

    Args: 
            lons : a grid of the longitudes
            lats : a grid of the latitudes
            rawdata : a grid of values
            contourAboveValue : the value of which to contour anything above
            Note : the grids must be the same dimension and size
    Returns:
            A list of polygons  
    """
    masked_grid = mask.masked_greater(rawdata, contourAboveValue)
    return contour(lons, lats, masked_grid)
Exemplo n.º 39
0
def test_mask_at_interp():
    """ Test the behavior of masked points with interp on|off

        As long as the filter is wide enough to capture at least
          one data point per point, the interp=True will return
    """
    N = 25
    t = np.arange(N)
    y = random(N)
    yn = y[y.argsort()[-5]]
    y = ma.masked_greater(y, yn)
    # Equivalent to interp=False
    h = maud.wmean_1D(y, t=t, l=11)
    assert (y.mask == h.mask).all()
    h = maud.wmean_1D(y, t=t, l=5, interp=True)
    assert (~h.mask).all()
Exemplo n.º 40
0
def id_features(data, threshold):
    '''
    Find the locations where features exist inside a data array
    set entries 1=feature, 0=no feature
    Separating these out into individual features is application dependent
    Returns an array of ints={0,1} of data.shape
    '''
    #Filter data. notouch masks covers the sections we are not examining. touch is the sections we want
    data_notouch = ma.masked_less(data, threshold)
    data_touch = ma.masked_greater(data, threshold)
    #Extract the mask to get where there are features. We will use this to id features to operate on
    regions = ma.getmask(data_touch) #Extract the mask from the touch array as the Trues will line up with the areas more than the threshold
    #Create the features map of 1's where we want features (greater than threshold), zeroes otherwise
    features = numpy.zeros(data.shape, dtype=numpy.int32)
    features[regions] = 1 #Define features
    return features
Exemplo n.º 41
0
def interp():
    """ Test interp option
    """
    lon = np.arange(-1, 10.01, 0.1)
    lat = np.arange(-5, 1.01, 0.1)
    Lon, Lat = np.meshgrid(lon, lat)
    h = ma.masked_greater(np.random.random(Lon.shape), 0.7)
    h_smooth = wmean_2D_latlon(Lat, Lon, h, l=2e5, interp=False)
    h_csmooth = cwindow_mean_2D_latlon(Lat, Lon, h, l=2e5, interp=False)
    h_smooth_i = wmean_2D_latlon(Lat, Lon, h, l=2e5, interp=True)
    h_csmooth_i = cwindow_mean_2D_latlon(Lat, Lon, h, l=2e5, interp=True)
    assert (h_smooth == h_csmooth).all()
    assert (h_smooth_i == h_csmooth_i).all()
    #assert (abs(h_smooth - h_smooth_i).sum() == 0)
    assert ((h_smooth - h_smooth_i) == 0).all()
    assert (h_smooth_i.compressed().size >= h_smooth.compressed().size)
Exemplo n.º 42
0
def landocmask(icenc,lsmasknc):
	""" generate an icefree ocean mask, and spits out a land mask for good measure
	Input: nc file of ice concentration, land-sea mask (as strings)
	Output: ice free ocean mask (1s and 0s)
	To use: ocean = ma.masked_where(icexp<1,tempsfc)
	example: 
	"""
	lsmask=io.readnc(lsmasknc,'lsm')[0][0,0,:,:]
	#Create an icefree ocean mask
	ice=io.readnc(icenc,'iceconc')[0]
	icesum=ice.sum(axis=0)
	icelsm=icesum+lsmask
	ifmask = ma.masked_greater(icelsm,0)+1
	icefree = ifmask.data*~ifmask.mask
	sansice = icefree[0,:,:]
	#ocean = ma.masked_where(icexp<1,tempsfc)
	return lsmask, sansice
Exemplo n.º 43
0
	def _select_working_set(self):
		return (0,1)
		
		I = ma.masked_less( self.alpha, 1e-8 )
		grad = self._grad( ma.array(self.X, mask=ma.getmask(I) ) )
		
		i = (-grad).argmax()
		
		P = self._P( self.X[ i % self.N ].reshape([1,self.d]), np.vstack( [self.X,]*kappa ) )		# P_IJ
		a = ma.masked_less( P[0,i] + np.diagonal(P) - 2*P, 0 )
		
		#NOTE the actual equation is -grad < -grad_i - i'm not sure if inversing the signs and equality operator is a problem
		grad_i =  self._grad( self.X[i % self.N] )
		b = ma.masked_greater(grad, grad_i) - grad_i
		
		j = ( (b**2) / -a ).argmin()
		
		return (i,j)
Exemplo n.º 44
0
def test_mask_at_interp():
    """ Test the behavior of masked points with interp on|off

        As long as the filter is wide enough to capture at least
          one data point per point, the interp=True will return
    """
    N = 25
    l = N/2

    grid = np.linspace(-10, 10, N)
    X, Y = np.meshgrid(grid, grid)
    data = np.ones((N, N))
    thr = np.percentile(data, 90)
    data = ma.masked_greater(data, thr)
    # Equivalent to interp=False
    h = wmean_2D(X, Y, data, l=l)
    assert (data.mask == h.mask).all()
    h = wmean_2D(X, Y, data, l=l, interp=True)
    assert (~h.mask).all()
Exemplo n.º 45
0
def corr_proba(r, ndata, ndataset=2, dof=False):
    """Probability of rejecting correlations

    - **r**: Correlation coefficient
    - **ndata**: Number of records use for correlations
    - **ndataset**, optional:  Number of datasets (1 for autocorrelations, else 2) [default: 2]

    .. todo::

        This must be rewritten using :mod:`scipy.stats`
    """

    # Basic tests
    ndata = MA.masked_equal(ndata,0,copy=0)
    r = MV2.masked_where(MA.equal(MA.absolute(r),1.),r,copy=0)

    # Degree of freedom
    if dof:
        df = ndata
    else:
        df = ndata-2-ndataset

    # Advanced test: prevent extreme values by locally decreasing the dof
    reduc = N.ones(r.shape)
    z = None
    while z is None or MA.count(MA.masked_greater(z,-600.)):
        if z is not None:
            imax = MA.argmin(z.ravel())
            reduc.flat[imax] += 1
        dfr = df/reduc
        t = r*MV2.sqrt(dfr/((1.0-r)* (1.0+r)))
        a = 0.5*dfr
        b = 0.5
        x = df/(dfr+t**2)
        z = _gammaln(a+b)-_gammaln(a)-_gammaln(b)+a*MA.log(x)+b*MA.log(1.0-x)

    # Perfom the test and format the variable
    prob = MV2.masked_array(betai(a,b,x),axes=r.getAxisList())*100
    prob.id = 'corr_proba' ; prob.name = prob.id
    prob.long_name = 'Probability of rejection'
    prob.units = '%'

    return prob
Exemplo n.º 46
0
def hardcoded_maskedarray():
    """Test if masked data is not considered in the average
    """
    h = np.array([[ 1e9,  1e9,  1e9],
         [ 1e9,  3.14,  1e9],
         [ 1e9,  1e9,  1e9]])

    h = ma.masked_greater(h, 10)

    lon = np.array([10.1, 10, 9.9])
    lat = np.array([-0.1, -0.09, -0.08])
    Lon, Lat = np.meshgrid(lon, lat)

    h_smooth = wmean_2D_latlon(Lat, Lon, h, l=1e10)
    h_smooth2 = cwindow_mean_2D_latlon(Lat, Lon, h, l=1e10)
    # maud and cmaud should return the very same result
    assert (h_smooth == h_smooth2).all()
    assert (h_smooth.mask == h.mask).all()
    #assert (h_smooth.compressed() == h.compressed()).all()
    assert (np.absolute(h_smooth - h).sum() == 0.)
Exemplo n.º 47
0
Arquivo: abf.py Projeto: hjoliver/iris
    def _read(self):
        """Read the field from the given filename."""
        basename = os.path.basename(self._filename)
        self.version = int(basename[9:11])
        self.year = int(basename[12:16])
        self.month = basename[16:19]
        self.period = basename[19:20]
        self.format = basename[21:24]

        self.month = month_numbers[self.month]

        # Data is 8 bit bigendian.
        self.data = np.fromfile(self._filename, dtype='>u1').reshape(X_SIZE,
                                                                     Y_SIZE)
        # Iris' preferred dimensional ordering is (y,x).
        self.data = self.data.transpose()
        # Flip, for a positive step through the Y dimension.
        self.data = self.data[::-1]
        # Any percentages greater than 100 represent missing data.
        self.data = ma.masked_greater(self.data, 100)
Exemplo n.º 48
0
def compare2func(f1, f2):
    N = 10
    l = N/2
    grid = np.linspace(-10, 10, N)
    X, Y = np.meshgrid(grid, grid)
    data = random(X.shape)
    h1 = f1(X, Y, data, l=l)
    h2 = f2(X, Y, data, l=l)
    assert (h1 == h2).all()

    data = ma.array(data)
    h = f1(X, Y, data, l=l)
    h2 = f2(X, Y, data, l=l)
    assert (h1 == h2).all()

    thr = np.percentile(data, 70)
    data = ma.masked_greater(data, thr)
    h = f1(X, Y, data, l=l)
    h2 = f2(X, Y, data, l=l)
    assert (h1 == h2).all()
Exemplo n.º 49
0
    def _clean_timestamps(self, track, stime, etime):
        '''
        Cut track in time and make array with timestamps monotonic.
        @param track:
        @type track:
        @return:
        @rtype:
        '''
        data = ma.masked_inside(track['timestamp'], stime, etime)
        track = track.compress(data.mask)
        # Eliminate repetitive points.
        times, indices = np.unique(track['timestamp'], return_index=True)
        track = track[indices]
        # Here we still can has points reversed in time. Fix it.
        tdifs = np.ediff1d(track['timestamp'], to_begin=1)
        # At first let's find ends of track's chunks delimited by timeout,
        # if any.
        chunk_end_idxs = np.where(tdifs > self.maxtimediff)[0]
        # Index of timestamp from which no chunks allowed, just track.
        safe_time_idx = np.where(track['timestamp'] <
                                 stime + self.safe_time)[0]
        if len(chunk_end_idxs) > 0:
            track_start_idx = 0
            track_end_idx = chunk_end_idxs[0]
            # Situation then there are little tracks exists before window
            # open time.
            if len(safe_time_idx) > 0:
                safe_time_idx = safe_time_idx[-1]
                for chunk_end_idx in chunk_end_idxs:
                    if chunk_end_idx < safe_time_idx:
                        track_start_idx = chunk_end_idx + 1
                    else:
                        track_end_idx = chunk_end_idx
                        break
            track = track[track_start_idx:track_end_idx]
            tdifs = tdifs[track_start_idx:track_end_idx]

        # Eliminate reverse points.
        data = ma.masked_greater(tdifs, 0)
        track = track.compress(data.mask)
        return track
Exemplo n.º 50
0
	def dynamic_mask(self, image, sigrange):
		"""
		Creates a numpy mask on the image, filtering out any
		pixel values that are more than sigrange*std from the median value

		Input: numpy array of the image, sigrange for multiplier on standard dev range
		Output: Masked numpy array covering any pixels above or below the standard dev range
		"""

		# Make a masked array using the static mask and imput image
		pre_masked = ma.array(image, mask=self.static_mask)

		# Mask saturated or empty
		masked1 = ma.masked_greater(pre_masked, 254)
		masked1 = ma.masked_less(masked1, 0)

		median = ma.median(masked1)
		mean = ma.mean(masked1)
		std = ma.std(masked1)

		return masked1, median, mean, std
Exemplo n.º 51
0
def update(val):
    
    # Get parameter
    vmin = slider_cutoff.val
    borderType = borderTypes[int(np.around(slider_bordertype.val)) ] # Border type
    d = int(np.around(slider_diameter.val)) # Diameter
    sigma = slider_sigma.val # Sigma
    sigma_color = slider_sigma_color.val # Sigma color (Bilateral filter)
    sigma_space = slider_sigma_space.val # Sigma space (Bilateral filter)
    nlm_h = slider_nlm.val
    
    # Apply filters
    box = cv2.blur(img, (d,d), borderType=borderType) # Box filter
    gaussian = cv2.GaussianBlur(img, (d,d), sigma, borderType=borderType) # Gaussian
    median = cv2.medianBlur(img, d) # Median 
    bilateral = cv2.bilateralFilter(img, d, sigma_color, sigma_space, borderType=borderType) # Bilateral   
    kernel_opening = cv2.getStructuringElement(cv2.MORPH_CROSS,(d,d))
    opening = cv2.morphologyEx(img, cv2.MORPH_OPEN, kernel_opening) 
    masked = ma.masked_greater(opening, 0)
    opening2 = np.copy(img)
    opening2[~masked.mask] = 0
    non_local_means = cv2.fastNlMeansDenoising(img, None, h=nlm_h, templateWindowSize=d, searchWindowSize=d)
    
    # Show results
    original.set_clim(vmin, 255.0)
    box_filter.set_data(box)
    box_filter.set_clim(vmin, 255.0)
    gaussian_filter.set_data(gaussian)   
    gaussian_filter.set_clim(vmin, 255.0)
    median_filter.set_data(median)
    median_filter.set_clim(vmin, 255.0)
    bilateral_filter.set_data(bilateral)
    bilateral_filter.set_clim(vmin, 255.0)
    opening_filter.set_data(opening)
    opening_filter.set_clim(vmin, 255.0)
    opening_filter2.set_data(opening2)
    opening_filter2.set_clim(vmin, 255.0)
    nlm_filter.set_data(non_local_means)
    nlm_filter.set_clim(vmin, 255.0)
Exemplo n.º 52
0
def random_maskedarray(N=10, res=0.1):
    #lon0, lat0 = random(2)
    #lon0 = 540*lon0-180 #[180, 360]
    #lat0 = 180*lat0-90

    grid = np.arange(-N/2, N/2)*res
    Lon, Lat = np.meshgrid(grid, grid)

    h = random(Lon.shape)
    h = ma.masked_greater(h, 0.7)

    h_smooth = wmean_2D_latlon(Lat, Lon, h, l=.1)
    h_csmooth = cwindow_mean_2D_latlon(Lat, Lon, h, l=.1)
    assert (h_smooth == h_csmooth).all()

    h_smooth = wmean_2D_latlon(Lat, Lon, h, l=1e10)
    h_csmooth = cwindow_mean_2D_latlon(Lat, Lon, h, l=1e10)
    # maud and cmaud should return the very same result
    assert (h_smooth == h_csmooth).all()
    assert (h_smooth.mask == h.mask).all()
    #assert (h_smooth.compressed() == h.compressed()).all()
    assert (np.absolute(h_smooth - h).sum() == 0.)
Exemplo n.º 53
0
    def _read(self):
        """Read the field from the given filename."""
        basename = os.path.basename(self._filename)
        self.version = int(basename[9:11])
        self.year = int(basename[12:16])
        self.month = basename[16:19]
        self.period = basename[19:20]
        self.format = basename[21:24]

        self.month = month_numbers[self.month]

        # Data is 8 bit bigendian.
        data = np.fromfile(self._filename, dtype='>u1').reshape(X_SIZE, Y_SIZE)
        # Iris' preferred dimensional ordering is (y,x).
        data = data.transpose()
        # Flip, for a positive step through the Y dimension.
        data = data[::-1]
        # Any percentages greater than 100 represent missing data.
        data = ma.masked_greater(data, 100)
        # The default fill value is 999999(!), so we choose something
        # more sensible. NB. 999999 % 256 = 63 = bad.
        data.fill_value = 255
        self.data = data
Exemplo n.º 54
0
def dry_spells(rainfall, threshold=0.254, start=True):
    """
    Compute the dry spells for a series of precipitations

    Parameters
    ----------
    rainfall : TimeSeries
        TimeSeries of precipitations.
    threshold : float, optional
        Minimum amount of precipitation defining a wet day.
    start : boolean, optional
        Whether the spells are associated with the first or last day of the spell.
    

    Returns
    -------
    dry_spells : TimeSeries
        A :class:`TimeSeries` giving the duration of the spell at either the first 
        or last date.
    """
    rdates = getattr(rainfall, 'dates', None)
    rdata = ma.masked_array(rainfall, subok=False)
    condition = ma.masked_greater(rdata, threshold)
    slices = ma.clump_unmasked(condition)
    # Get the durations and starting dates of each spell
    durations = [s.stop - s.start for s in slices]
    # Give up if we have no dates
    if rdates is None:
        return np.array(durations)
    # Get the dates, then...
    if start:
        dates = rdates[[s.start for s in slices]]
    else:
        dates = rdates[[s.stop - 1 for s in slices]]
    ensoi = getattr(rainfall, 'ensoindicator', None)
    spells = enso.climate_series(durations, dates=dates, ensoindicator=ensoi)
    return spells
Exemplo n.º 55
0
    def test_testOddFeatures(self):
        # Test of other odd features
        x = arange(20)
        x = x.reshape(4, 5)
        x.flat[5] = 12
        assert_(x[1, 0] == 12)
        z = x + 10j * x
        assert_(eq(z.real, x))
        assert_(eq(z.imag, 10 * x))
        assert_(eq((z * conjugate(z)).real, 101 * x * x))
        z.imag[...] = 0.0

        x = arange(10)
        x[3] = masked
        assert_(str(x[3]) == str(masked))
        c = x >= 8
        assert_(count(where(c, masked, masked)) == 0)
        assert_(shape(where(c, masked, masked)) == c.shape)
        z = where(c, x, masked)
        assert_(z.dtype is x.dtype)
        assert_(z[3] is masked)
        assert_(z[4] is masked)
        assert_(z[7] is masked)
        assert_(z[8] is not masked)
        assert_(z[9] is not masked)
        assert_(eq(x, z))
        z = where(c, masked, x)
        assert_(z.dtype is x.dtype)
        assert_(z[3] is masked)
        assert_(z[4] is not masked)
        assert_(z[7] is not masked)
        assert_(z[8] is masked)
        assert_(z[9] is masked)
        z = masked_where(c, x)
        assert_(z.dtype is x.dtype)
        assert_(z[3] is masked)
        assert_(z[4] is not masked)
        assert_(z[7] is not masked)
        assert_(z[8] is masked)
        assert_(z[9] is masked)
        assert_(eq(x, z))
        x = array([1., 2., 3., 4., 5.])
        c = array([1, 1, 1, 0, 0])
        x[2] = masked
        z = where(c, x, -x)
        assert_(eq(z, [1., 2., 0., -4., -5]))
        c[0] = masked
        z = where(c, x, -x)
        assert_(eq(z, [1., 2., 0., -4., -5]))
        assert_(z[0] is masked)
        assert_(z[1] is not masked)
        assert_(z[2] is masked)
        assert_(eq(masked_where(greater(x, 2), x), masked_greater(x, 2)))
        assert_(eq(masked_where(greater_equal(x, 2), x),
                   masked_greater_equal(x, 2)))
        assert_(eq(masked_where(less(x, 2), x), masked_less(x, 2)))
        assert_(eq(masked_where(less_equal(x, 2), x), masked_less_equal(x, 2)))
        assert_(eq(masked_where(not_equal(x, 2), x), masked_not_equal(x, 2)))
        assert_(eq(masked_where(equal(x, 2), x), masked_equal(x, 2)))
        assert_(eq(masked_where(not_equal(x, 2), x), masked_not_equal(x, 2)))
        assert_(eq(masked_inside(list(range(5)), 1, 3), [0, 199, 199, 199, 4]))
        assert_(eq(masked_outside(list(range(5)), 1, 3), [199, 1, 2, 3, 199]))
        assert_(eq(masked_inside(array(list(range(5)),
                                       mask=[1, 0, 0, 0, 0]), 1, 3).mask,
                   [1, 1, 1, 1, 0]))
        assert_(eq(masked_outside(array(list(range(5)),
                                        mask=[0, 1, 0, 0, 0]), 1, 3).mask,
                   [1, 1, 0, 0, 1]))
        assert_(eq(masked_equal(array(list(range(5)),
                                      mask=[1, 0, 0, 0, 0]), 2).mask,
                   [1, 0, 1, 0, 0]))
        assert_(eq(masked_not_equal(array([2, 2, 1, 2, 1],
                                          mask=[1, 0, 0, 0, 0]), 2).mask,
                   [1, 0, 1, 0, 1]))
        assert_(eq(masked_where([1, 1, 0, 0, 0], [1, 2, 3, 4, 5]),
                   [99, 99, 3, 4, 5]))
        atest = ones((10, 10, 10), dtype=np.float32)
        btest = zeros(atest.shape, MaskType)
        ctest = masked_where(btest, atest)
        assert_(eq(atest, ctest))
        z = choose(c, (-x, x))
        assert_(eq(z, [1., 2., 0., -4., -5]))
        assert_(z[0] is masked)
        assert_(z[1] is not masked)
        assert_(z[2] is masked)
        x = arange(6)
        x[5] = masked
        y = arange(6) * 10
        y[2] = masked
        c = array([1, 1, 1, 0, 0, 0], mask=[1, 0, 0, 0, 0, 0])
        cm = c.filled(1)
        z = where(c, x, y)
        zm = where(cm, x, y)
        assert_(eq(z, zm))
        assert_(getmask(zm) is nomask)
        assert_(eq(zm, [0, 1, 2, 30, 40, 50]))
        z = where(c, masked, 1)
        assert_(eq(z, [99, 99, 99, 1, 1, 1]))
        z = where(c, 1, masked)
        assert_(eq(z, [99, 1, 1, 99, 99, 99]))
Exemplo n.º 56
0
        load_time = t1-t0
        print "Loaded file %s in %.3f" % ( cached_file, load_time )
        if load_time > max_load_time:
            max_load_time = load_time
            max_file = cached_file

    print "--- "*20
    print "MAX Load time: file %s in %.3f" % ( max_file, max_load_time )

if __name__ == "__main__":
#    from testing import getVariable
#    CacheLevel = 10000.0
#    TestVariable = getVariable( 0, CacheLevel )
#    data = TestVariable.data

    import os, numpy, numpy.ma as ma
    cached_file = os.path.expanduser( '~/.cdas/testing/MERRA_mon_atmos_hur' )
    data = numpy.fromfile( cached_file, dtype=numpy.float32 ).reshape( (432, 144, 288) )
    FillVal = 1.00000002e+20
    mdata = ma.masked_greater( data, 0.90000002e+20 )
    t0 = time.time()
    a = ma.average(data,0)
    t1 = time.time()
    print "Result computed in %.2f, shape = %s, sample=%s" % ( (t1-t0), str(a.shape), a.flatten()[0:10])






Exemplo n.º 57
0
    def get_spc(self, *args):
        nargs = len(args)
        if nargs > 2:
            raise TypeError('get_spc expected at most 2 arguments, got %d' % nargs)
            
        item = args[0]

        values = []
        roles = []
        if item.exclude:
            item_spc_roles = [k for k in item.iter_species_roles()]
            item_spcs = [k[0] for k in item_spc_roles]
            for spc, role in self._stoic:
                if role == 'u' and spc in item_spcs and (spc, 'u') not in item_spc_roles:
                    val = self._stoic[spc, role]
                    if greater(val, 0).all():
                        role = 'p'
                    elif less(val, 0).all():
                        role = 'r'
                
                if not (spc, role) in item_spc_roles:
                    values.append(1 * self._stoic[spc, role])
                    roles.append(role)
                    
                
        else:
            for spc, props in item.spc_dict.iteritems():
                spc_roles = props['role']
                for role in spc_roles:
                    if (spc, role) in self._stoic:
                        values.append(item.spc_dict[spc]['stoic'] * self._stoic[spc, role])
                        roles.append(role)
                
                if 'u' not in spc_roles and (spc, 'u') in self._stoic:
                    val = self._stoic[spc, 'u']
                    if role == 'p':
                        values.append(item.spc_dict[spc]['stoic'] * masked_less(val, 0).filled(0))
                        roles.append(role)
                    elif role == 'r':
                        values.append(item.spc_dict[spc]['stoic'] * masked_greater(val, 0).filled(0))
                        roles.append(role)
                    
                
                
        if len(values) == 0:
            if self._safe:
                if nargs == 1:
                    if item.name in self:
                        warn("%s not in %s; trying all roles" % (item, self.sum()))
                        return self.get_spc(Species(item.name, exclude = item.exclude))
                    raise KeyError, "%s does not contain %s" % (str(self.sum()), str(item))
                else:
                    return args[1]
            else:
                raise KeyError, "%s does not contain %s" % (str(self.sum()), str(item))
            
                
        last_role = roles[-1]
        same_role = all([last_role == role for role in roles])
        
        if same_role:
            role = last_role
        else:
            role = 'u'
        return Stoic(sum(values, axis = 0), role = role)
import numpy as np
import numpy.ma as ma

a = np.arange(4)
a
ma.masked_greater(a, 2)
## Constructing masked arrays
#
# Masked arrays are intended to be able to replace normal np.ndarrays,
# and so the same construction functions work:
x = ma.array([5, 2, 9, -4])
y = ma.array([5, 2., 9, -4.])
o = ma.ones(5)
z = ma.zeros((3, 2))
e = ma.empty(3)
a = ma.arange(6)
# These functions create masked arrays with all elements not masked
# initially. However, masked arrays can be constructed in other ways,
# which automatically set more interesting masks.
mx = ma.masked_equal([5, 2, 9, -4], 2)
my = ma.masked_greater([5, 2., 9, -4.], 3)
mis = ma.masked_inside(x, 1, 6)
miv = ma.masked_invalid([1, np.nan, 5, np.inf])
## And more:
# ma.greater_equal
# ma.masked_less
# ma.masked_less_equal
# ma.masked_not_equal
# ma.masked_object
# ma.masked_outside
# ma.masked_values
# ma.masked_where

# To get a masked array's unmasked data, and fill in masked elements, use
# 'filled':
print "mx, with masked elements filled with 77", mx.filled(77)
#CREATE WAVELENGTH MASK
#edges of masked regions adjusted by hand in TestMasks.ipynb to
#block out the geocoronal lines.

#w1, f1, err1 = np.loadtxt('lcb201010_x1dsum.txt.bin30.unred', unpack=True)
w1, f1, stdev1, err1 = np.loadtxt('../Natalie/lcb201010_x1dsum.txt.bin30.unred.newerrors', unpack=True)
add_disp = 9.1e-18
#adding intrinsic dispersion
toterr1 = np.sqrt(err1**2.0 + add_disp**2.0)
w1_1 = ma.masked_less(w1, 1141)
w1_2 = ma.masked_inside(w1_1, 1178., 1250.)
w1_3 = ma.masked_inside(w1_2, 1292., 1318.)
w1_4 = ma.masked_inside(w1_3, 1348., 1367.)
w1_5 = ma.masked_inside(w1_4, 1332., 1340.)
dataw1 = ma.masked_greater(w1_5, 1725.)
dataw1c = 1.0*dataw1.compressed()

#w2, f2, err2 = np.loadtxt('lcb202010_x1dsum.txt.bin30.unred', unpack=True)
w2, f2, stdev2, err2 = np.loadtxt('../Natalie/lcb202010_x1dsum.txt.bin30.unred.newerrors', unpack=True)
#adding intrinsic dispersion
toterr2 = np.sqrt(err2**2.0 + add_disp**2.0)
w2_1 = ma.masked_less(w2, 1142)
w2_2 = ma.masked_inside(w2_1, 1182., 1250.)
w2_3 = ma.masked_inside(w2_2, 1290., 1318.)
w2_4 = ma.masked_inside(w2_3, 1348., 1368.)
w2_5 = ma.masked_inside(w2_4, 1330., 1340.)
#dataw2 = ma.masked_greater(w2_5, 1533.)
dataw2 = ma.masked_greater(w2_5, 1600.)
dataw2c = 1.0*dataw2.compressed()