Ejemplo n.º 1
0
    def test_testCopySize(self):
        # Tests of some subtle points of copying and sizing.
        n = [0, 0, 1, 0, 0]
        m = make_mask(n)
        m2 = make_mask(m)
        assert_(m is m2)
        m3 = make_mask(m, copy=True)
        assert_(m is not m3)

        x1 = np.arange(5)
        y1 = array(x1, mask=m)
        assert_(y1._data is not x1)
        assert_(allequal(x1, y1._data))
        assert_(y1._mask is m)

        y1a = array(y1, copy=0)
        # For copy=False, one might expect that the array would just
        # passed on, i.e., that it would be "is" instead of "==".
        # See gh-4043 for discussion.
        assert_(y1a._mask.__array_interface__ ==
                y1._mask.__array_interface__)

        y2 = array(x1, mask=m3, copy=0)
        assert_(y2._mask is m3)
        assert_(y2[2] is masked)
        y2[2] = 9
        assert_(y2[2] is not masked)
        assert_(y2._mask is m3)
        assert_(allequal(y2.mask, 0))

        y2a = array(x1, mask=m, copy=1)
        assert_(y2a._mask is not m)
        assert_(y2a[2] is masked)
        y2a[2] = 9
        assert_(y2a[2] is not masked)
        assert_(y2a._mask is not m)
        assert_(allequal(y2a.mask, 0))

        y3 = array(x1 * 1.0, mask=m)
        assert_(filled(y3).dtype is (x1 * 1.0).dtype)

        x4 = arange(4)
        x4[2] = masked
        y4 = resize(x4, (8,))
        assert_(eq(concatenate([x4, x4]), y4))
        assert_(eq(getmask(y4), [0, 0, 1, 0, 0, 0, 1, 0]))
        y5 = repeat(x4, (2, 2, 2, 2), axis=0)
        assert_(eq(y5, [0, 0, 1, 1, 2, 2, 3, 3]))
        y6 = repeat(x4, 2, axis=0)
        assert_(eq(y5, y6))
Ejemplo n.º 2
0
    def test_testCopySize(self):
        # Tests of some subtle points of copying and sizing.
        n = [0, 0, 1, 0, 0]
        m = make_mask(n)
        m2 = make_mask(m)
        assert_(m is m2)
        m3 = make_mask(m, copy=1)
        assert_(m is not m3)

        x1 = np.arange(5)
        y1 = array(x1, mask=m)
        assert_(y1._data is not x1)
        assert_(allequal(x1, y1._data))
        assert_(y1._mask is m)

        y1a = array(y1, copy=0)
        # For copy=False, one might expect that the array would just
        # passed on, i.e., that it would be "is" instead of "==".
        # See gh-4043 for discussion.
        assert_(y1a._mask.__array_interface__ ==
                y1._mask.__array_interface__)

        y2 = array(x1, mask=m3, copy=0)
        assert_(y2._mask is m3)
        assert_(y2[2] is masked)
        y2[2] = 9
        assert_(y2[2] is not masked)
        assert_(y2._mask is m3)
        assert_(allequal(y2.mask, 0))

        y2a = array(x1, mask=m, copy=1)
        assert_(y2a._mask is not m)
        assert_(y2a[2] is masked)
        y2a[2] = 9
        assert_(y2a[2] is not masked)
        assert_(y2a._mask is not m)
        assert_(allequal(y2a.mask, 0))

        y3 = array(x1 * 1.0, mask=m)
        assert_(filled(y3).dtype is (x1 * 1.0).dtype)

        x4 = arange(4)
        x4[2] = masked
        y4 = resize(x4, (8,))
        assert_(eq(concatenate([x4, x4]), y4))
        assert_(eq(getmask(y4), [0, 0, 1, 0, 0, 0, 1, 0]))
        y5 = repeat(x4, (2, 2, 2, 2), axis=0)
        assert_(eq(y5, [0, 0, 1, 1, 2, 2, 3, 3]))
        y6 = repeat(x4, 2, axis=0)
        assert_(eq(y5, y6))
Ejemplo n.º 3
0
    def test_testCopySize(self):
        # Tests of some subtle points of copying and sizing.
        n = [0, 0, 1, 0, 0]
        m = make_mask(n)
        m2 = make_mask(m)
        assert_(m is m2)
        m3 = make_mask(m, copy=1)
        assert_(m is not m3)

        x1 = np.arange(5)
        y1 = array(x1, mask=m)
        assert_(y1._data is not x1)
        assert_(allequal(x1, y1._data))
        assert_(y1.mask is m)

        y1a = array(y1, copy=0)
        assert_(y1a.mask is y1.mask)

        y2 = array(x1, mask=m3, copy=0)
        assert_(y2.mask is m3)
        assert_(y2[2] is masked)
        y2[2] = 9
        assert_(y2[2] is not masked)
        assert_(y2.mask is m3)
        assert_(allequal(y2.mask, 0))

        y2a = array(x1, mask=m, copy=1)
        assert_(y2a.mask is not m)
        assert_(y2a[2] is masked)
        y2a[2] = 9
        assert_(y2a[2] is not masked)
        assert_(y2a.mask is not m)
        assert_(allequal(y2a.mask, 0))

        y3 = array(x1 * 1.0, mask=m)
        assert_(filled(y3).dtype is (x1 * 1.0).dtype)

        x4 = arange(4)
        x4[2] = masked
        y4 = resize(x4, (8,))
        assert_(eq(concatenate([x4, x4]), y4))
        assert_(eq(getmask(y4), [0, 0, 1, 0, 0, 0, 1, 0]))
        y5 = repeat(x4, (2, 2, 2, 2), axis=0)
        assert_(eq(y5, [0, 0, 1, 1, 2, 2, 3, 3]))
        y6 = repeat(x4, 2, axis=0)
        assert_(eq(y5, y6))
Ejemplo n.º 4
0
    def test_testCopySize(self):
        # Tests of some subtle points of copying and sizing.
        n = [0, 0, 1, 0, 0]
        m = make_mask(n)
        m2 = make_mask(m)
        assert_(m is m2)
        m3 = make_mask(m, copy=1)
        assert_(m is not m3)

        x1 = np.arange(5)
        y1 = array(x1, mask=m)
        assert_(y1._data is not x1)
        assert_(allequal(x1, y1._data))
        assert_(y1.mask is m)

        y1a = array(y1, copy=0)
        assert_(y1a.mask is y1.mask)

        y2 = array(x1, mask=m3, copy=0)
        assert_(y2.mask is m3)
        assert_(y2[2] is masked)
        y2[2] = 9
        assert_(y2[2] is not masked)
        assert_(y2.mask is m3)
        assert_(allequal(y2.mask, 0))

        y2a = array(x1, mask=m, copy=1)
        assert_(y2a.mask is not m)
        assert_(y2a[2] is masked)
        y2a[2] = 9
        assert_(y2a[2] is not masked)
        assert_(y2a.mask is not m)
        assert_(allequal(y2a.mask, 0))

        y3 = array(x1 * 1.0, mask=m)
        assert_(filled(y3).dtype is (x1 * 1.0).dtype)

        x4 = arange(4)
        x4[2] = masked
        y4 = resize(x4, (8, ))
        assert_(eq(concatenate([x4, x4]), y4))
        assert_(eq(getmask(y4), [0, 0, 1, 0, 0, 0, 1, 0]))
        y5 = repeat(x4, (2, 2, 2, 2), axis=0)
        assert_(eq(y5, [0, 0, 1, 1, 2, 2, 3, 3]))
        y6 = repeat(x4, 2, axis=0)
        assert_(eq(y5, y6))
Ejemplo n.º 5
0
    def test_testCopySize(self):
        # Tests of some subtle points of copying and sizing.
        with suppress_warnings() as sup:
            sup.filter(
                np.ma.core.MaskedArrayFutureWarning,
                "setting an item on a masked array which has a "
                "shared mask will not copy")

            n = [0, 0, 1, 0, 0]
            m = make_mask(n)
            m2 = make_mask(m)
            self.assertTrue(m is m2)
            m3 = make_mask(m, copy=1)
            self.assertTrue(m is not m3)

            x1 = np.arange(5)
            y1 = array(x1, mask=m)
            self.assertTrue(y1._data is not x1)
            self.assertTrue(allequal(x1, y1._data))
            self.assertTrue(y1.mask is m)

            y1a = array(y1, copy=0)
            self.assertTrue(y1a.mask is y1.mask)

            y2 = array(x1, mask=m, copy=0)
            self.assertTrue(y2.mask is m)
            self.assertTrue(y2[2] is masked)
            y2[2] = 9
            self.assertTrue(y2[2] is not masked)
            self.assertTrue(y2.mask is not m)
            self.assertTrue(allequal(y2.mask, 0))

            y3 = array(x1 * 1.0, mask=m)
            self.assertTrue(filled(y3).dtype is (x1 * 1.0).dtype)

            x4 = arange(4)
            x4[2] = masked
            y4 = resize(x4, (8,))
            self.assertTrue(eq(concatenate([x4, x4]), y4))
            self.assertTrue(eq(getmask(y4), [0, 0, 1, 0, 0, 0, 1, 0]))
            y5 = repeat(x4, (2, 2, 2, 2), axis=0)
            self.assertTrue(eq(y5, [0, 0, 1, 1, 2, 2, 3, 3]))
            y6 = repeat(x4, 2, axis=0)
            self.assertTrue(eq(y5, y6))
Ejemplo n.º 6
0
    def test_testCopySize(self):
        # Tests of some subtle points of copying and sizing.
        with suppress_warnings() as sup:
            sup.filter(
                np.ma.core.MaskedArrayFutureWarning,
                "setting an item on a masked array which has a "
                "shared mask will not copy")

            n = [0, 0, 1, 0, 0]
            m = make_mask(n)
            m2 = make_mask(m)
            self.assertTrue(m is m2)
            m3 = make_mask(m, copy=1)
            self.assertTrue(m is not m3)

            x1 = np.arange(5)
            y1 = array(x1, mask=m)
            self.assertTrue(y1._data is not x1)
            self.assertTrue(allequal(x1, y1._data))
            self.assertTrue(y1.mask is m)

            y1a = array(y1, copy=0)
            self.assertTrue(y1a.mask is y1.mask)

            y2 = array(x1, mask=m, copy=0)
            self.assertTrue(y2.mask is m)
            self.assertTrue(y2[2] is masked)
            y2[2] = 9
            self.assertTrue(y2[2] is not masked)
            self.assertTrue(y2.mask is not m)
            self.assertTrue(allequal(y2.mask, 0))

            y3 = array(x1 * 1.0, mask=m)
            self.assertTrue(filled(y3).dtype is (x1 * 1.0).dtype)

            x4 = arange(4)
            x4[2] = masked
            y4 = resize(x4, (8,))
            self.assertTrue(eq(concatenate([x4, x4]), y4))
            self.assertTrue(eq(getmask(y4), [0, 0, 1, 0, 0, 0, 1, 0]))
            y5 = repeat(x4, (2, 2, 2, 2), axis=0)
            self.assertTrue(eq(y5, [0, 0, 1, 1, 2, 2, 3, 3]))
            y6 = repeat(x4, 2, axis=0)
            self.assertTrue(eq(y5, y6))
Ejemplo n.º 7
0
 def _h_arrows(self, length):
     """ length is in arrow width units """
     # It might be possible to streamline the code
     # and speed it up a bit by using complex (x,y)
     # instead of separate arrays; but any gain would be slight.
     minsh = self.minshaft * self.headlength
     N = len(length)
     length = length.reshape(N, 1)
     # x, y: normal horizontal arrow
     x = np.array([0, -self.headaxislength, -self.headlength, 0],
                  np.float64)
     x = x + np.array([0, 1, 1, 1]) * length
     y = 0.5 * np.array([1, 1, self.headwidth, 0], np.float64)
     y = np.repeat(y[np.newaxis, :], N, axis=0)
     # x0, y0: arrow without shaft, for short vectors
     x0 = np.array(
         [0, minsh - self.headaxislength, minsh - self.headlength, minsh],
         np.float64)
     y0 = 0.5 * np.array([1, 1, self.headwidth, 0], np.float64)
     ii = [0, 1, 2, 3, 2, 1, 0]
     X = x.take(ii, 1)
     Y = y.take(ii, 1)
     Y[:, 3:] *= -1
     X0 = x0.take(ii)
     Y0 = y0.take(ii)
     Y0[3:] *= -1
     shrink = length / minsh
     X0 = shrink * X0[np.newaxis, :]
     Y0 = shrink * Y0[np.newaxis, :]
     short = np.repeat(length < minsh, 7, axis=1)
     #print 'short', length < minsh
     # Now select X0, Y0 if short, otherwise X, Y
     X = ma.where(short, X0, X)
     Y = ma.where(short, Y0, Y)
     if self.pivot[:3] == 'mid':
         X -= 0.5 * X[:, 3, np.newaxis]
     elif self.pivot[:3] == 'tip':
         X = X - X[:, 3, np.newaxis]  #numpy bug? using -= does not
         # work here unless we multiply
         # by a float first, as with 'mid'.
     tooshort = length < self.minlength
     if tooshort.any():
         # Use a heptagonal dot:
         th = np.arange(0, 7, 1, np.float64) * (np.pi / 3.0)
         x1 = np.cos(th) * self.minlength * 0.5
         y1 = np.sin(th) * self.minlength * 0.5
         X1 = np.repeat(x1[np.newaxis, :], N, axis=0)
         Y1 = np.repeat(y1[np.newaxis, :], N, axis=0)
         tooshort = ma.repeat(tooshort, 7, 1)
         X = ma.where(tooshort, X1, X)
         Y = ma.where(tooshort, Y1, Y)
     return X, Y
Ejemplo n.º 8
0
 def _h_arrows(self, length):
     """ length is in arrow width units """
     # It might be possible to streamline the code
     # and speed it up a bit by using complex (x,y)
     # instead of separate arrays; but any gain would be slight.
     minsh = self.minshaft * self.headlength
     N = len(length)
     length = length.reshape(N, 1)
     # x, y: normal horizontal arrow
     x = np.array([0, -self.headaxislength,
                     -self.headlength, 0], np.float64)
     x = x + np.array([0,1,1,1]) * length
     y = 0.5 * np.array([1, 1, self.headwidth, 0], np.float64)
     y = np.repeat(y[np.newaxis,:], N, axis=0)
     # x0, y0: arrow without shaft, for short vectors
     x0 = np.array([0, minsh-self.headaxislength,
                     minsh-self.headlength, minsh], np.float64)
     y0 = 0.5 * np.array([1, 1, self.headwidth, 0], np.float64)
     ii = [0,1,2,3,2,1,0]
     X = x.take(ii, 1)
     Y = y.take(ii, 1)
     Y[:, 3:] *= -1
     X0 = x0.take(ii)
     Y0 = y0.take(ii)
     Y0[3:] *= -1
     shrink = length/minsh
     X0 = shrink * X0[np.newaxis,:]
     Y0 = shrink * Y0[np.newaxis,:]
     short = np.repeat(length < minsh, 7, axis=1)
     #print 'short', length < minsh
     # Now select X0, Y0 if short, otherwise X, Y
     X = ma.where(short, X0, X)
     Y = ma.where(short, Y0, Y)
     if self.pivot[:3] == 'mid':
         X -= 0.5 * X[:,3, np.newaxis]
     elif self.pivot[:3] == 'tip':
         X = X - X[:,3, np.newaxis]   #numpy bug? using -= does not
                                      # work here unless we multiply
                                      # by a float first, as with 'mid'.
     tooshort = length < self.minlength
     if tooshort.any():
         # Use a heptagonal dot:
         th = np.arange(0,7,1, np.float64) * (np.pi/3.0)
         x1 = np.cos(th) * self.minlength * 0.5
         y1 = np.sin(th) * self.minlength * 0.5
         X1 = np.repeat(x1[np.newaxis, :], N, axis=0)
         Y1 = np.repeat(y1[np.newaxis, :], N, axis=0)
         tooshort = ma.repeat(tooshort, 7, 1)
         X = ma.where(tooshort, X1, X)
         Y = ma.where(tooshort, Y1, Y)
     return X, Y
Ejemplo n.º 9
0
def atmosphere_turb(n_atms,
                    lons_mg,
                    lats_mg,
                    water_mask=None,
                    Lc=2000,
                    difference=False,
                    verbose=False,
                    interpolate_threshold=1e4,
                    mean_m=0.02):
    """ A function to create synthetic turbulent atmospheres based on the 
    methods in Lohman Simons 2005.  Note that due to memory issues,
    largers ones are made by interpolateing smaller ones.  Can return atmsopheres
    for an individual acquisition, or as the difference of two (as per an 
    interferogram).  Units are in metres.  
    
    Inputs:
        n_atms | int | number of atmospheres to generate
        lons_mg | rank 2 array | longitudes of the bottom left corner of each pixel.  
        lats_mg | rank 2 array | latitudes of the bottom left corner of each pixel.  
        water_mask | rank 2 array | If supplied, this is applied to the atmospheres generated, convering them to masked arrays.  
        Lc     | float | length scale of correlation, in metres.  If smaller, noise is patchier, and if larger, smoother.  
        difference | boolean | If difference, two atmospheres are generated and subtracted from each other to make a single atmosphere.  
        verbose | boolean | Controls info printed to screen when running.  
        interpolate_threshold | int | if n_pixs is greater than this, images will be generated at size so that the total number of pixels doesn't exceed this.  
                                     e.g. if set to 1e4 (10000, the default) and images are 120*120, they will be generated at 100*100 then upsampled to 120*120.  
        mean_m | float | average max or min value of atmospheres that are created.  e.g. if 3 atmospheres have max values of 0.02m, 0.03m, and 0.04m, their mean would be 0.03cm.  
    
    Outputs:
        ph_turb | r3 array | n_atms x n_pixs x n_pixs, UNITS ARE M.  Note that if a water_mask is provided, this is applied and a masked array is returned.  
        
    2019/09/13 | MEG | adapted extensively from a simple script
    2020/10/02 | MEG | Change so that a water mask is optional.  
    2020/10/05 | MEG | Change so that meshgrids of the longitudes and latitudes of each pixel are used to set resolution. 
                       Also fix a bug in how Lc is handled, so this is now in meters.  
    2020/10/06 | MEG | Add support for rectangular atmospheres, fix some bugs.  
    """

    import numpy as np
    import numpy.ma as ma
    from scipy.spatial import distance as sp_distance  # geopy also has a distance function.  Rename for safety.
    from scipy import interpolate as scipy_interpolate
    from auxiliary_functions import lon_lat_to_ijk

    def generate_correlated_noise(pixel_distances, Lc, shape):
        """ given a matrix of pixel distances (in meters) and a length scale for the noise (also in meters),
        generate some 2d spatially correlated noise.  
        Inputs:
            pixel_distances | rank 2 array | pixels x pixels, distance between each on in metres.  
            Lc | float | Length scale over which the noise is correlated.  units are metres.  
            shape | tuple | (nx, ny)  NOTE X FIRST!
        Returns:
            y_2d | rank 2 array | spatially correlated noise.  
        History:
            2019/06/?? | MEG | Written
            2020/10/05 | MEG | Overhauled to be in metres and use scipy cholesky
            2020/10/06 | MEG | Add support for rectangular atmospheres.  
        """
        import scipy
        nx = shape[0]
        ny = shape[1]
        Cd = np.exp(
            (-1 * pixel_distances) / Lc
        )  # from the matrix of distances, convert to covariances using exponential equation
        #Cd_L = np.linalg.cholesky(Cd)                                             # ie Cd = CD_L @ CD_L.T
        Cd_L = scipy.linalg.cholesky(
            Cd, lower=True)  # better error messages than the numpy version.
        x = np.random.randn(
            (ny * nx))  # Parsons 2007 syntax - x for uncorrelated noise
        y = Cd_L @ x  # y for correlated noise
        y_2d = np.reshape(y, (ny, nx))  # turn back to rank 2
        return y_2d

    def rescale_atmosphere(atm, atm_mean=0.02, atm_sigma=0.005):
        """ a function to rescale a 2d atmosphere with any scale to a mean centered
        one with a min and max value drawn from a normal distribution.  
        Inputs:
            atm | rank 2 array | a single atmosphere.  
            atm_mean | float | average max or min value of atmospheres that are created, in metres.  e.g. if 3 atmospheres have max values of 0.02m, 0.03m, and 0.04m, their mean would be 0.03m
            atm_sigma | float | standard deviation of Gaussian distribution used to generate atmosphere strengths.  
        Returns:
            atm | rank 2 array | a single atmosphere, rescaled to have a maximum signal of around that set by mean_m
        History:
            20YY/MM/DD | MEG | Written
            2020/10/02 | MEG | Standardise throughout to use metres for units.  
        """
        atm -= np.mean(atm)  # mean centre
        atm_strength = (
            atm_sigma * np.random.randn(1)
        ) + atm_mean  # maximum strength of signal is drawn from a gaussian distribution, mean and sigma set in metres.
        if np.abs(np.min(atm)) > np.abs(
                np.max(atm)):  # if range of negative numbers is larger
            atm *= (
                atm_strength / np.abs(np.min(atm))
            )  # strength is drawn from a normal distribution  with a mean set by mean_m (e.g. 0.02)
        else:
            atm *= (
                atm_strength / np.max(atm)
            )  # but if positive part is larger, rescale in the same way as above.
        return atm

    #1: determine if linear interpolation is required
    ny, nx = lons_mg.shape
    n_pixs = nx * ny

    if n_pixs > interpolate_threshold:
        if verbose:
            print(
                f"The number of pixels is larger than 'interpolate_threshold' ({interpolate_threshold}) so images will be created "
                f"with {interpolate_threshold} pixels and interpolated to the full resolution.  "
            )
        interpolate = True  # set boolean flag
        oversize_factor = n_pixs / interpolate_threshold  # determine how many times too many pixels we have.
        lons_ds = np.linspace(
            lons_mg[-1, 0], lons_mg[-1, -1],
            int(nx * (1 / np.sqrt(oversize_factor)))
        )  # make a downsampled vector of just the longitudes (square root as number of pixels is a measure of area, and this is length)
        lats_ds = np.linspace(
            lats_mg[0, 0], lats_mg[-1, 0],
            int(ny * (1 / np.sqrt(oversize_factor))))  # and for latitudes
        lons_mg_ds = np.repeat(lons_ds[np.newaxis, :], lats_ds.shape,
                               axis=0)  # make rank 2 again
        lats_mg_ds = np.repeat(lats_ds[:, np.newaxis], lons_ds.shape,
                               axis=1)  # and for latitudes
        ny_generate, nx_generate = lons_mg_ds.shape  # get the size of the downsampled grid we'll be generating at
    else:
        interpolate = False  # set boolean flag
        nx_generate = nx  # if not interpolating, these don't change.
        ny_generate = ny
        lons_mg_ds = lons_mg  # if not interpolating, don't need to downsample.
        lats_mg_ds = lats_mg

    #2: calculate distance between points
    ph_turb = np.zeros(
        (n_atms, ny_generate,
         nx_generate))  # initiate output as a rank 3 (ie n_images x ny x nx)
    xyz_m, pixel_spacing = lon_lat_to_ijk(
        lons_mg_ds, lats_mg_ds
    )  # get pixel positions in metres from origin in lower left corner (and also their size in x and y direction)
    xy = xyz_m[
        0:
        2].T  # just get the x and y positions (ie discard z), and make lots x 2 (ie two columns)
    pixel_distances = sp_distance.cdist(
        xy, xy, 'euclidean'
    )  # calcaulte all pixelwise pairs - slow as (pixels x pixels)

    #3: generate atmospheres
    if difference is False:  # this just generates a single turbulent atmosphere
        for i in range(n_atms):
            ph_turb[i, :, :] = generate_correlated_noise(
                pixel_distances, Lc,
                (nx_generate, ny_generate))  # generate noise
            if verbose:
                print(
                    f'Generated {i} of {n_atms} single acquisition atmospheres.  '
                )
    elif difference is True:  # but we can also generate the difference between two atmospheres, as you'd see in an interferogram.
        for i in range(n_atms):
            y_2d_1 = generate_correlated_noise(
                pixel_distances, Lc,
                (nx_generate, ny_generate))  # generate first noise
            y_2d_2 = generate_correlated_noise(
                pixel_distances, Lc,
                (nx_generate, ny_generate))  # generate second noise
            ph_turb[
                i, :, :] = y_2d_1 - y_2d_2  # difference between the two atmospheres
            if verbose:
                print(
                    f'Generated {i} of {n_atms} interferogram atmospheres.  ')
    else:
        raise Exception(
            "'difference' must be either True or False.  Quitting.  ")

    #3: possibly interplate to bigger size
    if interpolate:
        if verbose:
            print('Interpolating to the larger size...', end='')
        ph_turb_output = np.zeros(
            (n_atms, ny, nx)
        )  # initiate output at the upscaled size (ie the same as the original lons_mg shape)
        for atm_n, atm in enumerate(
                ph_turb
        ):  # loop through the 1st dimension of the rank 3 atmospheres.
            f = scipy_interpolate.interp2d(
                np.arange(0, nx_generate),
                np.arange(0, ny_generate),
                atm,
                kind='linear'
            )  # and interpolate them to a larger size.  First we give it  meshgrids and values for each point
            ph_turb_output[atm_n, :, :] = f(
                np.linspace(0, nx_generate,
                            nx), np.linspace(0, ny_generate, ny)
            )  # then new meshgrids at the original (full) resolution.
        if verbose:
            print('Done!')
    else:
        ph_turb_output = ph_turb  # if we're not interpolating, no change needed

    # 4: rescale to correct range (i.e. a couple of cm)
    ph_turb_m = np.zeros(ph_turb_output.shape)
    for atm_n, atm in enumerate(ph_turb_output):
        ph_turb_m[atm_n, ] = rescale_atmosphere(atm, mean_m)

    # 5: return back to the shape given, which can be a rectangle:
    ph_turb_m = ph_turb_m[:, :lons_mg.shape[0], :lons_mg.shape[1]]

    if water_mask is not None:
        water_mask_r3 = ma.repeat(water_mask[np.newaxis, ],
                                  ph_turb_m.shape[0],
                                  axis=0)
        ph_turb_m = ma.array(ph_turb_m, mask=water_mask_r3)

    return ph_turb_m
def augment_data(X, Y_class, Y_loc, n_data=500):
    """ A function to augment data and presserve the location label for any deformation.  
    Note that n_data is not particularly intelligent as many more data may be generated,
    and only n_data returned, so even if n_data is low, the function can still be slow.  
    Inputs:
        X           | rank 4 array | data.  
        Y_class     | rank 2 array | One hot encoding of class labels
        Y_loc       | rank 2 array | locations of deformation
        n_data      | int |
    Returns:
        X_aug           | rank 4 array | data.  
        Y_class_aug     | rank 2 array | One hot encoding of class labels
        Y_loc_aug       | rank 2 array | locations of deformation
    History:
        2019/??/?? | MEG | Written
        2020/10/29 | MEG | Write the docs.  
        2020_01_11 | MEG | Major rewrite to speed things up.  
    """
    import numpy as np
    import numpy.ma as ma

    flips = ['none', 'up_down', 'left_right',
             'both']  # the three possible types of flip

    # 0: get the correct nunber of data
    n_ifgs = X.shape[0]
    data_dict = {
        'X': X,  # package the data and labels together into a dict
        'Y_class': Y_class,
        'Y_loc': Y_loc
    }

    if n_ifgs < n_data:  # if we have fewer ifgs than we need, repeat them
        n_repeat = int(
            np.ceil(n_data / n_ifgs)
        )  # get the number of repeats needed (round up and make an int)
        data_dict['X'] = ma.repeat(data_dict['X'], axis=0, repeats=n_repeat)
        data_dict['Y_class'] = np.repeat(data_dict['Y_class'],
                                         axis=0,
                                         repeats=n_repeat)
        data_dict['Y_loc'] = np.repeat(data_dict['Y_loc'],
                                       axis=0,
                                       repeats=n_repeat)

    data_dict = shuffle_arrays(
        data_dict
    )  # shuffle (so that these aren't in the order of the class labels)
    for key in data_dict:  # then crop them to the correct number
        data_dict[key] = data_dict[key][:n_data, ]

    X_aug = data_dict[
        'X']  # and unpack as this function doesn't use dictionaries
    Y_class_aug = data_dict['Y_class']
    Y_loc_aug = data_dict['Y_loc']

    # 1: do the flips
    for data_n in range(n_data):
        flip = flips[np.random.randint(0,
                                       len(flips))]  # choose a flip at random
        if flip != 'none':
            X_aug[data_n:data_n +
                  1, ], Y_loc_aug[data_n:data_n + 1, ] = augment_flip(
                      X_aug[data_n:data_n + 1, ],
                      Y_loc_aug[data_n:data_n + 1, ],
                      flip)  # do the augmentaiton via one of the flips.

    # 2: do the rotations
    X_aug, Y_loc_aug = augment_rotate(X_aug, Y_loc_aug)  # rotate

    # 3: Do the translations
    X_aug, Y_loc_aug = augment_translate(X_aug,
                                         Y_loc_aug,
                                         max_translate=(20, 20))

    return X_aug, Y_class_aug, Y_loc_aug  # return, and select only the desired data.
Ejemplo n.º 11
0
def atmosphere_turb(n_atms,
                    lons_mg,
                    lats_mg,
                    method='fft',
                    mean_m=0.02,
                    water_mask=None,
                    difference=False,
                    verbose=False,
                    cov_interpolate_threshold=1e4,
                    cov_Lc=2000):
    """ A function to create synthetic turbulent atmospheres based on the  methods in Lohman Simons 2005, or using Andy Hooper and Lin Shen's fft method.  
    Note that due to memory issues, when using the covariance (Lohman) method, largers ones are made by interpolateing smaller ones.  
    Can return atmsopheres for an individual acquisition, or as the difference of two (as per an interferogram).  Units are in metres.  
    
    Inputs:
        n_atms | int | number of atmospheres to generate
        lons_mg | rank 2 array | longitudes of the bottom left corner of each pixel.  
        lats_mg | rank 2 array | latitudes of the bottom left corner of each pixel.  
        method | string | 'fft' or 'cov'.  Cov for the Lohmans Simons (sp?) method, fft for Andy Hooper/Lin Shen's fft method (which is much faster).  Currently no way to set length scale using fft method.  
        mean_m | float | average max or min value of atmospheres that are created.  e.g. if 3 atmospheres have max values of 0.02m, 0.03m, and 0.04m, their mean would be 0.03cm.  
        water_mask | rank 2 array | If supplied, this is applied to the atmospheres generated, convering them to masked arrays.  
        difference | boolean | If difference, two atmospheres are generated and subtracted from each other to make a single atmosphere.  
        verbose | boolean | Controls info printed to screen when running.  
        cov_Lc     | float | length scale of correlation, in metres.  If smaller, noise is patchier, and if larger, smoother.  
        cov_interpolate_threshold | int | if n_pixs is greater than this, images will be generated at size so that the total number of pixels doesn't exceed this.  
                                          e.g. if set to 1e4 (10000, the default) and images are 120*120, they will be generated at 100*100 then upsampled to 120*120.  
        
    
    Outputs:
        ph_turb | r3 array | n_atms x n_pixs x n_pixs, UNITS ARE M.  Note that if a water_mask is provided, this is applied and a masked array is returned.  
        
    2019/09/13 | MEG | adapted extensively from a simple script
    2020/10/02 | MEG | Change so that a water mask is optional.  
    2020/10/05 | MEG | Change so that meshgrids of the longitudes and latitudes of each pixel are used to set resolution. 
                       Also fix a bug in how cov_Lc is handled, so this is now in meters.  
    2020/10/06 | MEG | Add support for rectangular atmospheres, fix some bugs.  
    2020_03_01 | MEG | Add option to use Lin Shen/Andy Hooper's fft method which is quicker than the covariance method.  
    """

    import numpy as np
    import numpy.ma as ma
    from scipy.spatial import distance as sp_distance  # geopy also has a distance function.  Rename for safety.
    from scipy import interpolate as scipy_interpolate
    from auxiliary_functions import lon_lat_to_ijk

    def generate_correlated_noise_cov(pixel_distances, cov_Lc, shape):
        """ given a matrix of pixel distances (in meters) and a length scale for the noise (also in meters),
        generate some 2d spatially correlated noise.  
        Inputs:
            pixel_distances | rank 2 array | pixels x pixels, distance between each on in metres.  
            cov_Lc | float | Length scale over which the noise is correlated.  units are metres.  
            shape | tuple | (nx, ny)  NOTE X FIRST!
        Returns:
            y_2d | rank 2 array | spatially correlated noise.  
        History:
            2019/06/?? | MEG | Written
            2020/10/05 | MEG | Overhauled to be in metres and use scipy cholesky
            2020/10/06 | MEG | Add support for rectangular atmospheres.  
        """
        import scipy
        nx = shape[0]
        ny = shape[1]
        Cd = np.exp(
            (-1 * pixel_distances) / cov_Lc
        )  # from the matrix of distances, convert to covariances using exponential equation
        #Cd_L = np.linalg.cholesky(Cd)                                             # ie Cd = CD_L @ CD_L.T
        Cd_L = scipy.linalg.cholesky(
            Cd, lower=True)  # better error messages than the numpy version.
        x = np.random.randn(
            (ny * nx))  # Parsons 2007 syntax - x for uncorrelated noise
        y = Cd_L @ x  # y for correlated noise
        y_2d = np.reshape(y, (ny, nx))  # turn back to rank 2
        return y_2d

    def generate_correlated_noise_fft(nx, ny, std_long, sp):
        """ A function to create synthetic turbulent troposphere delay using an FFT approach. 
        The power of the turbulence is tuned by the weather model at the longer wavelengths.
        
        Inputs:
            nx (int) -- width of troposphere 
            Ny (int) -- length of troposphere 
            std_long (float) -- standard deviation of the weather model at the longer wavelengths. Default = ?
            sp | int | pixel spacing in km
            
        Outputs:
            APS (float): 2D array, Ny * nx, units are m.
            
        History:
            2020_??_?? | LS | Adapted from code by Andy Hooper.  
            2021_03_01 | MEG | Small change to docs and inputs to work with SyInterferoPy
        """

        import numpy as np
        import numpy.matlib as npm
        import math

        np.seterr(divide='ignore')

        cut_off_freq = 1 / 50  # drop wavelengths above 50 km

        x = np.arange(0, int(nx / 2))  # positive frequencies only
        y = np.arange(0, int(ny / 2))  # positive frequencies only
        freq_x = np.divide(x, nx * sp)
        freq_y = np.divide(y, ny * sp)
        Y, X = npm.meshgrid(freq_x, freq_y)
        freq = np.sqrt((X * X + Y * Y) / 2)  # 2D positive frequencies

        log_power = np.log10(freq) * -11 / 3  # -11/3 in 2D gives -8/3 in 1D
        ix = np.where(freq < 2 / 3)
        log_power[ix] = np.log10(freq[ix]) * -8 / 3 - math.log10(
            2 / 3)  # change slope at 1.5 km (2/3 cycles per km)

        bin_power = np.power(10, log_power)
        ix = np.where(freq < cut_off_freq)
        bin_power[ix] = 0

        APS_power = np.zeros(
            (ny, nx))  # mirror positive frequencies into other quadrants
        APS_power[0:int(ny / 2), 0:int(nx / 2)] = bin_power
        # APS_power[0:int(ny/2), int(nx/2):nx]=npm.fliplr(bin_power)
        # APS_power[int(ny/2):ny, 0:int(nx/2)]=npm.flipud(bin_power)
        # APS_power[int(ny/2):ny, int(nx/2):nx]=npm.fliplr(npm.flipud(bin_power))
        APS_power[0:int(ny / 2), int(np.ceil(nx / 2)):] = npm.fliplr(bin_power)
        APS_power[int(np.ceil(ny / 2)):, 0:int(nx / 2)] = npm.flipud(bin_power)
        APS_power[int(np.ceil(ny / 2)):,
                  int(np.ceil(nx / 2)):] = npm.fliplr(npm.flipud(bin_power))
        APS_filt = np.sqrt(APS_power)

        x = np.random.randn(ny, nx)  # white noise
        y_tmp = np.fft.fft2(x)
        y_tmp2 = np.multiply(y_tmp, APS_filt)  # convolve with filter
        y = np.fft.ifft2(y_tmp2)
        APS = np.real(y)

        APS = APS / np.std(
            APS
        ) * std_long  #  adjust the turbulence by the weather model at the longer wavelengths.
        APS = APS * 0.01  # convert from cm to m
        return APS

    def rescale_atmosphere(atm, atm_mean=0.02, atm_sigma=0.005):
        """ a function to rescale a 2d atmosphere with any scale to a mean centered
        one with a min and max value drawn from a normal distribution.  
        Inputs:
            atm | rank 2 array | a single atmosphere.  
            atm_mean | float | average max or min value of atmospheres that are created, in metres.  e.g. if 3 atmospheres have max values of 0.02m, 0.03m, and 0.04m, their mean would be 0.03m
            atm_sigma | float | standard deviation of Gaussian distribution used to generate atmosphere strengths.  
        Returns:
            atm | rank 2 array | a single atmosphere, rescaled to have a maximum signal of around that set by mean_m
        History:
            20YY/MM/DD | MEG | Written
            2020/10/02 | MEG | Standardise throughout to use metres for units.  
        """
        atm -= np.mean(atm)  # mean centre
        atm_strength = (
            atm_sigma * np.random.randn(1)
        ) + atm_mean  # maximum strength of signal is drawn from a gaussian distribution, mean and sigma set in metres.
        if np.abs(np.min(atm)) > np.abs(
                np.max(atm)):  # if range of negative numbers is larger
            atm *= (
                atm_strength / np.abs(np.min(atm))
            )  # strength is drawn from a normal distribution  with a mean set by mean_m (e.g. 0.02)
        else:
            atm *= (
                atm_strength / np.max(atm)
            )  # but if positive part is larger, rescale in the same way as above.
        return atm

    # 0: Check inputs
    if method not in ['fft', 'cov']:
        raise Exception(
            f"'method' must be either 'fft' (for the fourier transform based method), "
            f" or 'cov' (for the covariance based method).  {method} was supplied, so exiting.  "
        )

    #1: determine if linear interpolation is required
    ny, nx = lons_mg.shape
    n_pixs = nx * ny

    if (n_pixs > cov_interpolate_threshold) and (method == 'cov'):
        if verbose:
            print(
                f"The number of pixels ({n_pixs}) is larger than 'cov_interpolate_threshold' ({int(cov_interpolate_threshold)}) so images will be created "
                f"with {int(cov_interpolate_threshold)} pixels and interpolated to the full resolution.  "
            )
        interpolate = True  # set boolean flag
        oversize_factor = n_pixs / cov_interpolate_threshold  # determine how many times too many pixels we have.
        lons_ds = np.linspace(
            lons_mg[-1, 0], lons_mg[-1, -1],
            int(nx * (1 / np.sqrt(oversize_factor)))
        )  # make a downsampled vector of just the longitudes (square root as number of pixels is a measure of area, and this is length)
        lats_ds = np.linspace(
            lats_mg[0, 0], lats_mg[-1, 0],
            int(ny * (1 / np.sqrt(oversize_factor))))  # and for latitudes
        lons_mg_ds = np.repeat(lons_ds[np.newaxis, :], lats_ds.shape,
                               axis=0)  # make rank 2 again
        lats_mg_ds = np.repeat(lats_ds[:, np.newaxis], lons_ds.shape,
                               axis=1)  # and for latitudes
        ny_generate, nx_generate = lons_mg_ds.shape  # get the size of the downsampled grid we'll be generating at
    else:
        interpolate = False  # set boolean flag
        nx_generate = nx  # if not interpolating, these don't change.
        ny_generate = ny
        lons_mg_ds = lons_mg  # if not interpolating, don't need to downsample.
        lats_mg_ds = lats_mg

    #2: calculate distance between points
    ph_turb = np.zeros(
        (n_atms, ny_generate,
         nx_generate))  # initiate output as a rank 3 (ie n_images x ny x nx)
    xyz_m, pixel_spacing = lon_lat_to_ijk(
        lons_mg_ds, lats_mg_ds
    )  # get pixel positions in metres from origin in lower left corner (and also their size in x and y direction)
    xy = xyz_m[
        0:
        2].T  # just get the x and y positions (ie discard z), and make lots x 2 (ie two columns)

    #3: generate atmospheres, using either of the two methods.
    if difference == True:
        n_atms += 1  # if differencing atmospheres, create one extra so that when differencing we are left with the correct number

    if method == 'fft':
        for i in range(n_atms):
            ph_turb[i, :, :] = generate_correlated_noise_fft(
                nx_generate,
                ny_generate,
                std_long=1,
                sp=0.001 * np.mean((pixel_spacing['x'], pixel_spacing['y']))
            )  # generate noise using fft method.  pixel spacing is average in x and y direction (and m converted to km)
            if verbose:
                print(
                    f'Generated {i+1} of {n_atms} single acquisition atmospheres.  '
                )

    else:
        pixel_distances = sp_distance.cdist(
            xy, xy, 'euclidean'
        )  # calcaulte all pixelwise pairs - slow as (pixels x pixels)
        for i in range(n_atms):
            ph_turb[i, :, :] = generate_correlated_noise_cov(
                pixel_distances, cov_Lc,
                (nx_generate, ny_generate))  # generate noise
            if verbose:
                print(
                    f'Generated {i+1} of {n_atms} single acquisition atmospheres.  '
                )

    #3: possibly interplate to bigger size
    if interpolate:
        if verbose:
            print('Interpolating to the larger size...', end='')
        ph_turb_output = np.zeros(
            (n_atms, ny, nx)
        )  # initiate output at the upscaled size (ie the same as the original lons_mg shape)
        for atm_n, atm in enumerate(
                ph_turb
        ):  # loop through the 1st dimension of the rank 3 atmospheres.
            f = scipy_interpolate.interp2d(
                np.arange(0, nx_generate),
                np.arange(0, ny_generate),
                atm,
                kind='linear'
            )  # and interpolate them to a larger size.  First we give it  meshgrids and values for each point
            ph_turb_output[atm_n, :, :] = f(
                np.linspace(0, nx_generate,
                            nx), np.linspace(0, ny_generate, ny)
            )  # then new meshgrids at the original (full) resolution.
        if verbose:
            print('Done!')
    else:
        ph_turb_output = ph_turb  # if we're not interpolating, no change needed

    # 4: rescale to correct range (i.e. a couple of cm)
    ph_turb_m = np.zeros(ph_turb_output.shape)
    for atm_n, atm in enumerate(ph_turb_output):
        ph_turb_m[atm_n, ] = rescale_atmosphere(atm, mean_m)

    # 5: return back to the shape given, which can be a rectangle:
    ph_turb_m = ph_turb_m[:, :lons_mg.shape[0], :lons_mg.shape[1]]

    if water_mask is not None:
        water_mask_r3 = ma.repeat(water_mask[np.newaxis, ],
                                  ph_turb_m.shape[0],
                                  axis=0)
        ph_turb_m = ma.array(ph_turb_m, mask=water_mask_r3)

    return ph_turb_m