def test_testCopySize(self):
# Tests of some subtle points of copying and sizing.
n = [0, 0, 1, 0, 0]
m = make_mask(n)
m2 = make_mask(m)
assert_(m is m2)
m3 = make_mask(m, copy=True)
assert_(m is not m3)
x1 = np.arange(5)
y1 = array(x1, mask=m)
assert_(y1._data is not x1)
assert_(allequal(x1, y1._data))
assert_(y1._mask is m)
y1a = array(y1, copy=0)
# For copy=False, one might expect that the array would just
# passed on, i.e., that it would be "is" instead of "==".
# See gh-4043 for discussion.
assert_(y1a._mask.__array_interface__ ==
y1._mask.__array_interface__)
y2 = array(x1, mask=m3, copy=0)
assert_(y2._mask is m3)
assert_(y2[2] is masked)
y2[2] = 9
assert_(y2[2] is not masked)
assert_(y2._mask is m3)
assert_(allequal(y2.mask, 0))
y2a = array(x1, mask=m, copy=1)
assert_(y2a._mask is not m)
assert_(y2a[2] is masked)
y2a[2] = 9
assert_(y2a[2] is not masked)
assert_(y2a._mask is not m)
assert_(allequal(y2a.mask, 0))
y3 = array(x1 * 1.0, mask=m)
assert_(filled(y3).dtype is (x1 * 1.0).dtype)
x4 = arange(4)
x4[2] = masked
y4 = resize(x4, (8,))
assert_(eq(concatenate([x4, x4]), y4))
assert_(eq(getmask(y4), [0, 0, 1, 0, 0, 0, 1, 0]))
y5 = repeat(x4, (2, 2, 2, 2), axis=0)
assert_(eq(y5, [0, 0, 1, 1, 2, 2, 3, 3]))
y6 = repeat(x4, 2, axis=0)
assert_(eq(y5, y6))
def test_testCopySize(self):
# Tests of some subtle points of copying and sizing.
n = [0, 0, 1, 0, 0]
m = make_mask(n)
m2 = make_mask(m)
assert_(m is m2)
m3 = make_mask(m, copy=1)
assert_(m is not m3)
x1 = np.arange(5)
y1 = array(x1, mask=m)
assert_(y1._data is not x1)
assert_(allequal(x1, y1._data))
assert_(y1._mask is m)
y1a = array(y1, copy=0)
# For copy=False, one might expect that the array would just
# passed on, i.e., that it would be "is" instead of "==".
# See gh-4043 for discussion.
assert_(y1a._mask.__array_interface__ ==
y1._mask.__array_interface__)
y2 = array(x1, mask=m3, copy=0)
assert_(y2._mask is m3)
assert_(y2[2] is masked)
y2[2] = 9
assert_(y2[2] is not masked)
assert_(y2._mask is m3)
assert_(allequal(y2.mask, 0))
y2a = array(x1, mask=m, copy=1)
assert_(y2a._mask is not m)
assert_(y2a[2] is masked)
y2a[2] = 9
assert_(y2a[2] is not masked)
assert_(y2a._mask is not m)
assert_(allequal(y2a.mask, 0))
y3 = array(x1 * 1.0, mask=m)
assert_(filled(y3).dtype is (x1 * 1.0).dtype)
x4 = arange(4)
x4[2] = masked
y4 = resize(x4, (8,))
assert_(eq(concatenate([x4, x4]), y4))
assert_(eq(getmask(y4), [0, 0, 1, 0, 0, 0, 1, 0]))
y5 = repeat(x4, (2, 2, 2, 2), axis=0)
assert_(eq(y5, [0, 0, 1, 1, 2, 2, 3, 3]))
y6 = repeat(x4, 2, axis=0)
assert_(eq(y5, y6))
def test_testCopySize(self):
# Tests of some subtle points of copying and sizing.
n = [0, 0, 1, 0, 0]
m = make_mask(n)
m2 = make_mask(m)
assert_(m is m2)
m3 = make_mask(m, copy=1)
assert_(m is not m3)
x1 = np.arange(5)
y1 = array(x1, mask=m)
assert_(y1._data is not x1)
assert_(allequal(x1, y1._data))
assert_(y1.mask is m)
y1a = array(y1, copy=0)
assert_(y1a.mask is y1.mask)
y2 = array(x1, mask=m3, copy=0)
assert_(y2.mask is m3)
assert_(y2[2] is masked)
y2[2] = 9
assert_(y2[2] is not masked)
assert_(y2.mask is m3)
assert_(allequal(y2.mask, 0))
y2a = array(x1, mask=m, copy=1)
assert_(y2a.mask is not m)
assert_(y2a[2] is masked)
y2a[2] = 9
assert_(y2a[2] is not masked)
assert_(y2a.mask is not m)
assert_(allequal(y2a.mask, 0))
y3 = array(x1 * 1.0, mask=m)
assert_(filled(y3).dtype is (x1 * 1.0).dtype)
x4 = arange(4)
x4[2] = masked
y4 = resize(x4, (8,))
assert_(eq(concatenate([x4, x4]), y4))
assert_(eq(getmask(y4), [0, 0, 1, 0, 0, 0, 1, 0]))
y5 = repeat(x4, (2, 2, 2, 2), axis=0)
assert_(eq(y5, [0, 0, 1, 1, 2, 2, 3, 3]))
y6 = repeat(x4, 2, axis=0)
assert_(eq(y5, y6))
def test_testCopySize(self):
# Tests of some subtle points of copying and sizing.
n = [0, 0, 1, 0, 0]
m = make_mask(n)
m2 = make_mask(m)
assert_(m is m2)
m3 = make_mask(m, copy=1)
assert_(m is not m3)
x1 = np.arange(5)
y1 = array(x1, mask=m)
assert_(y1._data is not x1)
assert_(allequal(x1, y1._data))
assert_(y1.mask is m)
y1a = array(y1, copy=0)
assert_(y1a.mask is y1.mask)
y2 = array(x1, mask=m3, copy=0)
assert_(y2.mask is m3)
assert_(y2[2] is masked)
y2[2] = 9
assert_(y2[2] is not masked)
assert_(y2.mask is m3)
assert_(allequal(y2.mask, 0))
y2a = array(x1, mask=m, copy=1)
assert_(y2a.mask is not m)
assert_(y2a[2] is masked)
y2a[2] = 9
assert_(y2a[2] is not masked)
assert_(y2a.mask is not m)
assert_(allequal(y2a.mask, 0))
y3 = array(x1 * 1.0, mask=m)
assert_(filled(y3).dtype is (x1 * 1.0).dtype)
x4 = arange(4)
x4[2] = masked
y4 = resize(x4, (8, ))
assert_(eq(concatenate([x4, x4]), y4))
assert_(eq(getmask(y4), [0, 0, 1, 0, 0, 0, 1, 0]))
y5 = repeat(x4, (2, 2, 2, 2), axis=0)
assert_(eq(y5, [0, 0, 1, 1, 2, 2, 3, 3]))
y6 = repeat(x4, 2, axis=0)
assert_(eq(y5, y6))
def test_testCopySize(self):
# Tests of some subtle points of copying and sizing.
with suppress_warnings() as sup:
sup.filter(
np.ma.core.MaskedArrayFutureWarning,
"setting an item on a masked array which has a "
"shared mask will not copy")
n = [0, 0, 1, 0, 0]
m = make_mask(n)
m2 = make_mask(m)
self.assertTrue(m is m2)
m3 = make_mask(m, copy=1)
self.assertTrue(m is not m3)
x1 = np.arange(5)
y1 = array(x1, mask=m)
self.assertTrue(y1._data is not x1)
self.assertTrue(allequal(x1, y1._data))
self.assertTrue(y1.mask is m)
y1a = array(y1, copy=0)
self.assertTrue(y1a.mask is y1.mask)
y2 = array(x1, mask=m, copy=0)
self.assertTrue(y2.mask is m)
self.assertTrue(y2[2] is masked)
y2[2] = 9
self.assertTrue(y2[2] is not masked)
self.assertTrue(y2.mask is not m)
self.assertTrue(allequal(y2.mask, 0))
y3 = array(x1 * 1.0, mask=m)
self.assertTrue(filled(y3).dtype is (x1 * 1.0).dtype)
x4 = arange(4)
x4[2] = masked
y4 = resize(x4, (8,))
self.assertTrue(eq(concatenate([x4, x4]), y4))
self.assertTrue(eq(getmask(y4), [0, 0, 1, 0, 0, 0, 1, 0]))
y5 = repeat(x4, (2, 2, 2, 2), axis=0)
self.assertTrue(eq(y5, [0, 0, 1, 1, 2, 2, 3, 3]))
y6 = repeat(x4, 2, axis=0)
self.assertTrue(eq(y5, y6))
def test_testCopySize(self):
# Tests of some subtle points of copying and sizing.
with suppress_warnings() as sup:
sup.filter(
np.ma.core.MaskedArrayFutureWarning,
"setting an item on a masked array which has a "
"shared mask will not copy")
n = [0, 0, 1, 0, 0]
m = make_mask(n)
m2 = make_mask(m)
self.assertTrue(m is m2)
m3 = make_mask(m, copy=1)
self.assertTrue(m is not m3)
x1 = np.arange(5)
y1 = array(x1, mask=m)
self.assertTrue(y1._data is not x1)
self.assertTrue(allequal(x1, y1._data))
self.assertTrue(y1.mask is m)
y1a = array(y1, copy=0)
self.assertTrue(y1a.mask is y1.mask)
y2 = array(x1, mask=m, copy=0)
self.assertTrue(y2.mask is m)
self.assertTrue(y2[2] is masked)
y2[2] = 9
self.assertTrue(y2[2] is not masked)
self.assertTrue(y2.mask is not m)
self.assertTrue(allequal(y2.mask, 0))
y3 = array(x1 * 1.0, mask=m)
self.assertTrue(filled(y3).dtype is (x1 * 1.0).dtype)
x4 = arange(4)
x4[2] = masked
y4 = resize(x4, (8,))
self.assertTrue(eq(concatenate([x4, x4]), y4))
self.assertTrue(eq(getmask(y4), [0, 0, 1, 0, 0, 0, 1, 0]))
y5 = repeat(x4, (2, 2, 2, 2), axis=0)
self.assertTrue(eq(y5, [0, 0, 1, 1, 2, 2, 3, 3]))
y6 = repeat(x4, 2, axis=0)
self.assertTrue(eq(y5, y6))
def _h_arrows(self, length):
""" length is in arrow width units """
# It might be possible to streamline the code
# and speed it up a bit by using complex (x,y)
# instead of separate arrays; but any gain would be slight.
minsh = self.minshaft * self.headlength
N = len(length)
length = length.reshape(N, 1)
# x, y: normal horizontal arrow
x = np.array([0, -self.headaxislength, -self.headlength, 0],
np.float64)
x = x + np.array([0, 1, 1, 1]) * length
y = 0.5 * np.array([1, 1, self.headwidth, 0], np.float64)
y = np.repeat(y[np.newaxis, :], N, axis=0)
# x0, y0: arrow without shaft, for short vectors
x0 = np.array(
[0, minsh - self.headaxislength, minsh - self.headlength, minsh],
np.float64)
y0 = 0.5 * np.array([1, 1, self.headwidth, 0], np.float64)
ii = [0, 1, 2, 3, 2, 1, 0]
X = x.take(ii, 1)
Y = y.take(ii, 1)
Y[:, 3:] *= -1
X0 = x0.take(ii)
Y0 = y0.take(ii)
Y0[3:] *= -1
shrink = length / minsh
X0 = shrink * X0[np.newaxis, :]
Y0 = shrink * Y0[np.newaxis, :]
short = np.repeat(length < minsh, 7, axis=1)
#print 'short', length < minsh
# Now select X0, Y0 if short, otherwise X, Y
X = ma.where(short, X0, X)
Y = ma.where(short, Y0, Y)
if self.pivot[:3] == 'mid':
X -= 0.5 * X[:, 3, np.newaxis]
elif self.pivot[:3] == 'tip':
X = X - X[:, 3, np.newaxis] #numpy bug? using -= does not
# work here unless we multiply
# by a float first, as with 'mid'.
tooshort = length < self.minlength
if tooshort.any():
# Use a heptagonal dot:
th = np.arange(0, 7, 1, np.float64) * (np.pi / 3.0)
x1 = np.cos(th) * self.minlength * 0.5
y1 = np.sin(th) * self.minlength * 0.5
X1 = np.repeat(x1[np.newaxis, :], N, axis=0)
Y1 = np.repeat(y1[np.newaxis, :], N, axis=0)
tooshort = ma.repeat(tooshort, 7, 1)
X = ma.where(tooshort, X1, X)
Y = ma.where(tooshort, Y1, Y)
return X, Y
def _h_arrows(self, length):
""" length is in arrow width units """
# It might be possible to streamline the code
# and speed it up a bit by using complex (x,y)
# instead of separate arrays; but any gain would be slight.
minsh = self.minshaft * self.headlength
N = len(length)
length = length.reshape(N, 1)
# x, y: normal horizontal arrow
x = np.array([0, -self.headaxislength,
-self.headlength, 0], np.float64)
x = x + np.array([0,1,1,1]) * length
y = 0.5 * np.array([1, 1, self.headwidth, 0], np.float64)
y = np.repeat(y[np.newaxis,:], N, axis=0)
# x0, y0: arrow without shaft, for short vectors
x0 = np.array([0, minsh-self.headaxislength,
minsh-self.headlength, minsh], np.float64)
y0 = 0.5 * np.array([1, 1, self.headwidth, 0], np.float64)
ii = [0,1,2,3,2,1,0]
X = x.take(ii, 1)
Y = y.take(ii, 1)
Y[:, 3:] *= -1
X0 = x0.take(ii)
Y0 = y0.take(ii)
Y0[3:] *= -1
shrink = length/minsh
X0 = shrink * X0[np.newaxis,:]
Y0 = shrink * Y0[np.newaxis,:]
short = np.repeat(length < minsh, 7, axis=1)
#print 'short', length < minsh
# Now select X0, Y0 if short, otherwise X, Y
X = ma.where(short, X0, X)
Y = ma.where(short, Y0, Y)
if self.pivot[:3] == 'mid':
X -= 0.5 * X[:,3, np.newaxis]
elif self.pivot[:3] == 'tip':
X = X - X[:,3, np.newaxis] #numpy bug? using -= does not
# work here unless we multiply
# by a float first, as with 'mid'.
tooshort = length < self.minlength
if tooshort.any():
# Use a heptagonal dot:
th = np.arange(0,7,1, np.float64) * (np.pi/3.0)
x1 = np.cos(th) * self.minlength * 0.5
y1 = np.sin(th) * self.minlength * 0.5
X1 = np.repeat(x1[np.newaxis, :], N, axis=0)
Y1 = np.repeat(y1[np.newaxis, :], N, axis=0)
tooshort = ma.repeat(tooshort, 7, 1)
X = ma.where(tooshort, X1, X)
Y = ma.where(tooshort, Y1, Y)
return X, Y
def atmosphere_turb(n_atms,
lons_mg,
lats_mg,
water_mask=None,
Lc=2000,
difference=False,
verbose=False,
interpolate_threshold=1e4,
mean_m=0.02):
""" A function to create synthetic turbulent atmospheres based on the
methods in Lohman Simons 2005. Note that due to memory issues,
largers ones are made by interpolateing smaller ones. Can return atmsopheres
for an individual acquisition, or as the difference of two (as per an
interferogram). Units are in metres.
Inputs:
n_atms | int | number of atmospheres to generate
lons_mg | rank 2 array | longitudes of the bottom left corner of each pixel.
lats_mg | rank 2 array | latitudes of the bottom left corner of each pixel.
water_mask | rank 2 array | If supplied, this is applied to the atmospheres generated, convering them to masked arrays.
Lc | float | length scale of correlation, in metres. If smaller, noise is patchier, and if larger, smoother.
difference | boolean | If difference, two atmospheres are generated and subtracted from each other to make a single atmosphere.
verbose | boolean | Controls info printed to screen when running.
interpolate_threshold | int | if n_pixs is greater than this, images will be generated at size so that the total number of pixels doesn't exceed this.
e.g. if set to 1e4 (10000, the default) and images are 120*120, they will be generated at 100*100 then upsampled to 120*120.
mean_m | float | average max or min value of atmospheres that are created. e.g. if 3 atmospheres have max values of 0.02m, 0.03m, and 0.04m, their mean would be 0.03cm.
Outputs:
ph_turb | r3 array | n_atms x n_pixs x n_pixs, UNITS ARE M. Note that if a water_mask is provided, this is applied and a masked array is returned.
2019/09/13 | MEG | adapted extensively from a simple script
2020/10/02 | MEG | Change so that a water mask is optional.
2020/10/05 | MEG | Change so that meshgrids of the longitudes and latitudes of each pixel are used to set resolution.
Also fix a bug in how Lc is handled, so this is now in meters.
2020/10/06 | MEG | Add support for rectangular atmospheres, fix some bugs.
"""
import numpy as np
import numpy.ma as ma
from scipy.spatial import distance as sp_distance # geopy also has a distance function. Rename for safety.
from scipy import interpolate as scipy_interpolate
from auxiliary_functions import lon_lat_to_ijk
def generate_correlated_noise(pixel_distances, Lc, shape):
""" given a matrix of pixel distances (in meters) and a length scale for the noise (also in meters),
generate some 2d spatially correlated noise.
Inputs:
pixel_distances | rank 2 array | pixels x pixels, distance between each on in metres.
Lc | float | Length scale over which the noise is correlated. units are metres.
shape | tuple | (nx, ny) NOTE X FIRST!
Returns:
y_2d | rank 2 array | spatially correlated noise.
History:
2019/06/?? | MEG | Written
2020/10/05 | MEG | Overhauled to be in metres and use scipy cholesky
2020/10/06 | MEG | Add support for rectangular atmospheres.
"""
import scipy
nx = shape[0]
ny = shape[1]
Cd = np.exp(
(-1 * pixel_distances) / Lc
) # from the matrix of distances, convert to covariances using exponential equation
#Cd_L = np.linalg.cholesky(Cd) # ie Cd = CD_L @ CD_L.T
Cd_L = scipy.linalg.cholesky(
Cd, lower=True) # better error messages than the numpy version.
x = np.random.randn(
(ny * nx)) # Parsons 2007 syntax - x for uncorrelated noise
y = Cd_L @ x # y for correlated noise
y_2d = np.reshape(y, (ny, nx)) # turn back to rank 2
return y_2d
def rescale_atmosphere(atm, atm_mean=0.02, atm_sigma=0.005):
""" a function to rescale a 2d atmosphere with any scale to a mean centered
one with a min and max value drawn from a normal distribution.
Inputs:
atm | rank 2 array | a single atmosphere.
atm_mean | float | average max or min value of atmospheres that are created, in metres. e.g. if 3 atmospheres have max values of 0.02m, 0.03m, and 0.04m, their mean would be 0.03m
atm_sigma | float | standard deviation of Gaussian distribution used to generate atmosphere strengths.
Returns:
atm | rank 2 array | a single atmosphere, rescaled to have a maximum signal of around that set by mean_m
History:
20YY/MM/DD | MEG | Written
2020/10/02 | MEG | Standardise throughout to use metres for units.
"""
atm -= np.mean(atm) # mean centre
atm_strength = (
atm_sigma * np.random.randn(1)
) + atm_mean # maximum strength of signal is drawn from a gaussian distribution, mean and sigma set in metres.
if np.abs(np.min(atm)) > np.abs(
np.max(atm)): # if range of negative numbers is larger
atm *= (
atm_strength / np.abs(np.min(atm))
) # strength is drawn from a normal distribution with a mean set by mean_m (e.g. 0.02)
else:
atm *= (
atm_strength / np.max(atm)
) # but if positive part is larger, rescale in the same way as above.
return atm
#1: determine if linear interpolation is required
ny, nx = lons_mg.shape
n_pixs = nx * ny
if n_pixs > interpolate_threshold:
if verbose:
print(
f"The number of pixels is larger than 'interpolate_threshold' ({interpolate_threshold}) so images will be created "
f"with {interpolate_threshold} pixels and interpolated to the full resolution. "
)
interpolate = True # set boolean flag
oversize_factor = n_pixs / interpolate_threshold # determine how many times too many pixels we have.
lons_ds = np.linspace(
lons_mg[-1, 0], lons_mg[-1, -1],
int(nx * (1 / np.sqrt(oversize_factor)))
) # make a downsampled vector of just the longitudes (square root as number of pixels is a measure of area, and this is length)
lats_ds = np.linspace(
lats_mg[0, 0], lats_mg[-1, 0],
int(ny * (1 / np.sqrt(oversize_factor)))) # and for latitudes
lons_mg_ds = np.repeat(lons_ds[np.newaxis, :], lats_ds.shape,
axis=0) # make rank 2 again
lats_mg_ds = np.repeat(lats_ds[:, np.newaxis], lons_ds.shape,
axis=1) # and for latitudes
ny_generate, nx_generate = lons_mg_ds.shape # get the size of the downsampled grid we'll be generating at
else:
interpolate = False # set boolean flag
nx_generate = nx # if not interpolating, these don't change.
ny_generate = ny
lons_mg_ds = lons_mg # if not interpolating, don't need to downsample.
lats_mg_ds = lats_mg
#2: calculate distance between points
ph_turb = np.zeros(
(n_atms, ny_generate,
nx_generate)) # initiate output as a rank 3 (ie n_images x ny x nx)
xyz_m, pixel_spacing = lon_lat_to_ijk(
lons_mg_ds, lats_mg_ds
) # get pixel positions in metres from origin in lower left corner (and also their size in x and y direction)
xy = xyz_m[
0:
2].T # just get the x and y positions (ie discard z), and make lots x 2 (ie two columns)
pixel_distances = sp_distance.cdist(
xy, xy, 'euclidean'
) # calcaulte all pixelwise pairs - slow as (pixels x pixels)
#3: generate atmospheres
if difference is False: # this just generates a single turbulent atmosphere
for i in range(n_atms):
ph_turb[i, :, :] = generate_correlated_noise(
pixel_distances, Lc,
(nx_generate, ny_generate)) # generate noise
if verbose:
print(
f'Generated {i} of {n_atms} single acquisition atmospheres. '
)
elif difference is True: # but we can also generate the difference between two atmospheres, as you'd see in an interferogram.
for i in range(n_atms):
y_2d_1 = generate_correlated_noise(
pixel_distances, Lc,
(nx_generate, ny_generate)) # generate first noise
y_2d_2 = generate_correlated_noise(
pixel_distances, Lc,
(nx_generate, ny_generate)) # generate second noise
ph_turb[
i, :, :] = y_2d_1 - y_2d_2 # difference between the two atmospheres
if verbose:
print(
f'Generated {i} of {n_atms} interferogram atmospheres. ')
else:
raise Exception(
"'difference' must be either True or False. Quitting. ")
#3: possibly interplate to bigger size
if interpolate:
if verbose:
print('Interpolating to the larger size...', end='')
ph_turb_output = np.zeros(
(n_atms, ny, nx)
) # initiate output at the upscaled size (ie the same as the original lons_mg shape)
for atm_n, atm in enumerate(
ph_turb
): # loop through the 1st dimension of the rank 3 atmospheres.
f = scipy_interpolate.interp2d(
np.arange(0, nx_generate),
np.arange(0, ny_generate),
atm,
kind='linear'
) # and interpolate them to a larger size. First we give it meshgrids and values for each point
ph_turb_output[atm_n, :, :] = f(
np.linspace(0, nx_generate,
nx), np.linspace(0, ny_generate, ny)
) # then new meshgrids at the original (full) resolution.
if verbose:
print('Done!')
else:
ph_turb_output = ph_turb # if we're not interpolating, no change needed
# 4: rescale to correct range (i.e. a couple of cm)
ph_turb_m = np.zeros(ph_turb_output.shape)
for atm_n, atm in enumerate(ph_turb_output):
ph_turb_m[atm_n, ] = rescale_atmosphere(atm, mean_m)
# 5: return back to the shape given, which can be a rectangle:
ph_turb_m = ph_turb_m[:, :lons_mg.shape[0], :lons_mg.shape[1]]
if water_mask is not None:
water_mask_r3 = ma.repeat(water_mask[np.newaxis, ],
ph_turb_m.shape[0],
axis=0)
ph_turb_m = ma.array(ph_turb_m, mask=water_mask_r3)
return ph_turb_m
def augment_data(X, Y_class, Y_loc, n_data=500):
""" A function to augment data and presserve the location label for any deformation.
Note that n_data is not particularly intelligent as many more data may be generated,
and only n_data returned, so even if n_data is low, the function can still be slow.
Inputs:
X | rank 4 array | data.
Y_class | rank 2 array | One hot encoding of class labels
Y_loc | rank 2 array | locations of deformation
n_data | int |
Returns:
X_aug | rank 4 array | data.
Y_class_aug | rank 2 array | One hot encoding of class labels
Y_loc_aug | rank 2 array | locations of deformation
History:
2019/??/?? | MEG | Written
2020/10/29 | MEG | Write the docs.
2020_01_11 | MEG | Major rewrite to speed things up.
"""
import numpy as np
import numpy.ma as ma
flips = ['none', 'up_down', 'left_right',
'both'] # the three possible types of flip
# 0: get the correct nunber of data
n_ifgs = X.shape[0]
data_dict = {
'X': X, # package the data and labels together into a dict
'Y_class': Y_class,
'Y_loc': Y_loc
}
if n_ifgs < n_data: # if we have fewer ifgs than we need, repeat them
n_repeat = int(
np.ceil(n_data / n_ifgs)
) # get the number of repeats needed (round up and make an int)
data_dict['X'] = ma.repeat(data_dict['X'], axis=0, repeats=n_repeat)
data_dict['Y_class'] = np.repeat(data_dict['Y_class'],
axis=0,
repeats=n_repeat)
data_dict['Y_loc'] = np.repeat(data_dict['Y_loc'],
axis=0,
repeats=n_repeat)
data_dict = shuffle_arrays(
data_dict
) # shuffle (so that these aren't in the order of the class labels)
for key in data_dict: # then crop them to the correct number
data_dict[key] = data_dict[key][:n_data, ]
X_aug = data_dict[
'X'] # and unpack as this function doesn't use dictionaries
Y_class_aug = data_dict['Y_class']
Y_loc_aug = data_dict['Y_loc']
# 1: do the flips
for data_n in range(n_data):
flip = flips[np.random.randint(0,
len(flips))] # choose a flip at random
if flip != 'none':
X_aug[data_n:data_n +
1, ], Y_loc_aug[data_n:data_n + 1, ] = augment_flip(
X_aug[data_n:data_n + 1, ],
Y_loc_aug[data_n:data_n + 1, ],
flip) # do the augmentaiton via one of the flips.
# 2: do the rotations
X_aug, Y_loc_aug = augment_rotate(X_aug, Y_loc_aug) # rotate
# 3: Do the translations
X_aug, Y_loc_aug = augment_translate(X_aug,
Y_loc_aug,
max_translate=(20, 20))
return X_aug, Y_class_aug, Y_loc_aug # return, and select only the desired data.
def atmosphere_turb(n_atms,
lons_mg,
lats_mg,
method='fft',
mean_m=0.02,
water_mask=None,
difference=False,
verbose=False,
cov_interpolate_threshold=1e4,
cov_Lc=2000):
""" A function to create synthetic turbulent atmospheres based on the methods in Lohman Simons 2005, or using Andy Hooper and Lin Shen's fft method.
Note that due to memory issues, when using the covariance (Lohman) method, largers ones are made by interpolateing smaller ones.
Can return atmsopheres for an individual acquisition, or as the difference of two (as per an interferogram). Units are in metres.
Inputs:
n_atms | int | number of atmospheres to generate
lons_mg | rank 2 array | longitudes of the bottom left corner of each pixel.
lats_mg | rank 2 array | latitudes of the bottom left corner of each pixel.
method | string | 'fft' or 'cov'. Cov for the Lohmans Simons (sp?) method, fft for Andy Hooper/Lin Shen's fft method (which is much faster). Currently no way to set length scale using fft method.
mean_m | float | average max or min value of atmospheres that are created. e.g. if 3 atmospheres have max values of 0.02m, 0.03m, and 0.04m, their mean would be 0.03cm.
water_mask | rank 2 array | If supplied, this is applied to the atmospheres generated, convering them to masked arrays.
difference | boolean | If difference, two atmospheres are generated and subtracted from each other to make a single atmosphere.
verbose | boolean | Controls info printed to screen when running.
cov_Lc | float | length scale of correlation, in metres. If smaller, noise is patchier, and if larger, smoother.
cov_interpolate_threshold | int | if n_pixs is greater than this, images will be generated at size so that the total number of pixels doesn't exceed this.
e.g. if set to 1e4 (10000, the default) and images are 120*120, they will be generated at 100*100 then upsampled to 120*120.
Outputs:
ph_turb | r3 array | n_atms x n_pixs x n_pixs, UNITS ARE M. Note that if a water_mask is provided, this is applied and a masked array is returned.
2019/09/13 | MEG | adapted extensively from a simple script
2020/10/02 | MEG | Change so that a water mask is optional.
2020/10/05 | MEG | Change so that meshgrids of the longitudes and latitudes of each pixel are used to set resolution.
Also fix a bug in how cov_Lc is handled, so this is now in meters.
2020/10/06 | MEG | Add support for rectangular atmospheres, fix some bugs.
2020_03_01 | MEG | Add option to use Lin Shen/Andy Hooper's fft method which is quicker than the covariance method.
"""
import numpy as np
import numpy.ma as ma
from scipy.spatial import distance as sp_distance # geopy also has a distance function. Rename for safety.
from scipy import interpolate as scipy_interpolate
from auxiliary_functions import lon_lat_to_ijk
def generate_correlated_noise_cov(pixel_distances, cov_Lc, shape):
""" given a matrix of pixel distances (in meters) and a length scale for the noise (also in meters),
generate some 2d spatially correlated noise.
Inputs:
pixel_distances | rank 2 array | pixels x pixels, distance between each on in metres.
cov_Lc | float | Length scale over which the noise is correlated. units are metres.
shape | tuple | (nx, ny) NOTE X FIRST!
Returns:
y_2d | rank 2 array | spatially correlated noise.
History:
2019/06/?? | MEG | Written
2020/10/05 | MEG | Overhauled to be in metres and use scipy cholesky
2020/10/06 | MEG | Add support for rectangular atmospheres.
"""
import scipy
nx = shape[0]
ny = shape[1]
Cd = np.exp(
(-1 * pixel_distances) / cov_Lc
) # from the matrix of distances, convert to covariances using exponential equation
#Cd_L = np.linalg.cholesky(Cd) # ie Cd = CD_L @ CD_L.T
Cd_L = scipy.linalg.cholesky(
Cd, lower=True) # better error messages than the numpy version.
x = np.random.randn(
(ny * nx)) # Parsons 2007 syntax - x for uncorrelated noise
y = Cd_L @ x # y for correlated noise
y_2d = np.reshape(y, (ny, nx)) # turn back to rank 2
return y_2d
def generate_correlated_noise_fft(nx, ny, std_long, sp):
""" A function to create synthetic turbulent troposphere delay using an FFT approach.
The power of the turbulence is tuned by the weather model at the longer wavelengths.
Inputs:
nx (int) -- width of troposphere
Ny (int) -- length of troposphere
std_long (float) -- standard deviation of the weather model at the longer wavelengths. Default = ?
sp | int | pixel spacing in km
Outputs:
APS (float): 2D array, Ny * nx, units are m.
History:
2020_??_?? | LS | Adapted from code by Andy Hooper.
2021_03_01 | MEG | Small change to docs and inputs to work with SyInterferoPy
"""
import numpy as np
import numpy.matlib as npm
import math
np.seterr(divide='ignore')
cut_off_freq = 1 / 50 # drop wavelengths above 50 km
x = np.arange(0, int(nx / 2)) # positive frequencies only
y = np.arange(0, int(ny / 2)) # positive frequencies only
freq_x = np.divide(x, nx * sp)
freq_y = np.divide(y, ny * sp)
Y, X = npm.meshgrid(freq_x, freq_y)
freq = np.sqrt((X * X + Y * Y) / 2) # 2D positive frequencies
log_power = np.log10(freq) * -11 / 3 # -11/3 in 2D gives -8/3 in 1D
ix = np.where(freq < 2 / 3)
log_power[ix] = np.log10(freq[ix]) * -8 / 3 - math.log10(
2 / 3) # change slope at 1.5 km (2/3 cycles per km)
bin_power = np.power(10, log_power)
ix = np.where(freq < cut_off_freq)
bin_power[ix] = 0
APS_power = np.zeros(
(ny, nx)) # mirror positive frequencies into other quadrants
APS_power[0:int(ny / 2), 0:int(nx / 2)] = bin_power
# APS_power[0:int(ny/2), int(nx/2):nx]=npm.fliplr(bin_power)
# APS_power[int(ny/2):ny, 0:int(nx/2)]=npm.flipud(bin_power)
# APS_power[int(ny/2):ny, int(nx/2):nx]=npm.fliplr(npm.flipud(bin_power))
APS_power[0:int(ny / 2), int(np.ceil(nx / 2)):] = npm.fliplr(bin_power)
APS_power[int(np.ceil(ny / 2)):, 0:int(nx / 2)] = npm.flipud(bin_power)
APS_power[int(np.ceil(ny / 2)):,
int(np.ceil(nx / 2)):] = npm.fliplr(npm.flipud(bin_power))
APS_filt = np.sqrt(APS_power)
x = np.random.randn(ny, nx) # white noise
y_tmp = np.fft.fft2(x)
y_tmp2 = np.multiply(y_tmp, APS_filt) # convolve with filter
y = np.fft.ifft2(y_tmp2)
APS = np.real(y)
APS = APS / np.std(
APS
) * std_long # adjust the turbulence by the weather model at the longer wavelengths.
APS = APS * 0.01 # convert from cm to m
return APS
def rescale_atmosphere(atm, atm_mean=0.02, atm_sigma=0.005):
""" a function to rescale a 2d atmosphere with any scale to a mean centered
one with a min and max value drawn from a normal distribution.
Inputs:
atm | rank 2 array | a single atmosphere.
atm_mean | float | average max or min value of atmospheres that are created, in metres. e.g. if 3 atmospheres have max values of 0.02m, 0.03m, and 0.04m, their mean would be 0.03m
atm_sigma | float | standard deviation of Gaussian distribution used to generate atmosphere strengths.
Returns:
atm | rank 2 array | a single atmosphere, rescaled to have a maximum signal of around that set by mean_m
History:
20YY/MM/DD | MEG | Written
2020/10/02 | MEG | Standardise throughout to use metres for units.
"""
atm -= np.mean(atm) # mean centre
atm_strength = (
atm_sigma * np.random.randn(1)
) + atm_mean # maximum strength of signal is drawn from a gaussian distribution, mean and sigma set in metres.
if np.abs(np.min(atm)) > np.abs(
np.max(atm)): # if range of negative numbers is larger
atm *= (
atm_strength / np.abs(np.min(atm))
) # strength is drawn from a normal distribution with a mean set by mean_m (e.g. 0.02)
else:
atm *= (
atm_strength / np.max(atm)
) # but if positive part is larger, rescale in the same way as above.
return atm
# 0: Check inputs
if method not in ['fft', 'cov']:
raise Exception(
f"'method' must be either 'fft' (for the fourier transform based method), "
f" or 'cov' (for the covariance based method). {method} was supplied, so exiting. "
)
#1: determine if linear interpolation is required
ny, nx = lons_mg.shape
n_pixs = nx * ny
if (n_pixs > cov_interpolate_threshold) and (method == 'cov'):
if verbose:
print(
f"The number of pixels ({n_pixs}) is larger than 'cov_interpolate_threshold' ({int(cov_interpolate_threshold)}) so images will be created "
f"with {int(cov_interpolate_threshold)} pixels and interpolated to the full resolution. "
)
interpolate = True # set boolean flag
oversize_factor = n_pixs / cov_interpolate_threshold # determine how many times too many pixels we have.
lons_ds = np.linspace(
lons_mg[-1, 0], lons_mg[-1, -1],
int(nx * (1 / np.sqrt(oversize_factor)))
) # make a downsampled vector of just the longitudes (square root as number of pixels is a measure of area, and this is length)
lats_ds = np.linspace(
lats_mg[0, 0], lats_mg[-1, 0],
int(ny * (1 / np.sqrt(oversize_factor)))) # and for latitudes
lons_mg_ds = np.repeat(lons_ds[np.newaxis, :], lats_ds.shape,
axis=0) # make rank 2 again
lats_mg_ds = np.repeat(lats_ds[:, np.newaxis], lons_ds.shape,
axis=1) # and for latitudes
ny_generate, nx_generate = lons_mg_ds.shape # get the size of the downsampled grid we'll be generating at
else:
interpolate = False # set boolean flag
nx_generate = nx # if not interpolating, these don't change.
ny_generate = ny
lons_mg_ds = lons_mg # if not interpolating, don't need to downsample.
lats_mg_ds = lats_mg
#2: calculate distance between points
ph_turb = np.zeros(
(n_atms, ny_generate,
nx_generate)) # initiate output as a rank 3 (ie n_images x ny x nx)
xyz_m, pixel_spacing = lon_lat_to_ijk(
lons_mg_ds, lats_mg_ds
) # get pixel positions in metres from origin in lower left corner (and also their size in x and y direction)
xy = xyz_m[
0:
2].T # just get the x and y positions (ie discard z), and make lots x 2 (ie two columns)
#3: generate atmospheres, using either of the two methods.
if difference == True:
n_atms += 1 # if differencing atmospheres, create one extra so that when differencing we are left with the correct number
if method == 'fft':
for i in range(n_atms):
ph_turb[i, :, :] = generate_correlated_noise_fft(
nx_generate,
ny_generate,
std_long=1,
sp=0.001 * np.mean((pixel_spacing['x'], pixel_spacing['y']))
) # generate noise using fft method. pixel spacing is average in x and y direction (and m converted to km)
if verbose:
print(
f'Generated {i+1} of {n_atms} single acquisition atmospheres. '
)
else:
pixel_distances = sp_distance.cdist(
xy, xy, 'euclidean'
) # calcaulte all pixelwise pairs - slow as (pixels x pixels)
for i in range(n_atms):
ph_turb[i, :, :] = generate_correlated_noise_cov(
pixel_distances, cov_Lc,
(nx_generate, ny_generate)) # generate noise
if verbose:
print(
f'Generated {i+1} of {n_atms} single acquisition atmospheres. '
)
#3: possibly interplate to bigger size
if interpolate:
if verbose:
print('Interpolating to the larger size...', end='')
ph_turb_output = np.zeros(
(n_atms, ny, nx)
) # initiate output at the upscaled size (ie the same as the original lons_mg shape)
for atm_n, atm in enumerate(
ph_turb
): # loop through the 1st dimension of the rank 3 atmospheres.
f = scipy_interpolate.interp2d(
np.arange(0, nx_generate),
np.arange(0, ny_generate),
atm,
kind='linear'
) # and interpolate them to a larger size. First we give it meshgrids and values for each point
ph_turb_output[atm_n, :, :] = f(
np.linspace(0, nx_generate,
nx), np.linspace(0, ny_generate, ny)
) # then new meshgrids at the original (full) resolution.
if verbose:
print('Done!')
else:
ph_turb_output = ph_turb # if we're not interpolating, no change needed
# 4: rescale to correct range (i.e. a couple of cm)
ph_turb_m = np.zeros(ph_turb_output.shape)
for atm_n, atm in enumerate(ph_turb_output):
ph_turb_m[atm_n, ] = rescale_atmosphere(atm, mean_m)
# 5: return back to the shape given, which can be a rectangle:
ph_turb_m = ph_turb_m[:, :lons_mg.shape[0], :lons_mg.shape[1]]
if water_mask is not None:
water_mask_r3 = ma.repeat(water_mask[np.newaxis, ],
ph_turb_m.shape[0],
axis=0)
ph_turb_m = ma.array(ph_turb_m, mask=water_mask_r3)
return ph_turb_m
