def test_convolve_2D_kernels(self):
"""
Check if convolving two kernels with each other works correctly.
"""
gauss_1 = Gaussian2DKernel(3)
gauss_2 = Gaussian2DKernel(4)
test_gauss_3 = Gaussian2DKernel(5)
with pytest.warns(AstropyUserWarning, match=r'Both array and kernel '
r'are Kernel instances'):
gauss_3 = convolve(gauss_1, gauss_2)
assert np.all(np.abs((gauss_3 - test_gauss_3).array) < 0.01)
def test_Gaussian2DKernel_rotated(self):
with pytest.warns(AstropyDeprecationWarning) as w:
Gaussian2DKernel(stddev=10)
assert len(w) == 1
gauss = Gaussian2DKernel(
x_stddev=3, y_stddev=1.5, theta=0.7853981633974483,
x_size=5, y_size=5) # rotated 45 deg ccw
ans = [[0.02267712, 0.02464785, 0.02029238, 0.01265463, 0.00597762],
[0.02464785, 0.03164847, 0.03078144, 0.02267712, 0.01265463],
[0.02029238, 0.03078144, 0.03536777, 0.03078144, 0.02029238],
[0.01265463, 0.02267712, 0.03078144, 0.03164847, 0.02464785],
[0.00597762, 0.01265463, 0.02029238, 0.02464785, 0.02267712]]
assert_allclose(gauss, ans, rtol=0.001) # Rough comparison at 0.1 %
def smooth_heatmap(df, group_bins=100):
X = np.array(df.wgs_lon)
Y = np.array(df.wgs_lat)
bins_list = [group_bins, group_bins]
print("bins list", bins_list)
heatmap, xedges, yedges = np.histogram2d(x=X,
y=Y,
bins=bins_list,
range=[[xmin, xmax], [ymin,
ymax]])
fig, ax = plt.subplots(1, 1)
ax.plot(df.wgs_lon,
df.wgs_lat,
".",
markersize=3,
alpha=0.02,
color="#8B8B83")
conv_z = convolve(np.rot90(heatmap), Gaussian2DKernel(stddev=5))
pd.DataFrame(conv_z).to_csv("convolve_matrix_{}.csv".format(group_bins))
cs = ax.imshow(conv_z, cmap=cm.RdYlGn_r, extent=[xmin, xmax, ymin, ymax])
cx, cy, cz = gauss_kde_filt(df, group_bins * (1j))
ax.contour(cx, cy, cz, cmap=cm.hot_r, linewidths=2)
cb = plt.colorbar(cs)
labels = np.linspace(start=0, stop=cz.max() + 1, num=9, endpoint=True)
loc = np.linspace(-1, 4, 10, endpoint=True)
cb.set_ticks(loc)
cb.set_ticklabels(labels)
plt.title("Max density of logistics service {}".format(int(cz.max())))
plt.savefig("beijing_logistics_service_convolve.png")
plt.show()
def test_clean_sim():
n = m = 32
data = Gaussian2DKernel(stddev=3.0, x_size=n, y_size=m).array
# data = np.zeros((n, m))
# data[13,13] = 10.0
# data[12:14,12:14] = 10.0/4.0
half_log_space = np.logspace(np.log10(0.03030303), np.log10(0.48484848),
10)
theta = np.linspace(0, 2 * np.pi, 32)
theta = theta[np.newaxis, :]
theta = np.repeat(theta, 10, axis=0)
r = half_log_space
r = r[:, np.newaxis]
r = np.repeat(r, 32, axis=1)
x = r * np.sin(theta)
y = r * np.cos(theta)
sub_uv = np.vstack([x.flatten(), y.flatten()])
sub_uv = np.hstack([sub_uv, np.zeros((2, 1))]) / u.arcsec
# Factor of 9 is compensate for the factor of 3 * 3 increase in size
dirty_beam = idft_map(np.ones(321) * 9, (n * 3, m * 3), sub_uv)
vis = dft_map(data, sub_uv)
dirty_map = idft_map(vis, (n, m), sub_uv)
clean_map, res = clean(dirty_map, dirty_beam, clean_beam_width=0)
np.allclose(data, clean_map + res, atol=dirty_beam.max() * 0.1)
def test_multiply_kernel1d_kernel2d(self):
"""
Test that multiplying a 1D kernel with a 2D kernel raises an
exception.
"""
with pytest.raises(Exception):
Gaussian1DKernel(3) * Gaussian2DKernel(3)
def smooth_image(model: Image, width=1.0, normalise=True):
""" Smooth an image with a 2D Gaussian kernel
:param model: Image
:param width: Kernel width in pixels
:param normalise: Normalise kernel peak to unity
"""
assert isinstance(model, Image), model
assert image_is_canonical(model)
from astropy.convolution.kernels import Gaussian2DKernel
from astropy.convolution import convolve_fft
kernel = Gaussian2DKernel(width)
cmodel = create_empty_image_like(model)
nchan, npol, _, _ = model.shape
for pol in range(npol):
for chan in range(nchan):
cmodel.data[chan, pol, :, :] = convolve_fft(model.data[chan,
pol, :, :],
kernel,
normalize_kernel=False,
allow_huge=True)
if normalise and isinstance(kernel, Gaussian2DKernel):
cmodel.data *= 2 * numpy.pi * width**2
return cmodel
def test_uninterpolated_nan_regions(boundary, normalize_kernel):
#8086
# Test NaN interpolation of contiguous NaN regions with kernels of size
# identical and greater than that of the region of NaN values.
# Test case: kernel.shape == NaN_region.shape
kernel = Gaussian2DKernel(1, 5, 5)
nan_centroid = np.full(kernel.shape, np.nan)
image = np.pad(nan_centroid, pad_width=kernel.shape[0]*2, mode='constant',
constant_values=1)
with pytest.warns(AstropyUserWarning,
match="nan_treatment='interpolate', however, NaN values detected "
"post convolution. A contiguous region of NaN values, larger "
"than the kernel size, are present in the input array. "
"Increase the kernel size to avoid this."):
result = convolve(image, kernel, boundary=boundary, nan_treatment='interpolate',
normalize_kernel=normalize_kernel)
assert(np.any(np.isnan(result)))
# Test case: kernel.shape > NaN_region.shape
nan_centroid = np.full((kernel.shape[0]-1, kernel.shape[1]-1), np.nan) # 1 smaller than kerenel
image = np.pad(nan_centroid, pad_width=kernel.shape[0]*2, mode='constant',
constant_values=1)
result = convolve(image, kernel, boundary=boundary, nan_treatment='interpolate',
normalize_kernel=normalize_kernel)
assert(~np.any(np.isnan(result))) # Note: negation
def test_kernel2d_int_ysize(self):
"""
Test that an error is raised if ``Kernel2D`` ``y_size`` is not
an integer.
"""
with pytest.raises(TypeError):
Gaussian2DKernel(3, x_size=5, y_size=1.2)
def sigma_map(on_data, off_data, bins=100, range_axis=np.array([[-1, 1], [-1, 1]]), onoff_factor=1, resolution=0.2):
sigma = resolution
#bins = 20
# sigma pro kernel musi byt zadana v pixelech, ne ve stupnich!
range_x_deg = range_axis[0, 1] - range_axis[0, 0]
n_pix_deg = bins / range_x_deg # px / deg
sigma_px = sigma * n_pix_deg # px
kernel_x_size = int(10*sigma_px)
if kernel_x_size % 2 == 0: # kernel size must be odd
kernel_x_size = kernel_x_size + 1
kernel_y_size = kernel_x_size
kernel_x_size_deg = kernel_x_size / n_pix_deg
# overeno, ze tenhle kernel ma fakt maximum = 1 (staci proste vynasobit gaussovku tou normalizaci na integral)
kernel = 2 * np.pi * sigma_px**2 * Gaussian2DKernel(sigma_px, mode='oversample', factor=10, x_size=kernel_x_size, y_size=kernel_y_size).array
heatmap_on, xedges, yedges = np.histogram2d(on_data['reco_src_x_deg'], on_data['reco_src_y_deg'], bins=bins, range=range_axis)
heatmap_off, xedges, yedges = np.histogram2d(off_data['reco_src_x_deg'], off_data['reco_src_y_deg'], bins=bins, range=range_axis)
image_on = convolve(heatmap_on, kernel, normalize_kernel=False)
image_off = convolve(heatmap_off, kernel, normalize_kernel=False)
sigma_map = sigma_lima(image_on, image_off, onoff_factor)
# adding sign to significance
sign_mask = image_on < image_off*onoff_factor
sigma_map[sign_mask] = -sigma_map[sign_mask]
# Distribution of significance
# - should be gaussian with some tail
sigma_dist = sigma_map.flatten()
x_mid = (xedges[1:]+xedges[:-1])/2.0
y_mid = (yedges[1:]+yedges[:-1])/2.0
xx_mid, yy_mid = np.meshgrid(x_mid, y_mid)
xxx_mid = xx_mid.flatten()
yyy_mid = yy_mid.flatten()
theta2 = xxx_mid**2 + yyy_mid**2
return sigma_map
def test_masked_nddata(convfunc):
arr = np.zeros((11, 11))
arr[4, 5] = arr[6, 5] = arr[5, 4] = arr[5, 6] = 0.2
arr[5, 5] = 1.5
ndd_base = NDData(arr)
mask = arr < 0 # this is all False
mask[5, 5] = True
ndd_mask = NDData(arr, mask=mask)
arrnan = arr.copy()
arrnan[5, 5] = np.nan
ndd_nan = NDData(arrnan)
test_kernel = Gaussian2DKernel(1)
result_base = convfunc(ndd_base, test_kernel)
result_nan = convfunc(ndd_nan, test_kernel)
result_mask = convfunc(ndd_mask, test_kernel)
assert np.allclose(result_nan, result_mask)
assert not np.allclose(result_base, result_mask)
assert not np.allclose(result_base, result_nan)
# check to make sure the mask run doesn't talk back to the initial array
assert np.sum(np.isnan(ndd_base.data)) != np.sum(np.isnan(ndd_nan.data))
def read_thread(PID):
for fid in frameid2heatmap:
if fid == 'BEFORE-FIRST-FRAME':
continue
if int(fid[1]) % num_thread == PID:
pic = convolve(frameid2heatmap[fid], Gaussian2DKernel(stddev=1))
pic = m.to_rgba(pic)[:,:,:3]
plt.imsave(hashID2name[fid[0]][0]+'/' + hashID2name[fid[0]][1] + str(fid[1]) + '.png', pic)
def test_model_2D_kernel(self):
"""
Check Model2DKernel against Gaussian2Dkernel
"""
stddev = 5.
gauss = Gaussian2D(1. / (2 * np.pi * stddev**2), 0, 0, stddev, stddev)
model_gauss_kernel = Model2DKernel(gauss, x_size=21)
gauss_kernel = Gaussian2DKernel(stddev, x_size=21)
assert_almost_equal(model_gauss_kernel.array, gauss_kernel.array,
decimal=12)
def test_Gaussian2DKernel_rotated(self):
gauss = Gaussian2DKernel(x_stddev=3,
y_stddev=1.5,
theta=0.7853981633974483,
x_size=5,
y_size=5) # rotated 45 deg ccw
ans = [[0.04087193, 0.04442386, 0.03657381, 0.02280797, 0.01077372],
[0.04442386, 0.05704137, 0.05547869, 0.04087193, 0.02280797],
[0.03657381, 0.05547869, 0.06374482, 0.05547869, 0.03657381],
[0.02280797, 0.04087193, 0.05547869, 0.05704137, 0.04442386],
[0.01077372, 0.02280797, 0.03657381, 0.04442386, 0.04087193]]
assert_allclose(gauss, ans, rtol=0.001) # Rough comparison at 0.1 %
def test_Gaussian2DKernel_rotated(self):
gauss = Gaussian2DKernel(x_stddev=3,
y_stddev=1.5,
theta=0.7853981633974483,
x_size=5,
y_size=5) # rotated 45 deg ccw
ans = [[0.02267712, 0.02464785, 0.02029238, 0.01265463, 0.00597762],
[0.02464785, 0.03164847, 0.03078144, 0.02267712, 0.01265463],
[0.02029238, 0.03078144, 0.03536777, 0.03078144, 0.02029238],
[0.01265463, 0.02267712, 0.03078144, 0.03164847, 0.02464785],
[0.00597762, 0.01265463, 0.02029238, 0.02464785, 0.02267712]]
assert_allclose(gauss, ans, rtol=0.001) # Rough comparison at 0.1 %
def DrawHeat(matrix, p_normalFactor, p_file):
if p_normalFactor == 'sum':
l_sum = np.sum(matrix)
matrix = matrix / l_sum
elif p_normalFactor == 'max':
l_max = np.max(matrix)
matrix = matrix / l_max
r, c = np.nonzero(matrix)
print matrix[r, c]
# plt.imshow(matrix, cmap='hot', interpolation='nearest')
plt.imshow(convolve(matrix, Gaussian2DKernel(stddev=2)),
interpolation='none')
plt.savefig(p_file)
plt.close()
def smooth2d(data, fwhm, pa=None):
"""
Convolve data with 2d Gaussian kernel and return.
For cube data, the convolution applies only to RA-Dec plane.
Parameters:
data : ndarray (2d or 3d array)
fwhm : float, list or tuple
FWHM size of 2d Gaussian kernel in x and y before rotating.
if list or tuple, fwhm[0] = x_size and fwhm[1] = y_size.
pa : float
Position angle (counterclockwise) in degree.
Returns:
ndarray (smoothed data)
"""
if float(fwhm) == 0.:
return data
if isinstance(fwhm, int) or isinstance(fwhm, float):
kernel = Gaussian2DKernel(fwhm/_f2s)
elif len(fwhm) == 2:
if pa is None:
kernel = Gaussian2DKernel(fwhm[0]/_f2s, fwhm[1]/_f2s)
else:
theta = pa*u.deg.to(u.rad)
kernel = Gaussian2DKernel(fwhm[0]/_f2s, fwhm[1]/_f2s, theta)
else:
raise ValueError('2D-smoothing size: float (fwhm) or tuple (x_fwhm, y_fwhm)')
if len(data.shape) == 1:
raise TypeError('2D-smoothing can not be applied to 1D-array.')
elif len(data.shape) == 2:
return convolve(data, kernel, boundary='extend', preserve_nan=True)
elif len(data.shape) == 3:
smodata = np.full(data.shape, np.nan)
for i in range(len(data)):
smodata[i] = convolve(data[i], kernel, boundary='extend', preserve_nan=True)
return smodata
def spatial_covariance_structure(on, cov_struct=Gaussian2DKernel,
**cov_kwargs):
'''
Impose a covariance structure on the Bernoulli samples.
For every on sample, the next sample the total probability
of those that are turned on around it, weighted by the
structure model.
'''
kernel = Gaussian2DKernel(**cov_kwargs)
pvals = convolve(on, kernel)
return pvals
def test_basic_nddata():
arr = np.zeros((11, 11))
arr[5, 5] = 1
ndd = NDData(arr)
test_kernel = Gaussian2DKernel(1)
result = convolve(ndd, test_kernel)
x, y = np.mgrid[:11, :11]
expected = result[5, 5] * np.exp(-0.5 * ((x - 5)**2 + (y - 5)**2))
np.testing.assert_allclose(result, expected, atol=1e-6)
resultf = convolve_fft(ndd, test_kernel)
np.testing.assert_allclose(resultf, expected, atol=1e-6)
def generate_image(data, bins=100, range_axis=np.array([[-1, 1], [-1, 1]]), onoff_factor=1, resolution=0.2):
# convolving image with gaussian kernel - it effectively handles each event like 2d gaussian instead of a point (sigma = resolution)
range_x_deg = range_axis[0, 1] - range_axis[0, 0]
n_pix_deg = bins / range_x_deg # px / deg
sigma_px = sigma * n_pix_deg # px
kernel_x_size = int(10*sigma_px)
if kernel_x_size % 2 == 0: # kernel size must be odd
kernel_x_size = kernel_x_size + 1
kernel_y_size = kernel_x_size
kernel_x_size_deg = kernel_x_size / n_pix_deg
kernel = Gaussian2DKernel(sigma_px, mode='oversample', factor=100, x_size=kernel_x_size, y_size=kernel_y_size).array
heatmap, xedges, yedges = np.histogram2d(data['reco_src_x_deg'], data['reco_src_y_deg'], bins=bins, range=range_axis)
image = convolve(onoff_factor * heatmap, kernel)
return image
def test_scipy_filter_gaussian(self, width):
"""
Test GaussianKernel against SciPy ndimage gaussian filter.
"""
gauss_kernel_1D = Gaussian1DKernel(width)
gauss_kernel_1D.normalize()
gauss_kernel_2D = Gaussian2DKernel(width)
gauss_kernel_2D.normalize()
astropy_1D = convolve(delta_pulse_1D, gauss_kernel_1D, boundary='fill')
astropy_2D = convolve(delta_pulse_2D, gauss_kernel_2D, boundary='fill')
scipy_1D = filters.gaussian_filter(delta_pulse_1D, width)
scipy_2D = filters.gaussian_filter(delta_pulse_2D, width)
assert_almost_equal(astropy_1D, scipy_1D, decimal=12)
assert_almost_equal(astropy_2D, scipy_2D, decimal=12)
def test_array_keyword_not_allowed(self):
"""
Regression test for issue #10439
"""
x = np.ones([10, 10])
with pytest.raises(TypeError, match=r".* allowed .*"):
AiryDisk2DKernel(2, array=x)
Box1DKernel(2, array=x)
Box2DKernel(2, array=x)
Gaussian1DKernel(2, array=x)
Gaussian2DKernel(2, array=x)
RickerWavelet1DKernel(2, array=x)
RickerWavelet2DKernel(2, array=x)
Model1DKernel(Gaussian1D(1, 0, 2), array=x)
Model2DKernel(Gaussian2D(1, 0, 0, 2, 2), array=x)
Ring2DKernel(9, 8, array=x)
Tophat2DKernel(2, array=x)
Trapezoid1DKernel(2, array=x)
Trapezoid1DKernel(2, array=x)
def from_model(cls, model, from_filter=None, to_filter=None):
"""
This function ...
:param model:
:param from_filter:
:param to_filter:
:return:
"""
# Get properties
#center =
#sigma = fitting.sigma_symmetric(model)
# Create a kernel
#kernel = Gaussian2DKernel(sigma, x_size=kernel_size, y_size=kernel_size)
#kernel.normalize() # to suppress warning
if isinstance(model, Gaussian2D):
x_stddev = model.x_stddev
y_stddev = model.y_stddev
if not np.isclose(x_stddev, y_stddev):
raise ValueError("Model is not symmetric")
kernel = Gaussian2DKernel(x_stddev)
elif isinstance(model, AiryDisk2D):
radius = model.radius
kernel = AiryDisk2DKernel(radius)
else:
raise ValueError("Model not supported")
kernel.normalize()
# Get the FWHM
fwhm = fitting.fwhm_symmetric(model)
# Set metadata
extra_meta = dict()
if from_filter is not None: extra_meta["frmfltr"] = str(from_filter)
if to_filter is not None: extra_meta["tofltr"] = str(to_filter)
# Create instance of this class
return cls(kernel.array,
fwhm=fwhm,
extra_meta=extra_meta,
prepared=True)
def test_regressiontest_issue9168():
"""
Issue #9168 pointed out that kernels can be (unitless) quantities, which
leads to crashes when inplace modifications are made to arrays in
convolve/convolve_fft, so we now strip the quantity aspects off of kernels.
"""
x = np.array([[1., 2., 3.],
[4., 5., 6.],
[7., 8., 9.]],)
kernel_fwhm = 1*u.arcsec
pixel_size = 1*u.arcsec
kernel = Gaussian2DKernel(x_stddev=kernel_fwhm/pixel_size)
result = convolve_fft(x, kernel, boundary='fill', fill_value=np.nan,
preserve_nan=True)
result = convolve(x, kernel, boundary='fill', fill_value=np.nan,
preserve_nan=True)
def makeGraph(year, minOccPixels, minPoints, minSpecies, minEcoregions, toTest, allGraphs, outfile):
#Import summary files
def readData(directory, toTest, numFiles):
filePath = str('E:\\GIS_layers\\GBIF\\' + str(directory) + str(year) +'_0.csv')
data = pd.read_csv(filePath, delimiter = ',', usecols = (0,1,2,3,4,5,6,8,9,10,11), names = ['identity', 'numSpecies', 'numPoints', 'numPixels', 'occPixels', 'comps', 'numEcoregions', 'log', 'sqrt', 'linear', 'uncorrected'])
if numFiles > 1:
for x in range(numFiles-1):
y = x + 1
filePath = str('E:\\GIS_layers\\GBIF\\' + str(directory) + '\\ddm_summary_2017_jaccard_' + str(y) + '.csv')
data = data.append(pd.read_csv(filePath, delimiter = ',', usecols = (0,1,2,3,4,5,6,8,9,10,11), names = ['identity', 'numSpecies', 'numPoints', 'numPixels', 'occPixels', 'comps', 'numEcoregions', 'identity', 'log', 'sqrt', 'linear', 'uncorrected']))
data = data.drop_duplicates(subset=['identity'])
ND_value = -9999
minComps = 0
data = data[data['identity'] >= 0]
data = data[data['occPixels'] >=minOccPixels]
data = data[data['numSpecies'] >=minSpecies]
data = data[data['numEcoregions'] >=minEcoregions]
data = data[data['numPoints'] >=minPoints]
data = data[data['comps'] >=minComps]
if allGraphs == 'yes':
print float(len(data[data[toTest] >= 0.95])) /float(len(data)), directory
return data.copy()
toTest = 'log'
plants = readData('Plants', toTest, 4)
bugs = readData('Arthropods', toTest, 2)
mammals = readData('Mammals', toTest, 2)
herps = readData('Herps', toTest, 2)
birds = readData('Birds', toTest, 4)
fungi = readData('Fungi', toTest, 2)
reptiles = readData('Reptiles', toTest, 2)
amphibians = readData('Amphibians', toTest, 2)
taxa = [plants, bugs, herps, mammals, birds, fungi, reptiles, amphibians]
taxaS = ['plants', 'bugs', 'herps', 'mammals', 'birds', 'fungi', 'reptiles', 'amphibians']
if allGraphs == 'yes':
for x in range(len(taxa)):
statist, p = scipy.stats.kstest(taxa[x][toTest], 'uniform')
print 'ddm', str(taxaS[x]), 'uniform', statist, p
for y in range(len(taxa)):
statist, p = scipy.stats.ks_2samp(taxa[x][toTest], taxa[y][toTest])
print str(taxaS[x]), str(taxaS[y]), statist, p
y += 1
x += 1
y = 0
#Define plot style
plt.style.use(u'seaborn-white')
fig2 = plt.figure()
gs = GridSpec(2,4)
if allGraphs == 'yes':
fig2.set_size_inches(16,8)
ax = fig2.add_subplot(gs[:,1:3])
else:
fig2.set_size_inches(10,8)
ax = fig2.add_subplot(gs[:,:])
#Plot transect specific p-values from Species accumulation curves
data = [1-fungi[toTest], 1-bugs[toTest], 1-reptiles[toTest], 1-amphibians[toTest], 1-plants[toTest], 1-birds[toTest], 1-mammals[toTest]]
plt.violinplot(data, showmeans=False, showextrema=False, showmedians=True, vert = False)
ax.set_yticks([7,6,5,4,3,2,1])
ax.text(0.5, 5.25, 'Plants ' + '(n = ' + str(len(plants)) + ')', fontsize = 16, horizontalalignment='center')
ax.text(0.5, 2.25, 'Arthropods ' + '(n = ' + str(len(bugs)) + ')', fontsize = 16, horizontalalignment='center')
ax.text(0.5, 3.25, 'Reptiles ' + '(n = ' + str(len(reptiles)) + ')', fontsize = 16, horizontalalignment='center')
ax.text(0.5, 4.25, 'Amphibians ' + '(n = ' + str(len(amphibians)) + ')', fontsize = 16, horizontalalignment='center')
ax.text(0.5, 7.25, 'Mammals ' + '(n = ' + str(len(mammals)) + ')', fontsize = 16, horizontalalignment='center')
ax.text(0.5, 6.25, 'Birds ' + '(n = ' + str(len(birds)) + ')', fontsize = 16, horizontalalignment='center')
ax.text(0.5, 1.25, 'Fungi ' + '(n = ' + str(len(fungi)) + ')', fontsize = 16, horizontalalignment='center')
#ax.set_yticklabels(label, rotation = 45, ha = 'right')
ax.set(xlim=(0, 1))
ax.set(ylim=(0.5,7.5))
ax.set_title('Distance-Similarity Matrices', weight = 'bold', fontsize = 22)
ax.set_xlabel( u"\u2190" + ' More supportive of Sharp-Transition Hypothesis', fontsize = 16)
ax.set_yticklabels([])
im = image.imread('C:/Users/Jeffrey/Documents/Academic/Stanford/Steg/Drafts/v9/taxa/plant.tif')
ax.imshow(im, aspect='auto', extent=(0.9, 0.95,5, 5.5))
im = image.imread('C:/Users/Jeffrey/Documents/Academic/Stanford/Steg/Drafts/v9/taxa/bug.png')
ax.imshow(im, aspect='auto', extent=(0.89, 0.96, 2, 2.5))
im = image.imread('C:/Users/Jeffrey/Documents/Academic/Stanford/Steg/Drafts/v9/taxa/amphibian.tif')
ax.imshow(im, aspect='auto', extent=(0.87, 0.97, 4, 4.5))
im = image.imread('C:/Users/Jeffrey/Documents/Academic/Stanford/Steg/Drafts/v9/taxa/mammal.tif')
ax.imshow(im, aspect='auto', extent=(0.87, 0.97, 7, 7.5))
im = image.imread('C:/Users/Jeffrey/Documents/Academic/Stanford/Steg/Drafts/v9/taxa/bird.tif')
ax.imshow(im, aspect='auto', extent=(0.9, 0.95, 6, 6.5))
im = image.imread('C:/Users/Jeffrey/Documents/Academic/Stanford/Steg/Drafts/v9/taxa/reptile.png')
ax.imshow(im, aspect='auto', extent=(0.87, 0.97, 3, 3.5))
im = image.imread('C:/Users/Jeffrey/Documents/Academic/Stanford/Steg/Drafts/v9/taxa/fungi.png')
ax.imshow(im, aspect='auto', extent=(0.9, 0.95, 1, 1.5))
if allGraphs == 'no':
plt.tight_layout()
plt.savefig(outfile)
else:
ax.text(0.05, 7.6, 'C.', fontsize = 16, horizontalalignment='center')
ax.text(0.85, 7.2, 'A', fontsize = 16, horizontalalignment='center')
ax.text(0.85, 6.2, 'A', fontsize = 16, horizontalalignment='center')
ax.text(0.85, 5.2, 'A', fontsize = 16, horizontalalignment='center')
ax.text(0.85, 4.2, 'B', fontsize = 16, horizontalalignment='center')
ax.text(0.85, 3.2, 'B', fontsize = 16, horizontalalignment='center')
ax.text(0.85, 2.2, 'C', fontsize = 16, horizontalalignment='center')
ax.text(0.85, 1.2, 'D', fontsize = 16, horizontalalignment='center')
def distanceMatrix(pixelBySpeciesPivot):
def removeMinOccs(data, minOccs):
data = data[np.logical_or.reduce([data[:,-1] >= minOccs])]
return data
def reduceMaxOccs(data, maxOccs):
occupiedPixels = np.count_nonzero(data[:,-1])
#Randomize overrepresented pixels
npix = len(data)
stats = np.zeros(shape=(npix, 8))
#Loop through pixels
x = 0
for x in range(npix):
#Identify pixels with too many occs
if np.sum(data[x,6:-1]) > maxOccs:
#Extract that line to play with
subArray = np.reshape(data[x,6:-1], len(data[x,6:-1]))
#Make list to hold the randomly sampled occs
elements = []
#Loop through occs in that pixel
for z in range(len(subArray)):
#make a list with x entries for each species where x is the number of occs in that species in that pixel
for s in range(int(subArray[z])):
toAppend = ['Species.' + str(z)]
elements = elements + toAppend
#Randomly subsample from the array of species (where the number of occs in that species = the number of occs in that pixel)
chosen = []
#only choose up to the number of max occs
for y in range(maxOccs):
#ranomdly select an element
select = random.randint(0,len(elements)-1)
selected = elements[select]
#Delete that element from the original array and add it to your array of selected occurences
del elements[select]
chosen = chosen + [selected]
#Make an array equal in length to the number of species in your whole dataset
replacementRow = np.zeros(len(subArray))
#Parse out your selected occs into the right format to put back into data array
for p in range(len(chosen)):
speciesNumber = int(chosen[p].split('.')[1])
replacementRow[speciesNumber] += 1
#Replace the original line with your now subsetted line of data
data[x,6:-1] = replacementRow
return data
def setUpHoldingArray(data):
#Set up for randomly shuffling while mantaining ecorgion bin width
unique = list(np.unique(data[:,4]))
numUniques = len(unique)
holding = np.empty(len(unique))
npix = len(data)
#Set up yuor first holding array, x-y valude indicates how many pixels are in taht ecoregion (0) and the number of species after that ecoregion is sampled (1)
x = 0
y = 0
for x in range(npix):
if x > 0:
if data[x,4] == oldBiome:
oldBiome = data[x,4]
x += 1
else:
holding[y] = x
oldBiome = data[x,4]
x += 1
y += 1
else:
oldBiome = data[x,4]
x += 1
holding[numUniques-1] = npix - 1
oldSelect = 0
for y in range(len(holding)):
holding[y] = holding[y] - oldSelect
oldSelect += holding[y]
holdingOld = np.copy(holding)
return holding, unique, numUniques, npix, holdingOld
def calculatePredictedGrid(data, holding):
#Calculate real and stattistical differences between pixels
X_true = data[:,6:-1]
statDistances = scipy.spatial.distance.squareform(scipy.spatial.distance.pdist(X_true, metric='jaccard', p=2, w=None, V=None, VI=None)).astype('float')
statDistances = np.subtract(1, statDistances)
realDistances = scipy.spatial.distance.cdist(data[:,2:3], data[:,2:3], metric='euclidean', p=2, V=None, VI=None, w=None).astype('float')
x = 0
for x in range(len(X_true)):
statDistances[x,x] = np.nan
realDistances[x,x] = np.nan
if np.nansum(X_true[x,:]) == 0:
statDistances[x,:] = np.nan
statDistances[:,x] = np.nan
realDistances[statDistances == 0] = 'nan'
statDistances[statDistances == 0] = 'nan'
statDistances = np.log(statDistances)
statDistances = np.divide(np.subtract(statDistances, np.nanmean(statDistances)), np.nanstd(statDistances))
realDistances = np.sqrt(realDistances)
realDistances = np.multiply(np.divide(np.subtract(realDistances, np.nanmean(realDistances)), np.nanstd(realDistances)), -1)
flatStatA = np.ndarray.flatten(statDistances)
flatStat = flatStatA[~np.isnan(flatStatA)]
distFlat = np.ndarray.flatten(realDistances)
distFlat = distFlat[~np.isnan(flatStatA)]
d = {'distance': distFlat, 'jaccard': flatStat}
df = pd.DataFrame(data=d)
nick = smf.ols(formula = 'jaccard~distance', data = df)
res = nick.fit()
intercept = res.params[0]
distValue = res.params[1]
predictedGrid = np.add(np.multiply(realDistances, distValue), intercept)
observedMinusPredicted = np.subtract(statDistances, predictedGrid)
graphingGrid = np.zeros_like(observedMinusPredicted)
x_left = 0
y_left = 0
x_right = 0
y_right = 0
for i in range(len(holding)):
x_left = int(np.sum(holding[:i]))
x_right = int(np.sum(holding[:i+1]))
for j in range(len(holding)):
y_left = int(np.sum(holding[:j]))
y_right = int(np.sum(holding[:j+1]))
z = 0
try:
z = np.nansum(observedMinusPredicted[x_left:x_right, y_left:y_right])
except:
z = 0
if np.isfinite(z) == True:
graphingGrid[x_left:x_right, y_left:y_right] = np.nanmean(observedMinusPredicted[x_left:x_right, y_left:y_right])
j += 1
j = 0
i += 1
return observedMinusPredicted, X_true, graphingGrid
def doPermuatations(observedMinusPredicted, holding, trials):
gridValues = np.zeros(trials)
highest = -100000
lowest = 100000
for i in range(trials):
#Do modularity calculation
a = 0
b = 0
j = 0
for j in range(len(unique)):
a = int(np.sum(holding[:j]))
b = int(np.sum(holding[:j+1]))
z = 0
try:
z = np.nansum(observedMinusPredicted[a:b,a:b])
except:
continue
if np.isfinite(z) == True:
gridValues[i] += z
a += b
if gridValues[i] >= highest:
highest = gridValues[i]
highestA = np.copy(holding)
elif gridValues[i] <= lowest:
lowest = gridValues[i]
lowestA = np.copy(holding)
np.random.shuffle(holding)
randomB = np.copy(holding)
return gridValues, highestA, lowestA, randomB
#Read in species pivot table data
dataO = np.genfromtxt(pixelBySpeciesPivot, dtype= 'float' , delimiter=',', skip_header=1)
#Change non-occurences to zeros (important for futrue analysis)
data = np.nan_to_num(dataO)
#Filter out extra occurances and pixels with too few occurences
data = removeMinOccs(data, 0)
data = reduceMaxOccs(data, 50)
#Figure out where ecoregion boundaries fall
holding, unique, numUniques, npix, holdingOld = setUpHoldingArray(data)
observedMinusPredicted, X_true, graphingGrid = calculatePredictedGrid(data, holding)
trials = 5000
gridValues, highestA, lowestA, randomB = doPermuatations(observedMinusPredicted, holding, trials)
return gridValues, observedMinusPredicted, holdingOld, graphingGrid, highestA, lowestA, randomB
gridValues, observedMinusPredicted, holdingOld, graphingGrid, highestA, lowestA, randomB = distanceMatrix('E:\\GIS_layers\\GBIF\\Plants\\Outputs\\9660b.csv')
ax7 = fig2.add_subplot(gs[0,0])
ax7.set_title('Sharp-Transition' + '\n' + 'Hypothesis', weight = 'bold', fontsize = 18)
ax7.set_xlabel('Pixel Number', fontsize = 16, color = 'k')
ax7.set_ylabel('Pixel Number', fontsize = 16, color = 'k')
ax7.text(1, -30, 'A.', fontsize = 16, horizontalalignment='left')
setMin = (max(np.nanmin(observedMinusPredicted) * -1, np.nanmax(observedMinusPredicted)) * -1)/2
setMax = (max(np.nanmin(observedMinusPredicted) * -1, np.nanmax(observedMinusPredicted)))/2
plt.imshow(convolve(observedMinusPredicted, Gaussian2DKernel(stddev=8)), interpolation='none', cmap = 'bwr', vmin = setMin, vmax = setMax)
toPlot = np.zeros_like(holdingOld)
for i in range(len(holdingOld)):
toPlot[i] = np.sum(holdingOld[:i])
for xc in range(len(holdingOld)):
plt.axvline(x=toPlot[xc], linestyle='-', color = 'k')
plt.axhline(y=toPlot[xc], linestyle='-', color = 'k')
toPlot = np.zeros_like(randomB)
for i in range(len(randomB)):
toPlot[i] = np.sum(randomB[:i])
for xc in range(len(holdingOld)):
plt.axvline(x=toPlot[xc], linestyle=':', color = 'k')
plt.axhline(y=toPlot[xc], linestyle=':', color = 'k')
ax8 = fig2.add_subplot(gs[1,0])
ax8.set_ylabel('Frequency', fontsize = 16)
ax8.set_xlabel('Modularity explained', fontsize = 16)
ax8.set_title('B.', loc='left', fontsize = 16)
sns.kdeplot(gridValues, color = 'pink', kernel = 'gau', shade = True)
plt.axvline(x = gridValues[0], linestyle='-', color = 'k')
plt.axvline(x = gridValues[-1], linestyle=':', color = 'k')
ax8.set_yticklabels([])
ax8.set_xticklabels([])
gridValues, observedMinusPredicted, holdingOld, graphingGrid, highestA, lowestA, randomB = distanceMatrix('E:\\GIS_layers\\GBIF\\Plants\\Outputs\\1321a.csv')
ax9 = fig2.add_subplot(gs[0,1])
ax9.set_title('Gradual-Transition' + '\n' + 'Hypothesis', weight = 'bold', fontsize = 18)
ax9.set_xlabel('Pixel Number', fontsize = 16, color = 'k')
ax9.set_ylabel('Pixel Number', fontsize = 16, color = 'k')
ax9.text(1, -30, 'D.', fontsize = 16, horizontalalignment='left')
setMin = (max(np.nanmin(observedMinusPredicted) * -1, np.nanmax(observedMinusPredicted)) * -1)/2
setMax = (max(np.nanmin(observedMinusPredicted) * -1, np.nanmax(observedMinusPredicted)))/2
plt.imshow(convolve(observedMinusPredicted, Gaussian2DKernel(stddev=8)), interpolation='none', cmap = 'bwr', vmin = setMin, vmax = setMax)
toPlot = np.zeros_like(holdingOld)
for i in range(len(holdingOld)):
toPlot[i] = np.sum(holdingOld[:i])
for xc in range(len(holdingOld)):
plt.axvline(x=toPlot[xc], linestyle='-', color = 'k')
plt.axhline(y=toPlot[xc], linestyle='-', color = 'k')
toPlot = np.zeros_like(randomB)
for i in range(len(randomB)):
toPlot[i] = np.sum(randomB[:i])
for xc in range(len(holdingOld)):
plt.axvline(x=toPlot[xc], linestyle=':', color = 'k')
plt.axhline(y=toPlot[xc], linestyle=':', color = 'k')
ax6 = fig2.add_subplot(gs[1,1])
ax6.set_ylabel('Frequency', fontsize = 16)
ax6.set_xlabel('Modularity explained', fontsize = 16)
ax6.set_title('E.', loc='left', fontsize = 16)
sns.kdeplot(gridValues, color = 'pink', kernel = 'gau', shade = True)
plt.axvline(x = gridValues[0], linestyle='-', color = 'k')
plt.axvline(x = gridValues[-1], linestyle=':', color = 'k')
ax6.set_yticklabels([])
ax6.set_xticklabels([])
#Show the plot to fix formatting before saving (click on tight layout)
plt.savefig(outfile)
plt.close()
def gaussian_smooth(data, kernel_stddev_px):
return convolve_fft(data,
Gaussian2DKernel(kernel_stddev_px),
boundary="wrap")
def test_Gaussian2DKernel_even_size(self):
"""
Check if even size for GaussianKernel works.
"""
gauss = Gaussian2DKernel(3, x_size=10, y_size=10)
assert gauss.array.shape == (10, 10)
z = []
df_to_edit = pd.read_csv('./differenced_bus.csv')
for index, row in df_to_edit.iterrows():
x.append(row['x_bucket'])
y.append(row['y_bucket'])
z.append(row['difference_PM2.5'])
heatmap, xedges, yedges = np.histogram2d(x, y, bins=500, weights=z)
extent = [yedges[0], yedges[-1], xedges[0], xedges[-1]]
plt.clf()
plt.title('PM2_5 heatmap example')
plt.ylabel('y')
plt.xlabel('x')
#plt.pcolor(heatmap, cmap='Greys', norm=LogNorm(vmin=min(z), vmax=max(z)))
plt.imshow(convolve(heatmap, Gaussian2DKernel(stddev=2)),
extent=extent,
cmap='inferno')
plt.colorbar()
plt.show()
'''fig, ax = plt.subplots(2, 1)
pcm = ax[0].pcolor(x, y, z,
norm=colors.LogNorm(vmin=min(z), vmax=max(z)),
cmap='Greys')
fig.colorbar(pcm, ax=ax[0], extend='max')
pcm = ax[1].pcolor(x, y, z, cmap='Greys')
fig.colorbar(pcm, ax=ax[1], extend='max')
fig.show()'''
'''from astropy.convolution.kernels import Gaussian2DKernel
def test_multiply_kernel2d(self):
"""Test that multiplying two 2D kernels raises an exception."""
gauss = Gaussian2DKernel(3)
with pytest.raises(Exception):
gauss * gauss
from astropy.convolution.kernels import Moffat2DKernel
SHAPES_ODD = [[15, 15], [31, 31]]
SHAPES_EVEN = [[8, 8], [16, 16], [32, 32]] # FIXME: not used ?!
NOSHAPE = [[None, None]]
WIDTHS = [2, 3, 4, 5]
KERNELS = []
for shape in SHAPES_ODD + NOSHAPE:
for width in WIDTHS:
KERNELS.append(
Gaussian2DKernel(width,
x_size=shape[0],
y_size=shape[1],
mode='oversample',
factor=10))
KERNELS.append(
Box2DKernel(width,
x_size=shape[0],
y_size=shape[1],
mode='oversample',
factor=10))
KERNELS.append(
Tophat2DKernel(width,
x_size=shape[0],
y_size=shape[1],
mode='oversample',
def test_gaussian_2d_kernel_quantity():
# Make sure that the angle can be a quantity
kernel1 = Gaussian2DKernel(x_stddev=2, y_stddev=4, theta=45 * u.deg)
kernel2 = Gaussian2DKernel(x_stddev=2, y_stddev=4, theta=np.pi / 4)
assert_allclose(kernel1.array, kernel2.array)
