def smooth_log_spectrogram(S2,kernel_length,
num_window_samples,do_freq_smoothing=False):
S = S2.copy()
S[S== 0] = S[S!=0].min()
S = np.log(S)
# S = S[::-1,:]
# smooth the spectrogram
x = np.arange(0, kernel_length, 1, np.float64)
x0 = kernel_length // 2.
sigma = 1
g=1/(sigma * np.sqrt(2*np.pi)) *np.exp(-((x-x0)**2 / 2* sigma**2))
smoothing_kernel = g/g.sum()
# feature smoothing should leave only meaningful features
if do_freq_smoothing:
S_smoothed = scipy_convolve(S,smoothing_kernel.reshape(1,kernel_length),mode ='valid')
else:
S_smoothed = S
# preserve time length of spectrogram, smooth over time
S_smoothed= scipy_convolve(S_smoothed,smoothing_kernel.reshape(kernel_length,1),mode='same')
if num_window_samples > 256:
S_subsampled = S_smoothed[:,::2]
else:
S_subsampled = S_smoothed
return S_subsampled
def _find_absorption_lines(self, flux_threshold=None, width=None):
wave, flux = self.data_norm
self.width = width
kernel = [1, 0, -1]
dY = scipy_convolve(flux, kernel, mode='valid')
S = np.sign(dY)
ddS = scipy_convolve(S, kernel, mode='valid')
candidates = np.where(dY < 0)[0] + (len(kernel) - 1)
line_inds = sorted(
set(candidates).intersection(np.where(ddS == 2)[0] + 1))
if flux_threshold is not None:
line_inds = np.array(line_inds)[flux[line_inds] < -flux_threshold]
# Now group them and find the max highest point.
line_inds_grouped = self._consecutive(line_inds, stepsize=1)
if len(line_inds_grouped[0]) > 0:
absorption_inds = [
inds[np.argmin(flux[inds])] for inds in line_inds_grouped
]
else:
absorption_inds = []
if width is not None and absorption_inds:
ii = np.where(
np.abs(wave[absorption_inds] - self.line) <= width)[0]
absorption_inds = np.array(absorption_inds)[ii]
del ii
return wave[absorption_inds], absorption_inds
def smooth_log_spectrogram(S2,
kernel_length,
num_window_samples,
do_freq_smoothing=False):
S = S2.copy()
S[S == 0] = S[S != 0].min()
S = np.log(S)
# S = S[::-1,:]
# smooth the spectrogram
x = np.arange(0, kernel_length, 1, np.float64)
x0 = kernel_length // 2.
sigma = 1
g = 1 / (sigma * np.sqrt(2 * np.pi)) * np.exp(-(
(x - x0)**2 / 2 * sigma**2))
smoothing_kernel = g / g.sum()
# feature smoothing should leave only meaningful features
if do_freq_smoothing:
S_smoothed = scipy_convolve(S,
smoothing_kernel.reshape(1, kernel_length),
mode='valid')
else:
S_smoothed = S
# preserve time length of spectrogram, smooth over time
S_smoothed = scipy_convolve(S_smoothed,
smoothing_kernel.reshape(kernel_length, 1),
mode='same')
if num_window_samples > 256:
S_subsampled = S_smoothed[:, ::2]
else:
S_subsampled = S_smoothed
return S_subsampled
def filter(signal):
low_pass = scipy_convolve(signal,
filter_taps,
mode='same',
method='direct')
hight_pass = scipy_convolve(signal,
filter_taps_h,
mode='same',
method='direct')
return np.maximum(hight_pass - low_pass, 0)
def smooth_file(file1, size_gauss):
"""
Lisse une image avec une matrice gaussiene
"""
kernel = Gaussian2DKernel(size_gauss)
file1 = scipy_convolve(file1, kernel, mode='same', method='direct')
return file1
def test_torch_convolve_matches_scipy_direct_convolve(inputs, request):
x, y = request.getfixturevalue(inputs)
scipy_conv = torch.from_numpy(scipy_convolve(x, y, mode='full', method='direct'))
torch_dirconv = torch_directconvolve(x, y)
torch_fftconv = torch_fftconvolve(x, y)
assert torch.all(torch.isclose(torch_dirconv, scipy_conv))
assert torch.all(torch.isclose(torch_fftconv, scipy_conv))
def _find_emission_lines(self, wave, flux, flux_threshold=None):
kernel = [1, 0, -1]
dY = scipy_convolve(flux, kernel, 'valid')
S = np.sign(dY)
ddS = scipy_convolve(S, kernel, 'valid')
candidates = np.where(dY > 0)[0] + (len(kernel) - 1)
line_inds = sorted(
set(candidates).intersection(np.where(ddS == -2)[0] + 1))
if flux_threshold is not None:
line_inds = np.array(line_inds)[flux[line_inds] > flux_threshold]
line_inds_grouped = self._consecutive(line_inds, stepsize=1)
if len(line_inds_grouped[0]) > 0:
emission_inds = [
inds[np.argmax(flux[inds])] for inds in line_inds_grouped
]
else:
emission_inds = []
return wave[emission_inds], emission_inds
def abime(file2):
# file2 = get_image(my_file)
img = file2
img_zerod = img.copy()
# img_zerod[np.isnan(img)] = 0
kernel = np.array([[1 / 81 for i in range(9)] for j in range(9)])
# kernel = Gaussian2DKernel(1) # x_stddev=1)
file2 = scipy_convolve(img, kernel, mode='same', method='direct')
return file2
def to_new_fwhm_box1d(ll, ss, width, nsamp=3):
""" Use box kernel to convolve down and resample spectrum """
nres = np.round(width/dl)
npix = np.int(np.round(nres/nsamp))
# print("Width of {0:1.4f} is {1:1.2f} pix".format(width, nres))
kernel = CC.Box1DKernel(nres)
sp = scipy_convolve(ss, kernel, mode='same', method='direct')
return ll, sp, npix
def to_observed(ll, ss, width, N=125*1200, A_B=3e-9, nsamp=3):
l0 = np.median(ll).to(AA)
dl, GRF, kernel = create_GRF(l0, N, A_B)
_, sp, npix = to_new_fwhm_box1d(ll, ss, width, nsamp=nsamp)
spl = scipy_convolve(sp, kernel, mode='same', method='direct')
return ll[::npix], sp[::npix], spl[::npix], dl, GRF
def smooth_file(file1, valsmooth):
"""
Lisse une image avec une matrice gaussiene
"""
size_gauss = valsmooth
img = file1
img_zerod = img.copy()
# img_zerod[np.isnan(img)] = 0
# It is a 9x9 array
kernel = Gaussian2DKernel(size_gauss) # x_stddev=1)
file1 = scipy_convolve(img, kernel, mode='same', method='direct')
return file1
def to_new_fwhm_box1d(ll, ss, width, nsamp=3):
""" Use box kernel to convolve down and resample spectrum """
dl = np.median(np.diff(ll))
nres = width / dl # The number of pixels in ll that constitute a slit
# The number of pixels in ll that make one pixel in the observed spectrum
npix = np.int(np.round(nres / nsamp))
print("Width of {0:1.4f} is {1:1.2f} pix".format(width, nres))
kernel = CC.Box1DKernel(np.round(nres))
sp = scipy_convolve(ss, kernel, mode='same', method='direct')
return ll, sp, npix
def psf_match(filename, fileout, ker):
hdulist = fits.open(filename)
data = hdulist[0].data
hdu_ker = fits.open(ker)
ker_data = hdu_ker[0].data
ker_shift = np.pad(ker_data, ((0, 1), (0, 1)), mode='constant')
data_out = scipy_convolve(data, ker_shift, mode='same', method='fft') # convolve
hdulist[0].data = data_out
fits.writeto('test.fits', data=data_out - data, overwrite=True)
hdulist.writeto(fileout, overwrite=True, output_verify="ignore")
def test_convolve(self):
sample_rate = 16000
file_path = TEST_FIXTURES_DIR / "acoustic_guitar_0.wav"
samples, _ = librosa.load(file_path, sr=sample_rate)
ir_samples, _ = librosa.load(TEST_FIXTURES_DIR / "ir" /
"impulse_response_0.wav",
sr=sample_rate)
expected_output = scipy_convolve(samples, ir_samples)
actual_output = torch_convolve(torch.from_numpy(samples),
torch.from_numpy(ir_samples)).numpy()
assert_almost_equal(actual_output, expected_output, decimal=6)
def psf_match(filename, fileout, ker):
hdulist = fits.open(filename)
data = hdulist[0].data
hdu_ker = fits.open(ker)
ker_data = hdu_ker[0].data
ker_shift = np.pad(ker_data, ((0, 1), (0, 1)), mode='constant')
# dat1 = convolve(dat, ker1, boundary='extend')
data_out = scipy_convolve(data, ker_shift, mode='same')
hdulist[0].data = data_out
hdulist.writeto(fileout, overwrite=True, output_verify="ignore")
def to_observed(ll, ss, width, N=125 * 1200, A_B=3e-9, nsamp=3):
l0 = np.median(ll).to(AA)
dl, GRF, kernel = create_GRF(l0, N, A_B)
_, sp, npix = to_new_fwhm_box1d(ll, ss, width, nsamp=nsamp)
spl = scipy_convolve(sp, kernel, mode='same', method='direct')
fun = II(ll.to(AA).value, spl)
actual_ll = np.arange(ll[0].to(AA).value, ll[-1].to(AA).value,
width.to(AA).value / nsamp)
actual_sp = fun(actual_ll)
return actual_ll, sp[::npix], actual_sp, dl, GRF
def make_fake_noise_map(stackedmap):
mean, med, stdev = sigma_clipped_stats(stackedmap)
# find best factor
"""for j in range(18, 25):
fake_noise = np.random.normal(med, j*stdev, (300, 300))
kernel = Gaussian2DKernel(x_stddev=15/2.355)
scipy_conv = scipy_convolve(fake_noise, kernel, mode='same', method='direct')
print(np.std(scipy_conv)/stdev, j)
for j in range(1,10):
fake_noise = np.random.normal(med, j * 0.0001, (300, 300))
kernel = Gaussian2DKernel(x_stddev=15 / 2.355)
scipy_conv = scipy_convolve(fake_noise, kernel, mode='same', method='direct')
print(np.std(scipy_conv)/(j*0.0001))"""
fake_noise = np.random.normal(med, 22.5 * stdev, (len(stackedmap), len(stackedmap)))
kernel = Gaussian2DKernel(x_stddev=15 / 2.355)
scipy_conv = scipy_convolve(fake_noise, kernel, mode='same', method='direct')
return scipy_conv
def convolve(images, kernel, mode='same'):
"""It convolves the kernel on input images.
Arguments:
images {[numpy.array]} -- Images as numpy matrices.
kernel {numpy.array} -- A numpy array to convolute.
mode {string} -- A string indicating the mode to apply convolution.
The values accepted by scipy.signal.convolve() are valid.
Returns:
{[np.array]} -- Convoluted images.
"""
input_images = np.asarray(images).astype(np.int16)
convoluted_images = np.asarray(
[scipy_convolve(image, kernel, mode='same') for image in input_images])
convoluted_images[convoluted_images > 255] = 255
convoluted_images[convoluted_images < 0] = 0
return convoluted_images.astype(np.uint8)
def convolve_spike_population(self, t_start, t_finish, bin_size,
percentile):
'''Binarizes a population of spike times and then convolves the entire population with a
Gaussian kernel.
Parameters
----------
t_start: float
the starting time to bin over, in seconds
t_finish: float
the ending time to bin over, in seconds
bin_size : float
the length, in seconds, of each bin
percentile : float
to obtain the standard deviation of the Gaussian kernel, a percentile of the population
ISI is used.
Returns
-------
convolution : numpy array
num_neurons x num_convolutions array containing the convolution of each neuron in each row
'''
# binarize spike times
neuron_ids, binarized = self.binarize_spike_times_population(
t_start, t_finish, bin_size)
# choose sigma
isi = self.get_isi()
sigma = np.percentile(np.concatenate(isi.values()),
percentile) / np.sqrt(12)
# create gaussian kernel
kernel = self.create_gaussian_kernel(bin_size, sigma)
# perform the convolution
num_bins = binarized.shape[1]
convolution = np.zeros((self.num_neurons, num_bins + len(kernel) - 1))
for neuron_id in neuron_ids:
convolution[neuron_id, :] = scipy_convolve(kernel,
binarized[neuron_id, :])
return convolution
for iprog in range(5):
do_it = 1
if (i>0):
if (TIME[i-1,iprog]>10.0):
do_it = 0 # it was already taking too long
TIME[i, iprog] = 1e10
if (0): # we have problem on the Intel GPU -- i915 bug prevents long runs ??
if ((iprog==4)&(fwhm_pix>32)): do_it = False
if (do_it):
time.sleep(SLEEP)
t0 = time.time()
if (iprog==0):
A = convolve_map_fast(FITS[0].data.copy(), fwhm_pix, radius=1.7*fwhm_pix)
elif (iprog==1):
B = scipy_convolve(FITS[0].data.copy(), kernel, mode='same', method='direct')
elif (iprog==2):
C = astropy_convolve(FITS[0].data.copy(), kernel)
elif (iprog==3):
# ConvolveFitsPyCL(F, fwhm_rad, fwhm_orig=0.0, RinFWHM=1.7, GPU=0, platforms=[2,])
D = ConvolveFitsBeamPyCL(FITS, kernel, GPU=0)[0].data.copy()
elif (iprog==4): # on my system Intel GPU
# ConvolveFitsPyCL(F, fwhm_rad, fwhm_orig=0.0, RinFWHM=1.7, GPU=1, platforms=[0,])
E = ConvolveFitsBeamPyCL(FITS, kernel, GPU=1)[0].data.copy()
elif (iprog==5): # on my system NVidia GPU
# ConvolveFitsPyCL(F, fwhm_rad, fwhm_orig=0.0, RinFWHM=1.7, GPU=1, platforms=[1,])
F = ConvolveFitsBeamPyCL(FITS, kernel, GPU=1, platforms=[1,])[0].data.copy()
TIME[i, iprog] = time.time()-t0
if (i==-1):
clf()
subplot(221)
window = CosineBellWindow(alpha=0.35)
from scipy.signal import convolve as scipy_convolve
window = TopHatWindow(0.35)
dir1 = '/home/sourabh/ULIRG_package/data/OPTICAL_PSF/'
data_ref = fits.getdata(dir1 + 'f165psf.fits')
data_psf = fits.getdata(dir1 + 'PSF_775_gal4_rotate_cut.fits')
kernel = create_matching_kernel(data_psf, data_ref) # , window = window )
fits.writeto('ker.fits', data=kernel, overwrite=True)
plt.imshow(kernel, cmap='Greys_r', origin='lower')
filename = '/home/sourabh/ULIRG_package/data/IRASF10594+3818/gal1_HA.fits'
fileout = '/home/sourabh/ULIRG_package/data/IRASF10594+3818/gal1_HA_psfmatch.fits'
ker = 'ker.fits'
#ker_shift = np.pad(kernel, ((0, 1), (0, 1)), mode='constant')
data1 = scipy_convolve(data_psf, kernel, mode='same')
fits.writeto('test2.fits', data=data1, overwrite=True)
data3 = data1 - data_ref
fits.writeto('test3.fits', data=data3, overwrite=True)
def psf_match(filename, fileout, ker):
hdulist = fits.open(filename)
data = hdulist[0].data
hdu_ker = fits.open(ker)
ker_data = hdu_ker[0].data
ker_shift = np.pad(ker_data, ((0, 1), (0, 1)), mode='constant')
data_out = scipy_convolve(data, ker_shift, mode='same', method='fft') # convolve
print 'Done...!' nl = header1["NAXIS1"] map = cube[0, :, :] # Note the a convolution must be performed in surface brightness units # We first define the convolution kernel. # We will use a gaussian kernel for simplicity, but in reality a proper # PSF kernel should be used (See, e.g., Aniano+ 2011) # the sigma of the gaussian (PSF) should be defined in pixels pixsz1 = np.abs(header1["CDELT1"]) * 3600. pixsz2 = np.abs(header2["CDELT1"]) * 3600. # first of all, all maps must be in surface brightness units. print "I am converting the flux into surface brightness" flx_sb = map / (kk**2 * pixsz1**2) # converted in K/sr from K print "Building the gaussian simulating the PSF" # the following is the gaussian FWHM in pixels sigma = np.sqrt(psf2**2 - psf1**2) / pixsz1 #kernel = Gaussian2DKernel(stddev=sigma,x_size=xs,y_size=ys) kernel = Gaussian2DKernel(stddev=sigma, x_size=40, y_size=40) print "Convolving..." convolved = scipy_convolve(flx_sb, kernel, mode='same') #, method='direct') print "Done!" # now regrids to the new header flxmap = hcongrid(convolved, header1, header2) # Now converting from K/sr to K flx_reg = flxmap * kk**2 * pixsz2**2 outfile = 'convolved.fits' hdu_s = fits.PrimaryHDU(flx_reg, header=header2) hdu_s.writeto(outfile)
# scipy.convolve handle missing data, so we start by setting the
# brightest pixels to NaN to simulate a "saturated" data set
img[img > 2e1] = np.nan
# We also create a copy of the data and set those NaNs to zero. We'll
# use this for the scipy convolution
img_zerod = img.copy()
img_zerod[np.isnan(img)] = 0
# We smooth with a Gaussian kernel with stddev=1
# It is a 9x9 array
kernel = Gaussian2DKernel(stddev=1)
# Convolution: scipy's direct convolution mode spreads out NaNs (see
# panel 2 below)
scipy_conv = scipy_convolve(img, kernel, mode='same', method='direct')
# scipy's direct convolution mode run on the 'zero'd' image will not
# have NaNs, but will have some very low value zones where the NaNs were
# (see panel 3 below)
scipy_conv_zerod = scipy_convolve(img_zerod,
kernel,
mode='same',
method='direct')
# astropy's convolution replaces the NaN pixels with a kernel-weighted
# interpolation from their neighbors
astropy_conv = convolve(img, kernel)
# Now we do a bunch of plots. In the first two plots, the originally masked
# values are marked with red X's
def psf_match(params, params_gal, i, j):
"""Entry point for combining different extensions of FLT images
Args:
params: (dict) Dictionary of 'basic' section of config file (common for all ULIRGs)
params_gal: (dict) Dictionary of galaxy parameters from config file
i : (int) ULIRG number (It goes from 0 to 4)
j : (int) Filter number (It goes from 0 to 3) and correspond to (F125, F140, F150, F165)
"""
gal_name = params_gal['name']
primary_dir = params['data_dir'] + gal_name + '/'
UVPSF_DIR = params['uvpsf_dir']
if j < 4:
file_name = "%sgal%s_UV_%s_drz.fits" % (primary_dir + 'UV_DRZ/', i + 1,
a[j])
file_out = primary_dir + os.path.basename(file_name).replace(
"drz", "psfmatch")
hdulist = fits.open(file_name)
prihdr = hdulist[1].header # the primary HDU header
dat = hdulist[1].data
########## replacing nan with zeros #############
where_are_NaNs1 = np.isnan(dat)
dat[where_are_NaNs1] = 0.0
file_ker = fits.open('%sker%s_ref165.fits' %
(UVPSF_DIR, b[j].replace("F", "")))
ker = file_ker[0].data
ker1 = np.pad(ker, ((0, 1), (0, 1)), mode='constant')
# plt.figure()
# img = plt.imshow(ker1)
# plt.show()
print(
" i m running slowly. Takes 10 minutes for each galaxy filter = %s"
% (a[j]))
# dat1 = convolve(dat, ker1, boundary='extend')
dat1 = scipy_convolve(dat, ker1, mode='same')
# !!!!Remember no convolution for error !!!
hdulist[1].data = dat1
else: # gal1_HA_FR782N_NB_UV_align.fits
if j == 4:
file_name = "%sgal%s_HA_%s_UV_iraf_v2.fits" % (primary_dir, i + 1,
a[j])
file_out = file_name.replace("UV_iraf_v2", "psfmatch")
else:
file_name = "%sgal%s_HA_%s_UV_align_v2.fits" % (primary_dir, i + 1,
a[j])
file_out = file_name.replace("UV_align_v2", "psfmatch")
hdulist = fits.open(file_name)
dat = hdulist[0].data
file_ker = fits.open('%sker%s_rotate_ref165_gal%s.fits' %
(PSF_DIR, b[j], i + 1))
ker = file_ker[0].data
ker1 = np.pad(ker, ((0, 1), (0, 1)), mode='constant')
print(
" i m running slowly. Takes 10 minutes for each galaxy filter = %s"
% (a[j]))
# dat1 = convolve(dat, ker1, boundary='extend')
dat1 = scipy_convolve(dat, ker1, mode='same')
# !!!!Remember no convolution for error !!!
hdulist[0].data = dat1
'''
hdulist[0].header["CREATOR"] = "SC March 2nd 2018"
hdulist[0].header.insert("HISTORY", ("KERNEL", file_ker), after = True)# "kernel file use for convolution")
hdulist[0].header.insert("HISTORY", ("REFPSF", "F165"), after = True)# "reference PSF for psfmatching"
hdulist[0].header["COMMENTS"] = "PSF matched image for filter %s using scipy convolve with mode = 'same' "%(a[j], )
hdulist[0].header.insert("HISTORY", ("WARNING", "Dont't forget to pad kernel\
with zeros (np.pad(ker,((0,1),(0,1)),mode='constant')"), after = True)
'''
hdulist.writeto(file_out, overwrite=True, output_verify="ignore")
def _build_one_heat_map(self,
feature_df,
risk_min,
feature_min,
feature_max,
fine_tune=-1):
Logging().log("Processing Feature: " + feature_df.columns[1])
if fine_tune == -1:
try:
values = np.empty(len(feature_df))
values.fill(1)
# Assign X Y Z
X = feature_df.RISK.as_matrix()
Y = feature_df[feature_df.columns[1]].as_matrix()
Z = values
# create x-y points to be used in heatmap of identical size
risk_min = feature_df.RISK.min()
risk_max = 1
xi = np.linspace(risk_min, risk_max, self._grid_area)
yi = np.linspace(feature_min, feature_max, self._grid_area)
# Z is a matrix of x-y values interpolated (!)
zi = griddata((X, Y),
Z, (xi[None, :], yi[:, None]),
method=self._interpol_method)
zmin = 0
zmax = 1
zi[(zi < zmin) | (zi > zmax)] = None
# Convolve each point with a gaussian kernel giving the heat value at point xi,yi being Z
# Advantage: kee horizontal and vertical influence
grid_cur = np.nan_to_num(zi)
# Smooth with a Gaussian kernel
kernel = Gaussian2DKernel(stddev=self._std_gaus,
x_size=self._kernel_size,
y_size=self._kernel_size)
grad = scipy_convolve(grid_cur,
kernel,
mode='same',
method='direct')
# horizontal interpolation
for r in range(len(grad)):
# per dimension get first and last nonzero value
cur_line = grad[:, r]
nonzeros = numpy.where(cur_line > 0.0001)[0]
if list(nonzeros):
a = 4
# fill von 0 bis nonzeros[0]
v = numpy.average(cur_line[nonzeros[0]:(nonzeros[0] +
a)])
replacement = numpy.linspace(0, v, nonzeros[0] +
a)[:(nonzeros[0])]
grad[:len(replacement), r] = replacement
# fill von nonzeros[-1] bis len(grid)-1
v = numpy.average(cur_line[nonzeros[-1] -
a:(nonzeros[-1])])
replacement = numpy.linspace(
0, v,
len(cur_line) - nonzeros[-1])[::-1]
grad[nonzeros[-1]:, r] = replacement
# Store the model in memory
feature_name = feature_df.columns[1]
result = [
feature_name,
[
copy.deepcopy(np.absolute(grad)),
copy.deepcopy(xi),
copy.deepcopy(yi)
], grid_cur
]
except:
feature_name = feature_df.columns[1]
Logging().log(str(feature_df.columns[1]) + ": Feature skipped")
result = [feature_name, None, None]
else:
if fine_tune == 0:
feature_df = feature_df[feature_df["RISK"] <
0.5] # hier changed!!!!!!!!!!!!!
if fine_tune == 1:
feature_df = feature_df[feature_df["RISK"] > 0.25]
feature_df = feature_df[feature_df["RISK"] < 0.75]
if fine_tune == 2:
feature_df = feature_df[feature_df["RISK"] > 0.5]
try:
feature_df = self._remove_one_outlier(feature_df,
feature_df.columns[1])
values = np.empty(len(feature_df))
values.fill(1)
# Assign X Y Z
X = feature_df.RISK.as_matrix()
Y = feature_df[feature_df.columns[1]].as_matrix()
Z = values
risk_min = feature_df.RISK.min()
risk_max = 1
xi = np.linspace(risk_min, risk_max, self._grid_area)
yi = np.linspace(feature_min, feature_max, self._grid_area)
# Z is a matrix of x-y values interpolated (!)
zi = griddata((X, Y),
Z, (xi[None, :], yi[:, None]),
method=self._interpol_method)
zmin = 0
zmax = 1
zi[(zi < zmin) | (zi > zmax)] = None
# Convolve each point with a gaussian kernel giving the heat value at point xi,yi being Z
# Advantage: kee horizontal and vertical influence
grid_cur = np.nan_to_num(zi)
# Smooth with a Gaussian kernel
kernel = Gaussian2DKernel(stddev=self._std_gaus,
x_size=self._kernel_size,
y_size=self._kernel_size)
grad = scipy_convolve(grid_cur,
kernel,
mode='same',
method='direct')
# vertikale interpolation bis an Rand
for r in range(len(grad)):
# per dimension get first and last nonzero value
cur_line = grad[:, r]
nonzeros = numpy.where(cur_line > 0.0001)[0]
if list(nonzeros):
a = 4
# fill von 0 bis nonzeros[0]
v = numpy.average(cur_line[nonzeros[0]:(nonzeros[0] +
a)])
replacement = numpy.linspace(0, v, nonzeros[0] +
a)[:(nonzeros[0])]
grad[:len(replacement), r] = replacement
# fill von nonzeros[-1] bis len(grid)-1
v = numpy.average(cur_line[nonzeros[-1] -
a:(nonzeros[-1])])
replacement = numpy.linspace(
0, v,
len(cur_line) - nonzeros[-1])[::-1]
grad[nonzeros[-1]:, r] = replacement
# Store the model in memory
feature_name = feature_df.columns[1]
result = [
"fine_" + str(fine_tune) + "_" + feature_name,
[
copy.deepcopy(np.absolute(grad)),
copy.deepcopy(xi),
copy.deepcopy(yi)
], grid_cur
]
except:
#traceback.print_exc()
feature_name = feature_df.columns[1]
result = [feature_name, None, None]
return result
def _build_heat_map(self, output_dfs):
''' using convolution for each point of a 2d array risk vs. feature value per feature
a heat map is generated
:param output_dfs: list of dataframes with each having a column scaled_FEATURE_X (that is outlierfree and scaled now) and a column
risk which is the risk for that feature at its row
:return a dictionary is returned that contains the feature name as key and its 2d heatmap as output
'''
dimensions = {}
for feature_df in output_dfs: # each output_df has one risk and value
Logging().log("Processing Feature: " + feature_df.columns[1])
# Testmode
if self._test_mode and (feature_df.columns[1]
== "scaled_FEATURE_5"):
print("Testing thus, break now!")
break
try:
values = np.empty(len(feature_df))
values.fill(1)
# Assign X Y Z
X = feature_df.RISK.as_matrix()
Y = feature_df[feature_df.columns[1]].as_matrix()
Z = values
# create x-y points to be used in heatmap of identical size
risk_min = 0
risk_max = 1
feature_min = min([
rm
for rm in [df[df.columns[1]].min() for df in output_dfs]
if not math.isnan(rm)
])
feature_max = max([
rm
for rm in [df[df.columns[1]].max() for df in output_dfs]
if not math.isnan(rm)
])
xi = np.linspace(risk_min, risk_max, self._grid_area)
yi = np.linspace(feature_min, feature_max, self._grid_area)
# Z is a matrix of x-y values interpolated (!)
zi = griddata((X, Y),
Z, (xi[None, :], yi[:, None]),
method=self._interpol_method)
zmin = 0
zmax = 1
zi[(zi < zmin) | (zi > zmax)] = None
# Convolve each point with a gaussian kernel giving the heat value at point xi,yi being Z
# Advantage: kee horizontal and vertical influence
grid_cur = np.nan_to_num(zi)
# Smooth with a Gaussian kernel
kernel = Gaussian2DKernel(stddev=self._std_gaus,
x_size=self._kernel_size,
y_size=self._kernel_size)
grad = scipy_convolve(grid_cur,
kernel,
mode='same',
method='direct')
# no constant/zero values shall be allowed -> first - horizontal
# horizontal interpolation bis an Rand
Logging.log("I AM NEW")
for r in range(len(grad)):
# per dimension get first and last nonzero value
cur_line = grad[:, r]
nonzeros = numpy.where(cur_line > 0.0001)[0]
if list(nonzeros):
a = 20
# fill von 0 bis nonzeros[0]
v = numpy.average(cur_line[nonzeros[0]:(nonzeros[0] +
a)])
replacement = numpy.linspace(0, v, nonzeros[0] +
a)[:(nonzeros[0])]
grad[:len(replacement), r] = replacement
# fill von nonzeros[-1] bis len(grid)-1
v = numpy.average(cur_line[nonzeros[-1] -
a:(nonzeros[-1])])
replacement = numpy.linspace(
0, v,
len(cur_line) - nonzeros[-1])[::-1]
grad[nonzeros[-1]:, r] = replacement
# vertikale interpolation bis an Rand
for r in range(len(grad)):
# per dimension get first and last nonzero value
cur_line = grad[r, :]
nonzeros = numpy.where(cur_line > 0.0001)[0]
if list(nonzeros):
a = 20
# fill von 0 bis nonzeros[0]
v = numpy.average(cur_line[nonzeros[0]:(nonzeros[0] +
a)])
replacement = numpy.linspace(0, v, nonzeros[0] +
a)[:(nonzeros[0])]
grad[r, :len(replacement)] = replacement
# fill von nonzeros[-1] bis len(grid)-1
v = numpy.average(cur_line[nonzeros[-1] -
a:(nonzeros[-1])])
replacement = numpy.linspace(
0, v,
len(cur_line) - nonzeros[-1] + 1)[::-1]
grad[r, nonzeros[-1] - 1:] = replacement
# Store the model in memory
dimensions[feature_df.columns[1]] = [
copy.deepcopy(np.absolute(grad)),
copy.deepcopy(xi),
copy.deepcopy(yi)
]
if self._visualize_heatmap:
fig, (ax_orig, ax_mag) = plt.subplots(1, 2)
ax_orig.imshow(grid_cur[::-1, ::-1], cmap='RdYlGn')
ax_orig.set_title('Original')
ax_mag.imshow(
np.absolute(grad)[::-1, ::-1], cmap='RdYlGn'
) # https://matplotlib.org/examples/color/colormaps_reference.html
ax_mag.set_title('Heat')
fig.show()
plt.show()
except:
Logging().log("No chance")
#traceback.print_exc()
dimensions[feature_df.columns[1]] = None
return dimensions, xi
def _build_one_heat_map(self, feature_df, risk_min, feature_min,
feature_max):
Logging().log("Processing Feature: " + feature_df.columns[1])
try:
values = np.empty(len(feature_df))
values.fill(1)
# Assign X Y Z
X = feature_df.RISK.as_matrix()
Y = feature_df[feature_df.columns[1]].as_matrix()
Z = values
# create x-y points to be used in heatmap of identical size
#risk_min = min([rm for rm in [df.RISK.min() for df in output_dfs] if not math.isnan(rm)])
risk_max = 1
xi = np.linspace(risk_min, risk_max, self._grid_area)
yi = np.linspace(feature_min, feature_max, self._grid_area)
# Z is a matrix of x-y values interpolated (!)
zi = griddata((X, Y),
Z, (xi[None, :], yi[:, None]),
method=self._interpol_method)
zmin = 0
zmax = 1
zi[(zi < zmin) | (zi > zmax)] = None
# Convolve each point with a gaussian kernel giving the heat value at point xi,yi being Z
# Advantage: kee horizontal and vertical influence
grid_cur = np.nan_to_num(zi)
# Smooth with a Gaussian kernel
kernel = Gaussian2DKernel(stddev=self._std_gaus,
x_size=self._kernel_size,
y_size=self._kernel_size)
grad = scipy_convolve(grid_cur,
kernel,
mode='same',
method='direct')
# Store the model in memory
feature_name = feature_df.columns[1]
result = [
feature_name,
[
copy.deepcopy(np.absolute(grad)),
copy.deepcopy(xi),
copy.deepcopy(yi)
], grid_cur
]
except:
#traceback.print_exc()
feature_name = feature_df.columns[1]
Logging().log(
str(feature_df.columns[1]) +
": No heat map created due to error")
result = [feature_name, None, None]
return result
execution_times = {
# tuples of (description, batch size)
("scipy direct", 1): [],
("scipy FFT", 1): [],
("torch FFT CPU", 1): [],
("torch FFT CPU", num_examples): [],
("torch FFT CUDA", 1): [],
("torch FFT CUDA", num_examples): [],
}
for i in tqdm(range(num_examples),
desc="scipy, method='direct', batch size=1"):
with timer("scipy direct") as t:
expected_output = scipy_convolve(samples,
ir_samples,
method="direct")
execution_times[(t.description, 1)].append(t.execution_time)
for i in tqdm(range(num_examples),
desc="scipy, method='fft', batch size=1"):
with timer("scipy FFT") as t:
expected_output = scipy_convolve(samples, ir_samples, method="fft")
execution_times[(t.description, 1)].append(t.execution_time)
pytorch_samples_cpu = torch.from_numpy(samples)
pytorch_ir_samples_cpu = torch.from_numpy(ir_samples)
for i in tqdm(range(num_examples), desc="torch FFT CPU, batch size=1"):
with timer("torch FFT CPU") as t:
_ = torch_convolve(pytorch_samples_cpu,
