def _bin_sn_template(self, ageIdx):
# Undo continuum in the following step in preprocessing.py
if self.snidTemplate:
wave, flux = self.wave, self.fluxes[
ageIdx] # self.snReadSpectrumFile.snid_template_undo_processing(self.snWave, self.snFluxes[ageIdx], self.splineInfo, ageIdx)
binnedWave, binnedFlux, minIndex, maxIndex = self.preProcess.log_wavelength(
wave, flux)
binnedFluxNorm = normalise_spectrum(binnedFlux)
binnedFluxNorm = zero_non_overlap_part(binnedFluxNorm,
minIndex,
maxIndex,
outerVal=0.5)
else:
wave, flux = self.wave, medfilt(self.flux, kernel_size=3)
flux = normalise_spectrum(flux)
binnedWave, binnedFlux, minIndex, maxIndex = self.preProcess.log_wavelength(
wave, flux)
contRemovedFlux, continuum = self.preProcess.continuum_removal(
binnedWave, binnedFlux, self.numSplinePoints, minIndex,
maxIndex)
meanzero = self.preProcess.mean_zero(contRemovedFlux, minIndex,
maxIndex)
apodized = self.preProcess.apodize(meanzero, minIndex, maxIndex)
binnedFluxNorm = normalise_spectrum(apodized)
binnedFluxNorm = zero_non_overlap_part(binnedFluxNorm,
minIndex,
maxIndex,
outerVal=0.5)
return binnedWave, binnedFluxNorm, minIndex, maxIndex
def overlapped_spectra(self):
snWave, snFlux, snMinIndex, snMaxIndex = self.snInfo
galWave, galFlux, galMinIndex, galMaxIndex = self.galInfo
minIndex = max(snMinIndex, galMinIndex)
maxIndex = min(snMaxIndex, galMaxIndex)
snFlux = zero_non_overlap_part(snFlux, minIndex, maxIndex)
galFlux = zero_non_overlap_part(galFlux, minIndex, maxIndex)
return snWave, snFlux, galWave, galFlux, minIndex, maxIndex
def apodize(self, flux, minindex, maxindex, outerVal=0):
"""apodize with 5% cosine bell"""
percent = 0.05
fluxout = np.copy(flux) - outerVal
nsquash = int(self.nw * percent)
for i in range(0, nsquash):
arg = np.pi * i / (nsquash - 1)
factor = 0.5 * (1 - np.cos(arg))
if (minindex + i < self.nw) and (maxindex - i >= 0):
fluxout[minindex + i] = factor * fluxout[minindex + i]
fluxout[maxindex - i] = factor * fluxout[maxindex - i]
else:
print("INVALID FLUX IN PREPROCESSING.PY APODIZE()")
print("MININDEX=%d, i=%d" % (minindex, i))
break
if outerVal != 0:
fluxout = fluxout + outerVal
fluxout = zero_non_overlap_part(fluxout,
minindex,
maxindex,
outerVal=outerVal)
return fluxout
def template_data(self, snCoeff, galCoeff, z):
wave, flux, minIndex, maxIndex = self.sn_plus_gal(snCoeff, galCoeff)
wave, flux = self.processingTools.redshift_spectrum(wave, flux, z)
flux = zero_non_overlap_part(flux, minIndex, maxIndex, outerVal=0)
binnedWave, binnedFlux, minIndex, maxIndex = self.preProcess.log_wavelength(
wave[minIndex:maxIndex + 1], flux[minIndex:maxIndex + 1])
newFlux, continuum = self.preProcess.continuum_removal(
binnedWave, binnedFlux, self.numSplinePoints, minIndex, maxIndex)
meanZero = self.preProcess.mean_zero(newFlux, minIndex, maxIndex)
apodized = self.preProcess.apodize(meanZero, minIndex, maxIndex)
fluxNorm = normalise_spectrum(apodized)
fluxNorm = zero_non_overlap_part(fluxNorm,
minIndex,
maxIndex,
outerVal=0.5)
# Could median filter here, but trying without it now
return binnedWave, fluxNorm, (minIndex, maxIndex)
def continuum_removal(self, wave, flux, numSplinePoints, minIndex, maxIndex):
flux = flux + 1 # Important to keep this as +1
contRemovedFlux = np.copy(flux)
splineFit = self.spline_fit(wave, flux, numSplinePoints, minIndex, maxIndex)
contRemovedFlux[minIndex:maxIndex + 1] = flux[minIndex:maxIndex + 1] / splineFit[minIndex:maxIndex + 1]
contRemovedFluxNorm = normalise_spectrum(contRemovedFlux - 1)
contRemovedFluxNorm = zero_non_overlap_part(contRemovedFluxNorm, minIndex, maxIndex)
return contRemovedFluxNorm, splineFit - 1
def augment_data(self, flux, stdDevMean=0.05, stdDevStdDev=0.05):
minIndex, maxIndex = ProcessingTools().min_max_index(flux, outerVal=0.5)
noise = np.zeros(self.nw)
stdDev = abs(np.random.normal(stdDevMean, stdDevStdDev)) # randomised standard deviation
noise[minIndex:maxIndex] = np.random.normal(0, stdDev, maxIndex - minIndex)
# # Add white noise to regions outside minIndex to maxIndex
# noise[0:minIndex] = np.random.uniform(0.0, 1.0, minIndex)
# noise[maxIndex:] = np.random.uniform(0.0, 1.0, self.nw-maxIndex)
augmentedFlux = flux + noise
augmentedFlux = normalise_spectrum(augmentedFlux)
augmentedFlux = zero_non_overlap_part(augmentedFlux, minIndex, maxIndex, outerVal=0.5)
return augmentedFlux
def _bin_gal_template(self):
wave, flux = self.readSpectrumFile.two_col_input_spectrum(self.wave,
self.flux,
z=0)
# flux = normalise_spectrum(flux)
binnedWave, binnedFlux, minIndex, maxIndex = self.preProcess.log_wavelength(
wave, flux)
contRemovedFlux, continuum = self.preProcess.continuum_removal(
binnedWave, binnedFlux, self.numSplinePoints, minIndex, maxIndex)
newFlux = contRemovedFlux * continuum # Spectral features weighted by the continuum
fluxNorm = normalise_spectrum(newFlux)
fluxNorm = zero_non_overlap_part(fluxNorm,
minIndex,
maxIndex,
outerVal=0.5)
return binnedWave, fluxNorm, minIndex, maxIndex
if __name__ == '__main__':
# Plot comparison of superfit galaxies and Bsnip galaxies
from astrodash.array_tools import normalise_spectrum
import matplotlib.pyplot as plt
preProcess = PreProcessSpectrum(3500, 10000, 1024)
for gal in ['E', 'S0', 'Sa', 'Sb', 'Sc', 'SB1', 'SB2', 'SB3', 'SB4', 'SB5', 'SB6']:
sfReadSpectrum = ReadSpectrumFile('../templates/superfit_templates/gal/%s' % gal, 3500, 10000, 1024)
sfWave, sfFlux = sfReadSpectrum.file_extension()
sfWave, sfFlux = sfReadSpectrum.two_col_input_spectrum(sfWave, sfFlux, z=0)
# plt.plot(sfWave, sfFlux)
sfWave, sfFlux, sfMinIndex, sfMaxIndex = preProcess.log_wavelength(sfWave, sfFlux)
plt.plot(sfWave, normalise_spectrum(sfFlux))
sfFlux, continuum = preProcess.continuum_removal(sfWave, sfFlux, 13, sfMinIndex, sfMaxIndex)
# plt.plot(sfWave, continuum-1)
snidReadSpectrum = ReadSpectrumFile('../templates/bsnip_v7_snid_templates/kc%s.lnw' % gal, 3500, 10000, 1024)
snidSpectrum = snidReadSpectrum.file_extension()
snidWave, snidFluxes, ncols, ages, ttype, splineInfo = snidSpectrum
snidWave, snidFlux, snidMinIndex, snidMaxIndex = preProcess.log_wavelength(snidWave, snidFluxes[0])
snidFlux = normalise_spectrum(snidFlux)
fluxNorm = zero_non_overlap_part(snidFlux, snidMinIndex, snidMaxIndex, outerVal=0.5)
plt.title(gal)
plt.plot(sfWave, sfFlux, label='superfit')
# plt.plot(snidWave, snidFlux, label='BSNIP')
plt.plot(sfWave, normalise_spectrum(sfFlux * (continuum - 1)), label='superfit_continuumMultiply')
plt.legend()
plt.show()
def two_column_data(self, z, smooth, minWave, maxWave):
if self.redshiftFromFile is True:
self.wave, self.flux, z = self.spectrum
else:
self.wave, self.flux = self.spectrum
self.flux = normalise_spectrum(self.flux)
self.flux = limit_wavelength_range(self.wave, self.flux, minWave, maxWave)
self.wDensity = (self.w1 - self.w0)/self.nw # Average wavelength spacing
wavelengthDensity = (max(self.wave) - min(self.wave)) / len(self.wave)
filterSize = int(self.wDensity / wavelengthDensity * smooth / 2) * 2 + 1
preFiltered = medfilt(self.flux, kernel_size=filterSize)
wave, flux = self.readSpectrumFile.two_col_input_spectrum(self.wave, preFiltered, z)
if len(wave) < 2:
sys.exit("The redshifted spectrum of file: {0} is out of the classification range between {1} to {2} "
"Angstroms. Please remove this file from classification or reduce the redshift before re-running "
"the program.".format(self.filename, self.w0, self.w1))
binnedwave, binnedflux, minIndex, maxIndex = self.preProcess.log_wavelength(wave, flux)
newflux, continuum = self.preProcess.continuum_removal(binnedwave, binnedflux, self.numSplinePoints, minIndex, maxIndex)
meanzero = self.preProcess.mean_zero(newflux, minIndex, maxIndex)
apodized = self.preProcess.apodize(meanzero, minIndex, maxIndex)
#filterSize = smooth * 2 + 1
medianFiltered = medfilt(apodized, kernel_size=1)#filterSize)
fluxNorm = normalise_spectrum(medianFiltered)
fluxNorm = zero_non_overlap_part(fluxNorm, minIndex, maxIndex, outerVal=0.5)
# # PAPER PLOTS
# import matplotlib.pyplot as plt
#
# plt.figure(num=1, figsize=(10, 6))
# plt.plot(self.wave, self.flux, label='Raw', linewidth=1.3)
# plt.plot(self.wave, preFiltered, label='Filtered', linewidth=1.3)
# plt.ylim(-8, 8)
# plt.xlabel('Wavelength ($\mathrm{\AA}$)', fontsize=13)
# plt.ylabel('Relative Flux', fontsize=13)
# plt.legend(fontsize=11, loc=1)
# plt.xlim(2500, 9000)
# plt.tight_layout()
# plt.axhline(0, color='black', linewidth=0.5)
# plt.savefig('/Users/danmuth/OneDrive/Documents/DASH/Paper/Figures/Filtering.png')
#
# plt.figure(num=2, figsize=(10, 6))
# plt.plot(self.wave, preFiltered, label='Filtered', linewidth=1.3)
# plt.plot(binnedwave, binnedflux, label='De-redshifted and log-wavelength binned', linewidth=1.3)
# plt.xlabel('Wavelength ($\mathrm{\AA}$)', fontsize=13)
# plt.ylabel('Relative Flux', fontsize=13)
# plt.legend(fontsize=11, loc=1)
# plt.xlim(2500, 9000)
# plt.tight_layout()
# plt.axhline(0, color='black', linewidth=0.5)
# plt.savefig('/Users/danmuth/OneDrive/Documents/DASH/Paper/Figures/Deredshifting.png')
#
# plt.figure(num=3, figsize=(10, 6))
# plt.plot(binnedwave, binnedflux, label='Log-wavelength binned', linewidth=1.3)
# plt.plot(binnedwave, continuum, label='Continuum', linewidth=1.3)
# plt.plot(binnedwave, newflux, label='Continuum subtracted', linewidth=1.3)
# plt.xlabel('Wavelength ($\mathrm{\AA}$)', fontsize=13)
# plt.ylabel('Relative Flux', fontsize=13)
# plt.legend(fontsize=11, loc=1)
# plt.xlim(2500, 9000)
# plt.tight_layout()
# plt.axhline(0, color='black', linewidth=0.5)
# plt.savefig('/Users/danmuth/OneDrive/Documents/DASH/Paper/Figures/Continuum.png')
#
# plt.figure(num=4, figsize=(10, 6))
# plt.plot(binnedwave, newflux, label='Continuum subtracted', linewidth=1.3)
# plt.plot(binnedwave, apodized, label='Apodized', linewidth=1.3)
# fluxNorm = (apodized - min(apodized)) / (max(apodized) - min(apodized))
# plt.plot(binnedwave, fluxNorm, label='Normalised', linewidth=1.3)
# plt.xlabel('Wavelength ($\mathrm{\AA}$)', fontsize=13)
# plt.ylabel('Relative Flux', fontsize=13)
# plt.legend(fontsize=11, loc=1)
# plt.xlim(2500, 9000)
# plt.tight_layout()
# plt.axhline(0, color='black', linewidth=0.5)
# plt.savefig('/Users/danmuth/OneDrive/Documents/DASH/Paper/Figures/Apodize.png')
#
# plt.show()
return binnedwave, fluxNorm, minIndex, maxIndex, z
# fData = '/Users/danmuth/PycharmProjects/DASH/templates/OzDES_data/ATEL_9570_Run25/DES16C2ma_C2_combined_160926_v10_b00.dat'
# preData = PreProcessing(fData, 2500, 10000, 1024)
# waveData,fluxData,minIData,maxIData = preData.two_column_data(0.24, 5, 2500, 10000)