def _bin_sn_template(self, ageIdx):
# Undo continuum in the following step in preprocessing.py
if self.snidTemplate:
wave, flux = self.wave, self.fluxes[
ageIdx] # self.snReadSpectrumFile.snid_template_undo_processing(self.snWave, self.snFluxes[ageIdx], self.splineInfo, ageIdx)
binnedWave, binnedFlux, minIndex, maxIndex = self.preProcess.log_wavelength(
wave, flux)
binnedFluxNorm = normalise_spectrum(binnedFlux)
binnedFluxNorm = zero_non_overlap_part(binnedFluxNorm,
minIndex,
maxIndex,
outerVal=0.5)
else:
wave, flux = self.wave, medfilt(self.flux, kernel_size=3)
flux = normalise_spectrum(flux)
binnedWave, binnedFlux, minIndex, maxIndex = self.preProcess.log_wavelength(
wave, flux)
contRemovedFlux, continuum = self.preProcess.continuum_removal(
binnedWave, binnedFlux, self.numSplinePoints, minIndex,
maxIndex)
meanzero = self.preProcess.mean_zero(contRemovedFlux, minIndex,
maxIndex)
apodized = self.preProcess.apodize(meanzero, minIndex, maxIndex)
binnedFluxNorm = normalise_spectrum(apodized)
binnedFluxNorm = zero_non_overlap_part(binnedFluxNorm,
minIndex,
maxIndex,
outerVal=0.5)
return binnedWave, binnedFluxNorm, minIndex, maxIndex
def _bin_gal_template(self):
wave, flux = self.readSpectrumFile.two_col_input_spectrum(self.wave, self.flux, z=0)
flux = normalise_spectrum(flux)
binnedWave, binnedFlux, minIndex, maxIndex = self.preProcess.log_wavelength(wave, flux)
contRemovedFlux, continuum = self.preProcess.continuum_removal(binnedWave, binnedFlux, self.numSplinePoints, minIndex, maxIndex)
newFlux = contRemovedFlux * continuum # Spectral features weighted by the continuum
fluxNorm = normalise_spectrum(newFlux)
fluxNorm = zero_non_overlap_part(fluxNorm, minIndex, maxIndex, outerVal=0.5)
return binnedWave, fluxNorm, minIndex, maxIndex
def continuum_removal(self, wave, flux, numSplinePoints, minIndex, maxIndex):
flux = flux + 1 # Important to keep this as +1
contRemovedFlux = np.copy(flux)
splineFit = self.spline_fit(wave, flux, numSplinePoints, minIndex, maxIndex)
contRemovedFlux[minIndex:maxIndex + 1] = flux[minIndex:maxIndex + 1] / splineFit[minIndex:maxIndex + 1]
contRemovedFluxNorm = normalise_spectrum(contRemovedFlux - 1)
contRemovedFluxNorm = zero_non_overlap_part(contRemovedFluxNorm, minIndex, maxIndex)
return contRemovedFluxNorm, splineFit - 1
def template_data(self, snCoeff, galCoeff, z):
wave, flux, minIndex, maxIndex = self.sn_plus_gal(snCoeff, galCoeff)
wave, flux = self.processingTools.redshift_spectrum(wave, flux, z)
flux = zero_non_overlap_part(flux, minIndex, maxIndex, outerVal=0)
binnedWave, binnedFlux, minIndex, maxIndex = self.preProcess.log_wavelength(wave[minIndex:maxIndex+1], flux[minIndex:maxIndex+1])
newFlux, continuum = self.preProcess.continuum_removal(binnedWave, binnedFlux, self.numSplinePoints, minIndex, maxIndex)
meanZero = self.preProcess.mean_zero(newFlux, minIndex, maxIndex)
apodized = self.preProcess.apodize(meanZero, minIndex, maxIndex)
fluxNorm = normalise_spectrum(apodized)
fluxNorm = zero_non_overlap_part(fluxNorm, minIndex, maxIndex, outerVal=0.5)
# Could median filter here, but trying without it now
return binnedWave, fluxNorm, (minIndex, maxIndex)
def augment_data(self, flux, stdDevMean=0.05, stdDevStdDev=0.05):
minIndex, maxIndex = ProcessingTools().min_max_index(flux, outerVal=0.5)
noise = np.zeros(self.nw)
stdDev = abs(np.random.normal(stdDevMean, stdDevStdDev)) # randomised standard deviation
noise[minIndex:maxIndex] = np.random.normal(0, stdDev, maxIndex - minIndex)
# # Add white noise to regions outside minIndex to maxIndex
# noise[0:minIndex] = np.random.uniform(0.0, 1.0, minIndex)
# noise[maxIndex:] = np.random.uniform(0.0, 1.0, self.nw-maxIndex)
augmentedFlux = flux + noise
augmentedFlux = normalise_spectrum(augmentedFlux)
augmentedFlux = zero_non_overlap_part(augmentedFlux, minIndex, maxIndex, outerVal=0.5)
return augmentedFlux
def redshifting(self):
images = np.empty((0, int(self.nw)), np.float16) # Number of pixels
labels = np.empty((0, self.nLabels),
np.uint16) # Number of labels (SN types)
filenames = []
typeNames = []
redshifts = []
minMaxIndexes = []
readSpectra = ReadSpectra(self.w0, self.w1, self.nw, self.filename)
#Undo it's previous redshift)
wave, flux, minIndex, maxIndex, z = readSpectra.input_spectrum(
self.z, self.smooth, self.minWave, self.maxWave)
nonzeroflux = flux[minIndex:maxIndex + 1]
newflux = normalise_spectrum(nonzeroflux)
newflux2 = np.concatenate(
(flux[0:minIndex], newflux, flux[maxIndex + 1:]))
images = np.append(images, np.array([newflux2]),
axis=0) # images.append(newflux2)
filenames.append(str(self.filename) + "_" + str(-z))
redshifts.append(-z)
minMaxIndexes.append((minIndex, maxIndex))
# # Add white noise to regions outside minIndex to maxIndex
# noise = np.zeros(self.nw)
# noise[0:minIndex] = np.random.uniform(0.0, 1.0, minIndex)
# noise[maxIndex:] = np.random.uniform(0.0, 1.0, self.nw - maxIndex)
#
# augmentedFlux = flux + noise
# augmentedFlux = normalise_spectrum(augmentedFlux)
# augmentedFlux = zero_non_overlap_part(augmentedFlux, minIndex, maxIndex)
inputImages = np.array(images)
inputFilenames = np.array(filenames)
inputRedshifts = np.array(redshifts)
return inputImages, inputFilenames, inputRedshifts, self.typeNamesList, minMaxIndexes
preProcess = PreProcessSpectrum(3500, 10000, 1024)
for gal in [
'E', 'S0', 'Sa', 'Sb', 'Sc', 'SB1', 'SB2', 'SB3', 'SB4', 'SB5',
'SB6'
]:
sfReadSpectrum = ReadSpectrumFile(
'../templates/superfit_templates/gal/%s' % gal, 3500, 10000, 1024)
sfWave, sfFlux = sfReadSpectrum.file_extension()
sfWave, sfFlux = sfReadSpectrum.two_col_input_spectrum(sfWave,
sfFlux,
z=0)
# plt.plot(sfWave, sfFlux)
sfWave, sfFlux, sfMinIndex, sfMaxIndex = preProcess.log_wavelength(
sfWave, sfFlux)
plt.plot(sfWave, normalise_spectrum(sfFlux))
sfFlux, continuum = preProcess.continuum_removal(
sfWave, sfFlux, 13, sfMinIndex, sfMaxIndex)
# plt.plot(sfWave, continuum-1)
snidReadSpectrum = ReadSpectrumFile(
'../templates/bsnip_v7_snid_templates/kc%s.lnw' % gal, 3500, 10000,
1024)
snidSpectrum = snidReadSpectrum.file_extension()
snidWave, snidFluxes, ncols, ages, ttype, splineInfo = snidSpectrum
snidWave, snidFlux, snidMinIndex, snidMaxIndex = preProcess.log_wavelength(
snidWave, snidFluxes[0])
snidFlux = normalise_spectrum(snidFlux)
fluxNorm = zero_non_overlap_part(snidFlux,
snidMinIndex,
snidMaxIndex,
def two_column_data(self, z, smooth, minWave, maxWave):
if self.redshiftFromFile is True:
self.wave, self.flux, z = self.spectrum
else:
self.wave, self.flux = self.spectrum
self.flux = normalise_spectrum(self.flux)
self.flux = limit_wavelength_range(self.wave, self.flux, minWave,
maxWave)
self.wDensity = (self.w1 -
self.w0) / self.nw # Average wavelength spacing
wavelengthDensity = (max(self.wave) - min(self.wave)) / len(self.wave)
filterSize = int(
self.wDensity / wavelengthDensity * smooth / 2) * 2 + 1
preFiltered = medfilt(self.flux, kernel_size=filterSize)
wave, flux = self.readSpectrumFile.two_col_input_spectrum(
self.wave, preFiltered, z)
if len(wave) < 2:
sys.exit(
"The redshifted spectrum of file: {0} is out of the classification range between {1} to {2} "
"Angstroms. Please remove this file from classification or reduce the redshift before re-running "
"the program.".format(self.filename, self.w0, self.w1))
binnedwave, binnedflux, minIndex, maxIndex = self.preProcess.log_wavelength(
wave, flux)
newflux, continuum = self.preProcess.continuum_removal(
binnedwave, binnedflux, self.numSplinePoints, minIndex, maxIndex)
meanzero = self.preProcess.mean_zero(newflux, minIndex, maxIndex)
apodized = self.preProcess.apodize(meanzero, minIndex, maxIndex)
#filterSize = smooth * 2 + 1
medianFiltered = medfilt(apodized, kernel_size=1) #filterSize)
fluxNorm = normalise_spectrum(medianFiltered)
fluxNorm = zero_non_overlap_part(fluxNorm,
minIndex,
maxIndex,
outerVal=0.5)
# # PAPER PLOTS
# import matplotlib.pyplot as plt
#
# plt.figure(num=1, figsize=(10, 6))
# plt.plot(self.wave, self.flux, label='Raw', linewidth=1.3)
# plt.plot(self.wave, preFiltered, label='Filtered', linewidth=1.3)
# plt.ylim(-8, 8)
# plt.xlabel('Wavelength ($\mathrm{\AA}$)', fontsize=13)
# plt.ylabel('Relative Flux', fontsize=13)
# plt.legend(fontsize=11, loc=1)
# plt.xlim(2500, 9000)
# plt.tight_layout()
# plt.axhline(0, color='black', linewidth=0.5)
# plt.savefig('/Users/danmuth/OneDrive/Documents/DASH/Paper/Figures/Filtering.png')
#
# plt.figure(num=2, figsize=(10, 6))
# plt.plot(self.wave, preFiltered, label='Filtered', linewidth=1.3)
# plt.plot(binnedwave, binnedflux, label='De-redshifted and log-wavelength binned', linewidth=1.3)
# plt.xlabel('Wavelength ($\mathrm{\AA}$)', fontsize=13)
# plt.ylabel('Relative Flux', fontsize=13)
# plt.legend(fontsize=11, loc=1)
# plt.xlim(2500, 9000)
# plt.tight_layout()
# plt.axhline(0, color='black', linewidth=0.5)
# plt.savefig('/Users/danmuth/OneDrive/Documents/DASH/Paper/Figures/Deredshifting.png')
#
# plt.figure(num=3, figsize=(10, 6))
# plt.plot(binnedwave, binnedflux, label='Log-wavelength binned', linewidth=1.3)
# plt.plot(binnedwave, continuum, label='Continuum', linewidth=1.3)
# plt.plot(binnedwave, newflux, label='Continuum subtracted', linewidth=1.3)
# plt.xlabel('Wavelength ($\mathrm{\AA}$)', fontsize=13)
# plt.ylabel('Relative Flux', fontsize=13)
# plt.legend(fontsize=11, loc=1)
# plt.xlim(2500, 9000)
# plt.tight_layout()
# plt.axhline(0, color='black', linewidth=0.5)
# plt.savefig('/Users/danmuth/OneDrive/Documents/DASH/Paper/Figures/Continuum.png')
#
# plt.figure(num=4, figsize=(10, 6))
# plt.plot(binnedwave, newflux, label='Continuum subtracted', linewidth=1.3)
# plt.plot(binnedwave, apodized, label='Apodized', linewidth=1.3)
# fluxNorm = (apodized - min(apodized)) / (max(apodized) - min(apodized))
# plt.plot(binnedwave, fluxNorm, label='Normalised', linewidth=1.3)
# plt.xlabel('Wavelength ($\mathrm{\AA}$)', fontsize=13)
# plt.ylabel('Relative Flux', fontsize=13)
# plt.legend(fontsize=11, loc=1)
# plt.xlim(2500, 9000)
# plt.tight_layout()
# plt.axhline(0, color='black', linewidth=0.5)
# plt.savefig('/Users/danmuth/OneDrive/Documents/DASH/Paper/Figures/Apodize.png')
#
# plt.show()
return binnedwave, fluxNorm, minIndex, maxIndex, z
# fData = '/Users/danmuth/PycharmProjects/DASH/templates/OzDES_data/ATEL_9570_Run25/DES16C2ma_C2_combined_160926_v10_b00.dat'
# preData = PreProcessing(fData, 2500, 10000, 1024)
# waveData,fluxData,minIData,maxIData = preData.two_column_data(0.24, 5, 2500, 10000)
if __name__ == '__main__':
# Plot comparison of superfit galaxies and Bsnip galaxies
from dash.array_tools import normalise_spectrum
import matplotlib.pyplot as plt
preProcess = PreProcessSpectrum(3500, 10000, 1024)
for gal in ['E', 'S0', 'Sa', 'Sb', 'Sc', 'SB1', 'SB2', 'SB3', 'SB4', 'SB5', 'SB6']:
sfReadSpectrum = ReadSpectrumFile('../templates/superfit_templates/gal/%s' % gal, 3500, 10000, 1024)
sfWave, sfFlux = sfReadSpectrum.file_extension()
sfWave, sfFlux = sfReadSpectrum.two_col_input_spectrum(sfWave, sfFlux, z=0)
# plt.plot(sfWave, sfFlux)
sfWave, sfFlux, sfMinIndex, sfMaxIndex = preProcess.log_wavelength(sfWave, sfFlux)
plt.plot(sfWave, normalise_spectrum(sfFlux))
sfFlux, continuum = preProcess.continuum_removal(sfWave, sfFlux, 13, sfMinIndex, sfMaxIndex)
# plt.plot(sfWave, continuum-1)
snidReadSpectrum = ReadSpectrumFile('../templates/bsnip_v7_snid_templates/kc%s.lnw' % gal, 3500, 10000, 1024)
snidSpectrum = snidReadSpectrum.file_extension()
snidWave, snidFluxes, ncols, ages, ttype, splineInfo = snidSpectrum
snidWave, snidFlux, snidMinIndex, snidMaxIndex = preProcess.log_wavelength(snidWave, snidFluxes[0])
snidFlux = normalise_spectrum(snidFlux)
fluxNorm = zero_non_overlap_part(snidFlux, snidMinIndex, snidMaxIndex, outerVal=0.5)
plt.title(gal)
plt.plot(sfWave, sfFlux, label='superfit')
# plt.plot(snidWave, snidFlux, label='BSNIP')
plt.plot(sfWave, normalise_spectrum(sfFlux * (continuum - 1)), label='superfit_continuumMultiply')
plt.legend()