def EW(lamb,flux,name):
flux = flux * u.Unit('J cm-2 s-1 AA-1')
#flux = flux * u.Unit('erg cm-2 s-1 AA-1')
lamb= lamb * u.AA
spec = Spectrum1D(spectral_axis=lamb, flux=flux)
#cont_norm_spec = spec / fit_generic_continuum(spec)(spec.spectral_axis)
cont_norm_spec = spec
print('-----------'+name+'------------')
#line A
#EWa = equivalent_width(cont_norm_spec, regions=SpectralRegion(8493*u.AA, 8502*u.AA))
#FWHMa = fwhm(cont_norm_spec, regions=SpectralRegion(8493*u.AA, 8502*u.AA))
#print('EW A line: '+str(EWa))
#line B
EWb = equivalent_width(cont_norm_spec, regions=SpectralRegion(lineBlims[0]*u.AA, lineBlims[1]*u.AA))
print('EW B line: '+str(EWb))
#line C
EWc = equivalent_width(cont_norm_spec, regions=SpectralRegion(lineClims[0]*u.AA, lineClims[1]*u.AA))
print('EW C line: '+str(EWc))
#open log file
#nonlinear to metal-poor
V_VHB = -2.0
EWbc= (EWb+EWc)
EWbc= float(EWbc/(1. * u.AA))
EWp = (EWbc)**(-1.5)
#nonlinear to metal-poor
#Wl = float(EWb / (1. * u.AA)) + float(EWc / (1. * u.AA)) + (0.64 * V_VHB)
#FeH= -2.81 + 0.44*Wl
# FeH constants to V-VHB
a=-2.87
b=0.195
c=0.458
d=-0.913
e=0.0155
#float all
FeH = a + b * V_VHB + c * EWbc + d * EWp + e * EWbc * V_VHB
print('[Fe/H]: '+str(FeH))
return [EWb,EWc,FeH]
def test_statistics_gui_full_spectrum(specviz_gui):
# Ensure that the test is run on an unmodified workspace instance
workspace = new_workspace(specviz_gui)
hub = Hub(workspace=workspace)
# pull out stats dictionary
stats_dict = specviz_gui.current_workspace._plugin_bars['Statistics'].stats
# Generate truth comparisons
spectrum = hub.plot_item._data_item.spectrum
truth_dict = {
'mean': spectrum.flux.mean(),
'median': np.median(spectrum.flux),
'stddev': spectrum.flux.std(),
'centroid': centroid(spectrum, region=None),
'snr': "N/A",
'fwhm': fwhm(spectrum),
'ew': equivalent_width(spectrum),
'total': line_flux(spectrum),
'maxval': spectrum.flux.max(),
'minval': spectrum.flux.min()
}
# compare!
assert stats_dict == truth_dict
workspace.close()
def test_statistics_gui_roi_spectrum(specviz_gui):
# Ensure that the test is run on an unmodified workspace instance
workspace = new_workspace(specviz_gui)
hub = Hub(workspace=workspace)
# Make region of interest cutout, using default cutout at .3 from the
# middle in either direction
specviz_gui.current_workspace.current_plot_window.plot_widget._on_add_linear_region(
)
# Simulate cutout for truth data
spectrum = extract_region(hub.plot_item._data_item.spectrum,
SpectralRegion(*hub.selected_region_bounds))
# pull out stats dictionary
stats_dict = specviz_gui.current_workspace._plugin_bars['Statistics'].stats
# Generate truth comparisons
truth_dict = {
'mean': spectrum.flux.mean(),
'median': np.median(spectrum.flux),
'stddev': spectrum.flux.std(),
'centroid': centroid(spectrum, region=None),
'snr': "N/A",
'fwhm': fwhm(spectrum),
'ew': equivalent_width(spectrum),
'total': line_flux(spectrum),
'maxval': spectrum.flux.max(),
'minval': spectrum.flux.min()
}
# compare!
assert stats_dict == truth_dict
workspace.close()
def compute_stats(spectrum):
"""
Compute basic statistics for a spectral region.
Parameters
----------
spectrum : `~specutils.spectra.spectrum1d.Spectrum1D`
region: `~specutils.utils.SpectralRegion`
"""
flux = spectrum.flux
mean = flux.mean()
rms = np.sqrt(flux.dot(flux) / len(flux))
try:
snr_val = snr(spectrum)
except Exception as e:
snr_val = "N/A"
return {
'mean': mean,
'median': np.median(flux),
'stddev': flux.std(),
'centroid':
centroid(spectrum, region=None
), # we may want to adjust this for continuum subtraction
'rms': rms,
'snr': snr_val,
'fwhm': fwhm(spectrum),
'ew': equivalent_width(spectrum),
'total': line_flux(spectrum),
'maxval': flux.max(),
'minval': flux.min()
}
def test_statistics_gui_roi_spectrum(specviz_gui):
# Ensure that the test is run on an unmodified workspace instance
workspace = new_workspace(specviz_gui)
hub = Hub(workspace=workspace)
# Make region of interest cutout, using default cutout at .3 from the
# middle in either direction
specviz_gui.current_workspace.current_plot_window.plot_widget._on_add_linear_region()
# Simulate cutout for truth data
spectrum = extract_region(hub.plot_item._data_item.spectrum,
SpectralRegion(*hub.selected_region_bounds))
# pull out stats dictionary
stats_dict = specviz_gui.current_workspace._plugin_bars['Statistics'].stats
# Generate truth comparisons
truth_dict = {'mean': spectrum.flux.mean(),
'median': np.median(spectrum.flux),
'stddev': spectrum.flux.std(),
'centroid': centroid(spectrum, region=None),
'snr': "N/A",
'fwhm': fwhm(spectrum),
'ew': equivalent_width(spectrum),
'total': line_flux(spectrum),
'maxval': spectrum.flux.max(),
'minval': spectrum.flux.min()}
# compare!
assert stats_dict == truth_dict
workspace.close()
def test_statistics_gui_full_spectrum(specviz_gui):
# Ensure that the test is run on an unmodified workspace instance
workspace = new_workspace(specviz_gui)
hub = Hub(workspace=workspace)
# pull out stats dictionary
stats_dict = specviz_gui.current_workspace._plugin_bars['Statistics'].stats
# Generate truth comparisons
spectrum = hub.plot_item._data_item.spectrum
truth_dict = {'mean': spectrum.flux.mean(),
'median': np.median(spectrum.flux),
'stddev': spectrum.flux.std(),
'centroid': centroid(spectrum, region=None),
'snr': "N/A",
'fwhm': fwhm(spectrum),
'ew': equivalent_width(spectrum),
'total': line_flux(spectrum),
'maxval': spectrum.flux.max(),
'minval': spectrum.flux.min()}
# compare!
assert stats_dict == truth_dict
workspace.close()
def measure_ew_emission_line(model,
emline,
wave_margin=300,
redshift=0.0,
cont_low=5,
cont_upp=3,
cont_degree=2):
"""Measure the EW of an emission line after normalization."""
# Decide the wavelength range
wave_flag = ((model['wave'] >= emline['cen'] - wave_margin) &
(model['wave'] <= emline['cen'] + wave_margin))
wave_use = model['wave'][wave_flag]
flux_em = model['spec_em'][wave_flag]
flux_ne = model['spec_ne'][wave_flag]
# Normalize the spectrum, so the continuum level is 1.0
flux_em_norm = sigma_clipping_continuum(wave_use,
flux_em,
low=5,
upp=2,
degree=cont_degree)
flux_ne_norm = sigma_clipping_continuum(wave_use,
flux_ne,
low=2,
upp=5,
degree=cont_degree)
# Form a Spectrum1D object
ew_em = equivalent_width(
Spectrum1D(spectral_axis=wave_use * u.AA,
flux=flux_em_norm * u.Unit('erg cm-2 s-1 AA-1')),
regions=SpectralRegion(emline['low'] * u.AA * (1.0 + redshift),
emline['upp'] * u.AA * (1.0 + redshift)),
continuum=1).value
ew_ne = equivalent_width(
Spectrum1D(spectral_axis=wave_use * u.AA,
flux=flux_ne_norm * u.Unit('erg cm-2 s-1 AA-1')),
regions=SpectralRegion(emline['low'] * u.AA * (1.0 + redshift),
emline['upp'] * u.AA * (1.0 + redshift)),
continuum=1).value
return ew_ne - ew_em
def compute_stats(spectrum):
"""
Compute basic statistics for a spectral region.
Parameters
----------
spectrum : `~specutils.spectra.spectrum1d.Spectrum1D`
region: `~specutils.utils.SpectralRegion`
"""
try:
cent = centroid(spectrum, region=None) # we may want to adjust this for continuum subtraction
except Exception as e:
logging.debug(e)
cent = "Error"
try:
snr_val = snr(spectrum)
except Exception as e:
logging.debug(e)
snr_val = "N/A"
try:
fwhm_val = fwhm(spectrum)
except Exception as e:
logging.debug(e)
fwhm_val = "Error"
try:
ew = equivalent_width(spectrum)
except Exception as e:
logging.debug(e)
ew = "Error"
try:
total = line_flux(spectrum)
except Exception as e:
logging.debug(e)
total = "Error"
return {'mean': spectrum.flux.mean(),
'median': np.median(spectrum.flux),
'stddev': spectrum.flux.std(),
'centroid': cent,
'snr': snr_val,
'fwhm': fwhm_val,
'ew': ew,
'total': total,
'maxval': spectrum.flux.max(),
'minval': spectrum.flux.min()}
def _calc_EW(self, wavelengths, flux, continuum):
"""
Generic function to calculate (using SpecUtils) the Equivalent width of a line
requires: Wavelengths (assumed to be in nm)
Flux (assumed to be in photons)
Continuum (assumed to be in photons)
The equivalent width will be measured on the normalized spectrum (flux/continuum)
output: Equivalent_Width (Quantity)
"""
norm_spec = (flux / continuum) * u.photon
norm_spec_wave = wavelengths * u.nanometer
not_nan = ~np.isnan(norm_spec)
spec = Spectrum1D(spectral_axis=norm_spec_wave[not_nan][::-1],
flux=norm_spec[not_nan][::-1])
spec_region = SpectralRegion(
np.min(wavelengths) * u.nm,
np.max(wavelengths) * u.nm)
return equivalent_width(spec, regions=spec_region)
ew_list = []
cont_lst = []
##iterate through each observation to calculate Intensity.
for cont, spec in zip(cont_list, data):
ew_iter = []
cont_iter = []
for inp, region in zip(input_list, region_list):
cont_iter.append(
extract_region(cont,
SpectralRegion(region[0] * u.AA,
region[1] * u.AA)).flux)
ew_iter.append(
equivalent_width(spec,
regions=SpectralRegion(
int(inp.split(' ')[0]) * u.AA,
int(inp.split(' ')[1]) * u.AA),
continuum=extract_region(
cont,
SpectralRegion(region[0] * u.AA,
region[1] * u.AA)).flux))
ew_list.append(ew_iter)
cont_lst.append(cont_iter)
ew1 = []
ew2 = []
ew3 = []
ew4 = []
for i in ew_list:
ew1.append(np.mean(np.array(i[0])))
ew2.append(np.mean(np.array(i[1])))
ew3.append(np.mean(np.array(i[2])))
ew4.append(np.mean(np.array(i[3])))
y_cn = y_model / y_cont
"""dcontplot = DataPlot()
dcont = Data1D("normalized", mplot.x, y_cn)
dcontplot.prepare(dcont)
os.chdir(r"/home/dtyler/Desktop/DocumentsDT/Programs/images/images_"+fits_image_filename)
plt.savefig(fits_image_filename[:-4]+"_normalized_to_continuum.png")
os.chdir(r"/home/dtyler/Desktop/DocumentsDT/Programs")"""
const_y = np.ones(len(mplot2.x))
constant_cont = [mplot2.x, const_y]
###################################
### !!! calculate equivalent width
spectrum = Spectrum1D(flux=y_cn * u.dimensionless_unscaled,
spectral_axis=mplot.x * u.AA)
eqw = equivalent_width(spectrum)
"""print("EW", eqw.value)
print('norm factor', norm_factor)
for par in bmdl.pars:
print(par.fullname, par.val)"""
### !!! print output to file
os.chdir(r"/home/dtyler/Desktop/DocumentsDT/outputs/images_" +
fits_image_filename)
fil = open(fits_image_filename[:-4] + ".txt", 'w')
fil.write('eqw' + '\t' + str(eqw.value) + '\n')
fil.write('norm' + '\t' + str(norm_factor) + '\n')
for par in bmdl.pars:
fil.write(par.fullname + '\t' + str(par.val) + '\n')
#plt.figure()
#plt.plot(heii_wave, heii_norm.flux)
center = 6565 #4685
lower = center - 10
upper = center + 10
plt.figure()
plt.plot(new_wave, new_flux, 'b')
plt.xlim(lower, upper)
peak = new_wave[(new_wave < upper) & (new_wave > lower)][np.argmax(
new_flux[(new_wave < upper) & (new_wave > lower)])]
print(
peak,
equivalent_width(new_spec,
regions=SpectralRegion(lower * u.AA,
upper * u.AA)))
#
# center = 4686
# lower = center - 10
# upper = center + 10
# plt.figure(gal)
# plt.plot(heii_wave,heii_norm.flux,'b')
# plt.xlim(lower,upper)
# #plt.ylim(0,3)
# print(heii_wave[np.argmax(heii_flux)],
# equivalent_width(heii_norm, regions=SpectralRegion(lower*u.AA, upper*u.AA)))
#from spectres import spectres
#x = spectres(wave, new_wave, flux)
def get_toi837_li_equivalent_width():
spectrum_path = '../data/spectra/TOI-837_FEROS.fits'
plotpath = '../results/TOI_837/feros_spectrum_get_li_equivalent_width.png'
#
# fit out the continuum to get the continuum normalized flux, over the
# window of 6670 angstrom to 6713 angstrom.
#
xlim = [6670, 6713]
hdul = fits.open(spectrum_path)
d = hdul[0].data
wav = d[0, 0]
flx = d[3, 0]
if isinstance(xlim, list):
xmin = xlim[0]
xmax = xlim[1]
sel = (wav > xmin) & (wav < xmax)
wav = wav[sel]
flx = flx[sel]
spec = Spectrum1D(spectral_axis=wav * u.AA,
flux=flx * u.dimensionless_unscaled)
if isinstance(xlim, list):
exclude_regions = []
if xmin < 6709.2 and xmax > 6710.2:
exclude_regions.append(SpectralRegion(6709.2 * u.AA,
6710.2 * u.AA))
if xmin < 6679 and xmax > 6681:
exclude_regions.append(SpectralRegion(6679 * u.AA, 6681 * u.AA))
cont_flx = (fit_generic_continuum(spec, exclude_regions=exclude_regions)(
spec.spectral_axis))
cont_norm_spec = spec / cont_flx
#
# to fit gaussians, look at 1-flux.
#
full_spec = Spectrum1D(spectral_axis=cont_norm_spec.wavelength,
flux=(1 - cont_norm_spec.flux))
#
# get the Li EW
#
region = SpectralRegion(6708.5 * u.AA, 6711.5 * u.AA)
li_equiv_width = equivalent_width(cont_norm_spec, regions=region)
li_centroid = centroid(full_spec, region)
#
# fit a gaussian too, and get ITS equiv width
# https://specutils.readthedocs.io/en/stable/fitting.html
#
g_init = models.Gaussian1D(amplitude=0.2 * u.dimensionless_unscaled,
mean=6709.7 * u.AA,
stddev=0.5 * u.AA)
g_fit = fit_lines(full_spec, g_init, window=(region.lower, region.upper))
y_fit = g_fit(full_spec.wavelength)
fitted_spec = Spectrum1D(spectral_axis=full_spec.wavelength,
flux=(1 - y_fit) * u.dimensionless_unscaled)
fitted_li_equiv_width = equivalent_width(fitted_spec, regions=region)
#
# print bestfit params
#
print(42 * '=')
print('got Li equiv width of {}'.format(li_equiv_width))
print('got fitted Li equiv width of {}'.format(fitted_li_equiv_width))
print('got Li centroid of {}'.format(li_centroid))
print('fit gaussian1d params are\n{}'.format(repr(g_fit)))
print(42 * '=')
#
# plot the results
#
f, axs = plt.subplots(nrows=4, ncols=1, figsize=(6, 8))
axs[0].plot(wav, flx, c='k', zorder=3)
axs[0].plot(wav, cont_flx, c='r', zorder=2)
axs[1].plot(cont_norm_spec.wavelength, cont_norm_spec.flux, c='k')
axs[2].plot(cont_norm_spec.wavelength, cont_norm_spec.flux, c='k')
axs[3].plot(full_spec.wavelength, full_spec.flux, c='k')
axs[3].plot(full_spec.wavelength, y_fit, c='g')
txt = ('gaussian1d\namplitude:{:.3f}\nmean:{:.3f}\nstd:{:.3f}\nEW:{:.3f}'.
format(g_fit.amplitude.value, g_fit.mean.value, g_fit.stddev.value,
fitted_li_equiv_width))
axs[3].text(0.95,
0.95,
txt,
ha='right',
va='top',
transform=axs[3].transAxes,
fontsize='xx-small')
axs[0].set_ylabel('flux')
axs[1].set_ylabel('contnorm flux')
axs[2].set_ylabel('contnorm flux [zoom]')
axs[3].set_ylabel('1 - (contnorm flux)')
if isinstance(xlim, list):
for ax in axs:
ax.set_xlim(xlim)
axs[2].set_xlim([6708.5, 6711.5])
axs[3].set_xlim([6708.5, 6711.5])
axs[-1].set_xlabel('wavelength [angstrom]')
for ax in axs:
format_ax(ax)
outpath = '../results/TOI_837/toi837_li_equivalent_width_routine.png'
savefig(f, outpath)
def EW(specname,name):
lamb, flux= np.genfromtxt(specname, skip_header=1, unpack=True)
flux = flux * u.Unit('J cm-2 s-1 AA-1')
#flux = flux * u.Unit('erg cm-2 s-1 AA-1')
lamb= lamb * u.AA
spec = Spectrum1D(spectral_axis=lamb, flux=flux)
# normalization is not so good
cont_norm_spec = spec / fit_generic_continuum(spec)(spec.spectral_axis)
print('-----------'+name+'------------')
#line A
EWa = equivalent_width(cont_norm_spec, regions=SpectralRegion(8493*u.AA, 8502*u.AA))
#FWHMa = fwhm(cont_norm_spec, regions=SpectralRegion(8493*u.AA, 8502*u.AA))
print('EW A line: '+str(EWa))
#line B
EWb = equivalent_width(cont_norm_spec, regions=SpectralRegion(8533*u.AA, 8551*u.AA))
print('EW B line: '+str(EWb))
#line C
EWc = equivalent_width(cont_norm_spec, regions=SpectralRegion(8655*u.AA, 8670*u.AA))
print('EW C line: '+str(EWc))
#open log file
#nonlinear to metal-poor
V_VHB = -2.0
EWbc= (EWb+EWc)
EWbc= float(EWbc/(1. * u.AA))
EWp = (EWbc)**(-1.5)
#nonlinear to metal-poor
#Wl = float(EWb / (1. * u.AA)) + float(EWc / (1. * u.AA)) + (0.64 * V_VHB)
#FeH= -2.81 + 0.44*Wl
# FeH constants to V-VHB
a=-2.87
b=0.195
c=0.458
d=-0.913
e=0.0155
#float all
FeH = a + b * V_VHB + c * EWbc + d * EWp + e * EWbc * V_VHB
print('[Fe/H]: '+str(FeH))
#change relampled spectrum to noise spectrum
LOG = open('./EWs/EWfile-'+name+'.txt', 'w')
#LOG = open('./EWs/EWfileRE-'+name+'.txt', 'w')
LOG.write('Log file of '+ name +' \n \n')
LOG.write('Input Spectrum: '+ specname +' \n \n')
LOG.write('EW A line: '+ str(EWa) +' \n')
LOG.write('EW B line: '+ str(EWb) +' \n')
LOG.write('EW C line: '+ str(EWc) +' \n')
LOG.write('[Fe/H]_CaT: '+ str(FeH) +' \n')
f1 = plt.figure(figsize=(16,9))
ax = f1.add_subplot(111)
ax.plot(cont_norm_spec.spectral_axis, cont_norm_spec.flux)
ax.set_xlim([8480,8690])
ax.set_ylabel('Flux (J cm-2 s-1 AA-1)')
ax.set_xlabel('Wavelength ( $\AA$ )')
ax.axvspan(8498-float(EWa / (2. * u.AA)) , 8498+float(EWa / (2. * u.AA)) , alpha=0.2, color='red')
ax.axvspan(8542-float(EWb / (2. * u.AA)) , 8542+float(EWb / (2. * u.AA)) , alpha=0.2, color='red')
ax.axvspan(8662-float(EWc / (2. * u.AA)) , 8662+float(EWc / (2. * u.AA)) , alpha=0.2, color='red')
#change relampled spectrum to noise spectrum
plt.savefig('./EWs/EW-figs/EW'+name+'.pdf')
print(f3[0], " ... ", f3[nfreq - 1])
# km/s
v3 = s3.velocity
print(v3[0], " ... ", v3[nfreq - 1])
# cm
l3 = s3.wavelength
print(l3[0], " ... ", l3[nfreq - 1])
# some EW work, make a EW box so it can be overplotted on spectrum
from specutils.analysis import equivalent_width
#s3.to_dispersion("km/s", rest = restfrq)
ew = equivalent_width(s3)
ipeak = s3.flux.argmax()
xpeak = v3[ipeak].value
ypeak = s3.flux[ipeak].value
dx = ew.value
print("PJT EW", ipeak, xpeak, ypeak, dx)
rect = [(xpeak - 0.5 * dx, 0.0), dx, ypeak]
# moments around the peak
m = 5
x = v3[ipeak - m:ipeak + m]
y = s3.flux[ipeak - m:ipeak + m]
xmean = (x * y).sum() / y.sum()
xdisp = (x * x * y).sum() / y.sum() - xmean * xmean
print("MOMENTS:", xmean, xdisp)
def Load_Files(file_1, file_2, N_sample, objts, classification=False):
print('INFO:')
#hdul = fitsio.FITS(file_1) # Open file 1 -- 'truth_DR12Q.fits'
#info=hdul.info() # File info
hdul = fits.open(file_1, mode='denywrite')
#data=hdul[1].read() # Database of spectra with human-expert classifications
data = hdul[1].data
#print('The file {} have {} objects. \n'.format(file_1,data.shape[0]))
print('INFO:')
# Reading data from data_dr12.fits. This file had the spectra from data dr12.
#hdul_2 = fitsio.FITS(file_2) # Open file 2 -- 'data_dr12.fits'
#info2=hdul_2.info() # File info
#data2=hdul_2[1].read() # Database of spectra
#spectra=hdul_2[0].read() # Spectrum of each object
hdul_2 = fits.open(file_2, mode='denywrite')
data2 = hdul_2[1].data # Database of spectra
spectra = hdul_2[0].data # Spectrum of each object
#print('The file {} have {} spectra. \n'.format(file_2,spectra.shape[0]))
# Subset of PLATE parameters of both data
data_PLATE_1 = data['PLATE']
data_PLATE_2 = data2['PLATE']
# Subset of MJD parameters of both data
data_MJD_1 = data['MJD']
data_MJD_2 = data2['MJD']
# Subset of FIBERID parameters of both data
data_FIBERID_1 = data['FIBERID']
data_FIBERID_2 = data2['FIBERID']
data_ID_1 = data['THING_ID']
data_ID_2 = data2['TARGETID']
objts = np.asarray(objts)
# The column 'CLASS_PERSON' have a class identifier for each spectrum: STARS=1, GALAXY=4, QSO=3 and QSO_BAL=30.
C_P = data['CLASS_PERSON'] #Class Person column
STAR = C_P[C_P == 1] # objects classified as stars
GALAXY = C_P[C_P == 4] # objects classified as galaxies
QSO = C_P[C_P == 3] # objects classified as QSO (Quasars)
QSO_BAL = C_P[
C_P ==
30] # objects classified as QSO BAL (Quasars with Broad Absortions Lines)
N_C = C_P[C_P != 30]
N_C = N_C[N_C != 3]
N_C = N_C[N_C != 1]
N_C = N_C[N_C != 4] # objects wrong classified
print('INFO: There is available')
print('-->Star:', STAR.shape[0])
print('-->Galaxy:', GALAXY.shape[0])
print('-->QSO:', QSO.shape[0])
print('-->QSO BAL:', QSO_BAL.shape[0])
print('-->NN: {}\n'.format(N_C.shape[0]))
# I create two DataFrame for Superset_DR12Q and data_dr12 with only three parameters
data1 = {
'PLATE': data_PLATE_1,
'MJD': data_MJD_1,
'FIBERID': data_FIBERID_1,
'ID': data_ID_1
}
data1 = pd.DataFrame(data=data1)
data2 = {
'PLATE': data_PLATE_2,
'MJD': data_MJD_2,
'FIBERID': data_FIBERID_2,
'ID': data_ID_2
}
data2 = pd.DataFrame(data=data2)
# I convert all objects in both set to string chain in orden to combine them as one new ID.
data1['PLATE'] = data1['PLATE'].astype(str)
data1['MJD'] = data1['MJD'].astype(str)
data1['FIBERID'] = data1['FIBERID'].astype(str)
data1['PM'] = data1['MJD'].str.cat(data1['FIBERID'], sep="-")
data1['NEWID'] = data1['PLATE'].str.cat(data1['PM'], sep="-")
data_1 = data1.drop(columns=['PLATE', 'MJD', 'FIBERID', 'ID', 'PM']).values
data2['PLATE'] = data2['PLATE'].astype(str)
data2['MJD'] = data2['MJD'].astype(str)
data2['FIBERID'] = data2['FIBERID'].astype(str)
data2['PM'] = data2['MJD'].str.cat(data2['FIBERID'], sep="-")
data2['NEWID'] = data2['PLATE'].str.cat(data2['PM'], sep="-")
data_2 = data2.drop(columns=['PLATE', 'MJD', 'FIBERID', 'ID', 'PM']
).values # New set of database 2 with new ID's
# With the routine of numpy intersect1d, I find the intersections elements in both sets. This elements
data_CO = np.array(np.intersect1d(data_1, data_2, return_indices=True))
data_CO_objects = data_CO[
0] # The unique new ID of each element in both sets
data_CO_ind1 = data_CO[
1] # Indices of intersected elements from the original data 1 (Superset_DR12Q.fits)
data_CO_ind2 = data_CO[
2] # Indices of intersected elements form the original data 2 (data_dr12.fits)
print('INFO:')
print('I find {} objects with spectra from DR12 \n'.format(
len(data_CO_objects)))
indi = {'ind1': data_CO_ind1, 'ind2': data_CO_ind2}
ind = pd.DataFrame(data=indi, index=data_CO_ind1)
cp = np.array(data['CLASS_PERSON'], dtype=float)
z = np.array(data['Z_VI'], dtype=float)
zc = np.array(data['Z_CONF_PERSON'], dtype=float)
bal = np.array(data['BAL_FLAG_VI'], dtype=float)
bi = np.array(data['BI_CIV'], dtype=float)
d = {
'CLASS_PERSON': cp,
'Z_VI': z,
'Z_CONF_PERSON': zc,
'BAL_FLAG_VI': bal,
'BI_CIV': bi
}
data_0 = pd.DataFrame(data=d)
obj = data_0.loc[data_CO_ind1]
if (classification != True):
if (objts[0] == 'QSO'):
qsos = obj.loc[obj['CLASS_PERSON'] == 3]
qsos = qsos.loc[qsos['Z_CONF_PERSON'] == 3]
sample_objects = qsos.sample(n=int(N_sample),
weights='CLASS_PERSON',
random_state=5)
indi = np.array(sample_objects.index)
indi1 = ind.loc[indi].values
elif (objts[0] == 'QSO_BAL'):
qsos_bal = obj.loc[obj['CLASS_PERSON'] == 30]
qsos_bal = qsos_bal.loc[qsos_bal['Z_CONF_PERSON'] == 3]
sample_objects = qsos_bal.sample(n=int(N_sample),
weights='CLASS_PERSON',
random_state=5)
indi = np.array(sample_objects.index)
indi1 = ind.loc[indi].values
elif (len(objts) == 2):
qsos = obj.loc[obj['CLASS_PERSON'] == 3]
qsos = qsos.loc[qsos['Z_CONF_PERSON'] == 3]
qsos_bal = obj.loc[obj['CLASS_PERSON'] == 30]
qsos_bal = qsos_bal.loc[qsos_bal['Z_CONF_PERSON'] == 3]
sample_qso = qsos.sample(n=int(N_sample / 2),
weights='CLASS_PERSON',
random_state=5)
sample_qso_bal = qsos_bal.sample(n=int(N_sample / 2),
weights='CLASS_PERSON',
random_state=5)
sample_objects = pd.concat([sample_qso, sample_qso_bal])
ind_qso = np.array(sample_qso.index)
ind_qso_bal = np.array(sample_qso_bal.index)
indi = np.concatenate((ind_qso, ind_qso_bal), axis=None)
indi1 = ind.loc[indi].values
spectra_ = np.zeros((N_sample, 886))
j = 0
kernel_size = 5
flux_threshold = 1.1
parameters = np.zeros(
(N_sample, 7)
) #Number of lines // FHWM of max emission line // EW of max emission line // Spectrum Mean // Spectrum STDV // Spectrum Flux Integral // Spectrum SNR
for i in indi:
k = indi1[j, 1]
x = np.linspace(3600, 10500, 443)
zero_spectrum = spectra[k, :443]
spectrum = Spectrum1D(flux=zero_spectrum * u.Jy,
spectral_axis=x * u.AA)
#Continuum fit and gaussian smooth
g1_fit = fit_generic_continuum(spectrum)
y_continuum_fitted = g1_fit(x * u.AA)
spec_nw_2 = spectrum / y_continuum_fitted
spectrum_smooth = gaussian_smooth(spec_nw_2, kernel_size)
#Number of lines
lines_1 = find_lines_derivative(spectrum_smooth,
flux_threshold=flux_threshold)
l = lines_1[lines_1['line_type'] == 'emission']
number_lines = l['line_center_index'].shape[0]
parameters[j, 0] = number_lines
#FWHM
parameters[j, 1] = fwhm(spectrum_smooth).value
#EW
parameters[j, 2] = equivalent_width(spectrum_smooth).value
#Spectrum Mean
parameters[j, 3] = np.mean(spectrum_smooth.flux)
#Spectrum STDV
parameters[j, 4] = np.std(spectrum_smooth.flux)
#Spectrum Flux Integral
parameters[j, 5] = line_flux(spectrum_smooth).value
#Spectrum SNR
parameters[j, 6] = snr_derived(spectrum_smooth).value
j += 1
d = {
'Lines_Number': parameters[:, 0],
'FHWM': parameters[:, 1],
'EW': parameters[:, 2],
'Mean': parameters[:, 3],
'STDV': parameters[:, 4],
'STDV': parameters[:, 4],
'Spectrum_Flux': parameters[:, 5],
'SNR': parameters[:, 6]
}
parameters = pd.DataFrame(data=d)
#X=spectra_.values
#mean_flx= np.ma.average(X[:,:443],axis=1)
#ll=(X[:,:443]-mean_flx.reshape(-1,1))**2
#aveflux=np.ma.average(ll, axis=1)
#sflux = np.sqrt(aveflux)
#X = (X[:,:443]-mean_flx.reshape(-1,1))/sflux.reshape(-1,1)
y = sample_objects['Z_VI']
y = np.array(y, dtype=float)
#y_max=np.max(y)
#y=y/y_max
return parameters, y
stars = obj.loc[obj['CLASS_PERSON'] == 1]
galaxies = obj.loc[obj['CLASS_PERSON'] == 4]
qsos = obj.loc[obj['CLASS_PERSON'] == 3]
qsos_bal = obj.loc[obj['CLASS_PERSON'] == 30]
sample_star = stars.sample(n=int(N_sample / 4),
weights='CLASS_PERSON',
random_state=5)
sample_galaxy = galaxies.sample(n=int(N_sample / 4),
weights='CLASS_PERSON',
random_state=5)
sample_qso = qsos.sample(n=int(N_sample / 4),
weights='CLASS_PERSON',
random_state=5)
sample_qso_bal = qsos_bal.sample(n=int(N_sample / 4),
weights='CLASS_PERSON',
random_state=5)
sample_objects = pd.concat(
[sample_star, sample_galaxy, sample_qso, sample_qso_bal])
ind_star = np.array(sample_star.index)
ind_galaxy = np.array(sample_galaxy.index)
ind_qso = np.array(sample_qso.index)
ind_qso_bal = np.array(sample_qso_bal.index)
indi = np.concatenate((ind_star, ind_galaxy, ind_qso, ind_qso_bal),
axis=None)
indi1 = ind.loc[indi].values
spectra_ = np.zeros((N_sample, 886))
j = 0
for i in indi:
k = indi1[j, 1]
spectra_[j, :] = spectra[k, :]
j = j + 1
spectra_ = pd.DataFrame(spectra_)
X = spectra_.values
#Renormalize spectra
mean_flx = np.ma.average(X[:, :443], axis=1)
ll = (X[:, :443] - mean_flx.reshape(-1, 1))**2
aveflux = np.ma.average(ll, axis=1)
sflux = np.sqrt(aveflux)
X = (X[:, :443] - mean_flx.reshape(-1, 1)) / sflux.reshape(-1, 1)
y = sample_objects['CLASS_PERSON']
y = y.replace([1, 4, 3, 30], [0, 1, 2, 3]).values
y = np.array(y, dtype=float)
return X, y
