def vue_spectral_smooth(self, *args, **kwargs):
# Testing inputs to make sure putting smoothed spectrum into
# datacollection works
# input_flux = Quantity(np.array([0.2, 0.3, 2.2, 0.3]), u.Jy)
# input_spaxis = Quantity(np.array([1, 2, 3, 4]), u.micron)
# spec1 = Spectrum1D(input_flux, spectral_axis=input_spaxis)
size = float(self.stddev)
try:
spec = self._selected_data.get_object(cls=Spectrum1D)
except TypeError:
snackbar_message = SnackbarMessage(
f"Unable to perform smoothing over selected data.",
color="error",
sender=self)
self.hub.broadcast(snackbar_message)
return
# Takes the user input from the dialog (stddev) and uses it to
# define a standard deviation for gaussian smoothing
spec_smoothed = gaussian_smooth(spec, stddev=size)
label = f"Smoothed {self._selected_data.label}"
self.data_collection[label] = spec_smoothed
snackbar_message = SnackbarMessage(
f"Data set '{self._selected_data.label}' smoothed successfully.",
color="success",
sender=self)
self.hub.broadcast(snackbar_message)
def vue_spectral_smooth(self, *args, **kwargs):
# Testing inputs to make sure putting smoothed spectrum into
# datacollection works
# input_flux = Quantity(np.array([0.2, 0.3, 2.2, 0.3]), u.Jy)
# input_spaxis = Quantity(np.array([1, 2, 3, 4]), u.micron)
# spec1 = Spectrum1D(input_flux, spectral_axis=input_spaxis)
size = float(self.stddev)
try:
spec = self._selected_data.get_object(cls=Spectrum1D,
statistic=None)
except TypeError:
snackbar_message = SnackbarMessage(
"Unable to perform smoothing over selected data.",
color="error",
sender=self)
self.hub.broadcast(snackbar_message)
return
# Takes the user input from the dialog (stddev) and uses it to
# define a standard deviation for gaussian smoothing
spec_smoothed = gaussian_smooth(spec, stddev=size)
label = f"Smoothed {self._selected_data.label} stddev {size}"
if label in self.data_collection:
snackbar_message = SnackbarMessage(
"Data with selected stddev already exists, canceling operation.",
color="error",
sender=self)
self.hub.broadcast(snackbar_message)
return
# add data to the collection
self.app.add_data(spec_smoothed, label)
if self.add_replace_results:
viewer = "spectrum-viewer" if self.selected_data_is_1d or self.app.config == 'cubeviz' else "spectrum-2d-viewer" # noqa
self.app.add_data_to_viewer(
viewer, label, clear_other_data=viewer != "spectrum-viewer")
if not self.selected_data_is_1d and self.selected_viewer != 'None':
# replace the contents in the selected viewer with the smoothed cube
self.app.add_data_to_viewer(self.viewer_to_id.get(
self.selected_viewer),
label,
clear_other_data=True)
snackbar_message = SnackbarMessage(
f"Data set '{self._selected_data.label}' smoothed successfully.",
color="success",
sender=self)
self.hub.broadcast(snackbar_message)
def vue_gaussian_smooth(self, *args, **kwargs):
# Testing inputs to make sure putting smoothed spectrum into
# datacollection works
# input_flux = Quantity(np.array([0.2, 0.3, 2.2, 0.3]), u.Jy)
# input_spaxis = Quantity(np.array([1, 2, 3, 4]), u.micron)
# spec1 = Spectrum1D(input_flux, spectral_axis=input_spaxis)
size = float(self.stddev)
spec = self._selected_data.get_object(cls=Spectrum1D)
# Takes the user input from the dialog (stddev) and uses it to
# define a standard deviation for gaussian smoothing
spec_smoothed = gaussian_smooth(spec, stddev=size)
self.data_collection[
f"Smoothed {self._selected_data.label}"] = spec_smoothed
self.dialog = False
def make_artificial_spectra(line_table, wavelength_range = None, dw = 0.1,
sigma=1e-2, wavelength_unit=u.angstrom, flux_unit=u.electron):
"""Given a table with identified spectral lines and their fluxes, create an artifical Spectrum1D
with each of the spectral features.
Parameters
----------
line_table: Astropy.Table
A table of lines in the observed arc. The table should have a column labeled
'wavelength' and a column labeled 'flux'.
wavelength_range: list
wavelenght range for the artificial spectrum. If None, then it will default to the
minimum and maximum in line_table.
dw: float
Sampling to use in the dispersion of the artificial spectrum
wavelength_units: Atropy.Units
Units of the wavelength array. If None, it will assume Angstroms.
flux_units: Astropy.Units
Units of the flux array.If None, it will assume electrons.
"""
if wavelength_range is None:
wavelength_range=[line_table['wavelength'].min(), line_table['wavelength'].max()]
warr = np.arange(wavelength_range[0], wavelength_range[1], dw)
flux = np.zeros(len(warr), dtype=float)
for w, f in line_table:
i = abs(warr - w).argmin()
flux[i] = f
if wavelength_unit is None:
wavelength
spec = Spectrum1D(spectral_axis=warr*wavelength_unit, flux = flux * flux_unit)
spec = gaussian_smooth(spec, stddev=sigma)
return spec
def vue_gaussian_smooth(self, *args, **kwargs):
# Testing inputs to make sure putting smoothed spectrum into
# datacollection works
# input_flux = Quantity(np.array([0.2, 0.3, 2.2, 0.3]), u.Jy)
# input_spaxis = Quantity(np.array([1, 2, 3, 4]), u.micron)
# spec1 = Spectrum1D(input_flux, spectral_axis=input_spaxis)
size = float(self.stddev)
try:
spec = self._selected_data.get_object(cls=Spectrum1D)
except TypeError:
snackbar_message = SnackbarMessage(
f"Unable to perform smoothing over selected data.",
color="error",
sender=self)
self.hub.broadcast(snackbar_message)
return
# Takes the user input from the dialog (stddev) and uses it to
# define a standard deviation for gaussian smoothing
spec_smoothed = gaussian_smooth(spec, stddev=size)
label = f"Smoothed {self._selected_data.label}"
self.data_collection[label] = spec_smoothed
# Link the new dataset pixel-wise to the original dataset. In general
# direct pixel to pixel links are the most efficient and should be
# used in cases like this where we know there is a 1-to-1 mapping of
# pixel coordinates. Here the smoothing returns a 1-d spectral object
# which we can link to the first dimension of the original dataset
# (whcih could in principle be a cube or a spectrum)
self.data_collection.add_link(
LinkSame(self._selected_data.pixel_component_ids[0],
self.data_collection[label].pixel_component_ids[0]))
snackbar_message = SnackbarMessage(
f"Data set '{self._selected_data.label}' smoothed successfully.",
color="success",
sender=self)
self.hub.broadcast(snackbar_message)
fwhm = 0.20
#SIGMA=8
SIGMA = fwhm / 2.35482 * 100
wl, fl = np.genfromtxt(INPUT_spec, skip_header=2, unpack=True)
wl2, fl2 = np.genfromtxt('fluxCS31_0.norm.nulbad.0.200',
skip_header=2,
unpack=True)
spec1 = Spectrum1D(spectral_axis=wl * u.A, flux=fl * u.Jy)
#spec1_tsmooth = trapezoid_smooth(spec1, width=3)
#spec1_bsmooth = box_smooth(spec1, width=3)
#spec1_msmooth = median_smooth(spec1, width=3)
spec1_gsmooth = gaussian_smooth(spec1, stddev=SIGMA)
g = Gaussian1DKernel(stddev=SIGMA)
# Convolve data
z = convolve(fl, g)
#plt.plot(spec1.spectral_axis, spec1.flux)
plt.plot(wl2, fl2)
plt.plot(wl, z)
#plt.plot(spec1_gsmooth.spectral_axis, spec1_gsmooth.flux)
#snr_threshold(spectrum, value[, op])
def Load_Files(file_1, file_2, N_sample, objts, classification=False):
print('INFO:')
#hdul = fitsio.FITS(file_1) # Open file 1 -- 'truth_DR12Q.fits'
#info=hdul.info() # File info
hdul = fits.open(file_1, mode='denywrite')
#data=hdul[1].read() # Database of spectra with human-expert classifications
data = hdul[1].data
#print('The file {} have {} objects. \n'.format(file_1,data.shape[0]))
print('INFO:')
# Reading data from data_dr12.fits. This file had the spectra from data dr12.
#hdul_2 = fitsio.FITS(file_2) # Open file 2 -- 'data_dr12.fits'
#info2=hdul_2.info() # File info
#data2=hdul_2[1].read() # Database of spectra
#spectra=hdul_2[0].read() # Spectrum of each object
hdul_2 = fits.open(file_2, mode='denywrite')
data2 = hdul_2[1].data # Database of spectra
spectra = hdul_2[0].data # Spectrum of each object
#print('The file {} have {} spectra. \n'.format(file_2,spectra.shape[0]))
# Subset of PLATE parameters of both data
data_PLATE_1 = data['PLATE']
data_PLATE_2 = data2['PLATE']
# Subset of MJD parameters of both data
data_MJD_1 = data['MJD']
data_MJD_2 = data2['MJD']
# Subset of FIBERID parameters of both data
data_FIBERID_1 = data['FIBERID']
data_FIBERID_2 = data2['FIBERID']
data_ID_1 = data['THING_ID']
data_ID_2 = data2['TARGETID']
objts = np.asarray(objts)
# The column 'CLASS_PERSON' have a class identifier for each spectrum: STARS=1, GALAXY=4, QSO=3 and QSO_BAL=30.
C_P = data['CLASS_PERSON'] #Class Person column
STAR = C_P[C_P == 1] # objects classified as stars
GALAXY = C_P[C_P == 4] # objects classified as galaxies
QSO = C_P[C_P == 3] # objects classified as QSO (Quasars)
QSO_BAL = C_P[
C_P ==
30] # objects classified as QSO BAL (Quasars with Broad Absortions Lines)
N_C = C_P[C_P != 30]
N_C = N_C[N_C != 3]
N_C = N_C[N_C != 1]
N_C = N_C[N_C != 4] # objects wrong classified
print('INFO: There is available')
print('-->Star:', STAR.shape[0])
print('-->Galaxy:', GALAXY.shape[0])
print('-->QSO:', QSO.shape[0])
print('-->QSO BAL:', QSO_BAL.shape[0])
print('-->NN: {}\n'.format(N_C.shape[0]))
# I create two DataFrame for Superset_DR12Q and data_dr12 with only three parameters
data1 = {
'PLATE': data_PLATE_1,
'MJD': data_MJD_1,
'FIBERID': data_FIBERID_1,
'ID': data_ID_1
}
data1 = pd.DataFrame(data=data1)
data2 = {
'PLATE': data_PLATE_2,
'MJD': data_MJD_2,
'FIBERID': data_FIBERID_2,
'ID': data_ID_2
}
data2 = pd.DataFrame(data=data2)
# I convert all objects in both set to string chain in orden to combine them as one new ID.
data1['PLATE'] = data1['PLATE'].astype(str)
data1['MJD'] = data1['MJD'].astype(str)
data1['FIBERID'] = data1['FIBERID'].astype(str)
data1['PM'] = data1['MJD'].str.cat(data1['FIBERID'], sep="-")
data1['NEWID'] = data1['PLATE'].str.cat(data1['PM'], sep="-")
data_1 = data1.drop(columns=['PLATE', 'MJD', 'FIBERID', 'ID', 'PM']).values
data2['PLATE'] = data2['PLATE'].astype(str)
data2['MJD'] = data2['MJD'].astype(str)
data2['FIBERID'] = data2['FIBERID'].astype(str)
data2['PM'] = data2['MJD'].str.cat(data2['FIBERID'], sep="-")
data2['NEWID'] = data2['PLATE'].str.cat(data2['PM'], sep="-")
data_2 = data2.drop(columns=['PLATE', 'MJD', 'FIBERID', 'ID', 'PM']
).values # New set of database 2 with new ID's
# With the routine of numpy intersect1d, I find the intersections elements in both sets. This elements
data_CO = np.array(np.intersect1d(data_1, data_2, return_indices=True))
data_CO_objects = data_CO[
0] # The unique new ID of each element in both sets
data_CO_ind1 = data_CO[
1] # Indices of intersected elements from the original data 1 (Superset_DR12Q.fits)
data_CO_ind2 = data_CO[
2] # Indices of intersected elements form the original data 2 (data_dr12.fits)
print('INFO:')
print('I find {} objects with spectra from DR12 \n'.format(
len(data_CO_objects)))
indi = {'ind1': data_CO_ind1, 'ind2': data_CO_ind2}
ind = pd.DataFrame(data=indi, index=data_CO_ind1)
cp = np.array(data['CLASS_PERSON'], dtype=float)
z = np.array(data['Z_VI'], dtype=float)
zc = np.array(data['Z_CONF_PERSON'], dtype=float)
bal = np.array(data['BAL_FLAG_VI'], dtype=float)
bi = np.array(data['BI_CIV'], dtype=float)
d = {
'CLASS_PERSON': cp,
'Z_VI': z,
'Z_CONF_PERSON': zc,
'BAL_FLAG_VI': bal,
'BI_CIV': bi
}
data_0 = pd.DataFrame(data=d)
obj = data_0.loc[data_CO_ind1]
if (classification != True):
if (objts[0] == 'QSO'):
qsos = obj.loc[obj['CLASS_PERSON'] == 3]
qsos = qsos.loc[qsos['Z_CONF_PERSON'] == 3]
sample_objects = qsos.sample(n=int(N_sample),
weights='CLASS_PERSON',
random_state=5)
indi = np.array(sample_objects.index)
indi1 = ind.loc[indi].values
elif (objts[0] == 'QSO_BAL'):
qsos_bal = obj.loc[obj['CLASS_PERSON'] == 30]
qsos_bal = qsos_bal.loc[qsos_bal['Z_CONF_PERSON'] == 3]
sample_objects = qsos_bal.sample(n=int(N_sample),
weights='CLASS_PERSON',
random_state=5)
indi = np.array(sample_objects.index)
indi1 = ind.loc[indi].values
elif (len(objts) == 2):
qsos = obj.loc[obj['CLASS_PERSON'] == 3]
qsos = qsos.loc[qsos['Z_CONF_PERSON'] == 3]
qsos_bal = obj.loc[obj['CLASS_PERSON'] == 30]
qsos_bal = qsos_bal.loc[qsos_bal['Z_CONF_PERSON'] == 3]
sample_qso = qsos.sample(n=int(N_sample / 2),
weights='CLASS_PERSON',
random_state=5)
sample_qso_bal = qsos_bal.sample(n=int(N_sample / 2),
weights='CLASS_PERSON',
random_state=5)
sample_objects = pd.concat([sample_qso, sample_qso_bal])
ind_qso = np.array(sample_qso.index)
ind_qso_bal = np.array(sample_qso_bal.index)
indi = np.concatenate((ind_qso, ind_qso_bal), axis=None)
indi1 = ind.loc[indi].values
spectra_ = np.zeros((N_sample, 886))
j = 0
kernel_size = 5
flux_threshold = 1.1
parameters = np.zeros(
(N_sample, 7)
) #Number of lines // FHWM of max emission line // EW of max emission line // Spectrum Mean // Spectrum STDV // Spectrum Flux Integral // Spectrum SNR
for i in indi:
k = indi1[j, 1]
x = np.linspace(3600, 10500, 443)
zero_spectrum = spectra[k, :443]
spectrum = Spectrum1D(flux=zero_spectrum * u.Jy,
spectral_axis=x * u.AA)
#Continuum fit and gaussian smooth
g1_fit = fit_generic_continuum(spectrum)
y_continuum_fitted = g1_fit(x * u.AA)
spec_nw_2 = spectrum / y_continuum_fitted
spectrum_smooth = gaussian_smooth(spec_nw_2, kernel_size)
#Number of lines
lines_1 = find_lines_derivative(spectrum_smooth,
flux_threshold=flux_threshold)
l = lines_1[lines_1['line_type'] == 'emission']
number_lines = l['line_center_index'].shape[0]
parameters[j, 0] = number_lines
#FWHM
parameters[j, 1] = fwhm(spectrum_smooth).value
#EW
parameters[j, 2] = equivalent_width(spectrum_smooth).value
#Spectrum Mean
parameters[j, 3] = np.mean(spectrum_smooth.flux)
#Spectrum STDV
parameters[j, 4] = np.std(spectrum_smooth.flux)
#Spectrum Flux Integral
parameters[j, 5] = line_flux(spectrum_smooth).value
#Spectrum SNR
parameters[j, 6] = snr_derived(spectrum_smooth).value
j += 1
d = {
'Lines_Number': parameters[:, 0],
'FHWM': parameters[:, 1],
'EW': parameters[:, 2],
'Mean': parameters[:, 3],
'STDV': parameters[:, 4],
'STDV': parameters[:, 4],
'Spectrum_Flux': parameters[:, 5],
'SNR': parameters[:, 6]
}
parameters = pd.DataFrame(data=d)
#X=spectra_.values
#mean_flx= np.ma.average(X[:,:443],axis=1)
#ll=(X[:,:443]-mean_flx.reshape(-1,1))**2
#aveflux=np.ma.average(ll, axis=1)
#sflux = np.sqrt(aveflux)
#X = (X[:,:443]-mean_flx.reshape(-1,1))/sflux.reshape(-1,1)
y = sample_objects['Z_VI']
y = np.array(y, dtype=float)
#y_max=np.max(y)
#y=y/y_max
return parameters, y
stars = obj.loc[obj['CLASS_PERSON'] == 1]
galaxies = obj.loc[obj['CLASS_PERSON'] == 4]
qsos = obj.loc[obj['CLASS_PERSON'] == 3]
qsos_bal = obj.loc[obj['CLASS_PERSON'] == 30]
sample_star = stars.sample(n=int(N_sample / 4),
weights='CLASS_PERSON',
random_state=5)
sample_galaxy = galaxies.sample(n=int(N_sample / 4),
weights='CLASS_PERSON',
random_state=5)
sample_qso = qsos.sample(n=int(N_sample / 4),
weights='CLASS_PERSON',
random_state=5)
sample_qso_bal = qsos_bal.sample(n=int(N_sample / 4),
weights='CLASS_PERSON',
random_state=5)
sample_objects = pd.concat(
[sample_star, sample_galaxy, sample_qso, sample_qso_bal])
ind_star = np.array(sample_star.index)
ind_galaxy = np.array(sample_galaxy.index)
ind_qso = np.array(sample_qso.index)
ind_qso_bal = np.array(sample_qso_bal.index)
indi = np.concatenate((ind_star, ind_galaxy, ind_qso, ind_qso_bal),
axis=None)
indi1 = ind.loc[indi].values
spectra_ = np.zeros((N_sample, 886))
j = 0
for i in indi:
k = indi1[j, 1]
spectra_[j, :] = spectra[k, :]
j = j + 1
spectra_ = pd.DataFrame(spectra_)
X = spectra_.values
#Renormalize spectra
mean_flx = np.ma.average(X[:, :443], axis=1)
ll = (X[:, :443] - mean_flx.reshape(-1, 1))**2
aveflux = np.ma.average(ll, axis=1)
sflux = np.sqrt(aveflux)
X = (X[:, :443] - mean_flx.reshape(-1, 1)) / sflux.reshape(-1, 1)
y = sample_objects['CLASS_PERSON']
y = y.replace([1, 4, 3, 30], [0, 1, 2, 3]).values
y = np.array(y, dtype=float)
return X, y