def fetch_spectra_SDSS(object_name, save_dir, coords):
"""
saves a textfile in self.save_dir where the first column is the wavelength in angstroms and the second
column is flux in erg cm-2 s-1 AA-1
:return: the path to the saved spectrum file
"""
if os.path.exists(save_dir + object_name + 'spectrum.dat'):
getLogger(__name__).info('Spectrum already loaded, will not be reloaded')
spectrum_file = save_dir + object_name + 'spectrum.dat'
return spectrum_file
getLogger(__name__).info('Looking for {} spectrum in SDSS catalog'.format(object_name))
result = SDSS.query_region(coords, spectro=True)
if not result:
getLogger(__name__).warning('Could not find spectrum for {} at {},{} in SDSS catalog'.format(object_name, coords.ra, coords.dec))
spectrum_file = None
return spectrum_file
spec = SDSS.get_spectra(matches=result)
data = spec[0][1].data
lamb = 10**data['loglam'] * u.AA
flux = data['flux'] * 10 ** -17 * u.Unit('erg cm-2 s-1 AA-1')
spectrum = Spectrum1D(spectral_axis=lamb, flux=flux)
res = np.array([spectrum.spectral_axis, spectrum.flux])
res = res.T
spectrum_file = save_dir + object_name + 'spectrum.dat'
np.savetxt(spectrum_file, res, fmt='%1.4e')
getLogger(__name__).info('Spectrum loaded for {} from SDSS catalog'.format(object_name))
return spectrum_file
def download_sdss_spec(self, add_r=0):
sdss = SDSS()
self.spec_data_table = sdss.query_region(self.loc,
radius=(self.r + add_r) *
u.arcsec,
spectro=True)
if self.has_spec() == True:
print("在r=%f角秒内找到了%d个光谱" %
(self.r + add_r, len(self.spec_data_table)))
self.spec = SDSS.get_spectra(matches=self.spec_data_table)
i = 0
for match in self.spec_data_table:
self.spec[i] = Spectra(match['objid'], self.spec[i])
i = i + 1
else:
print("在r=%f角秒内没有找到光谱" % (self.r + add_r))
print Name, website
pv.SaveParameter_ObjLog(CodeName, FileFolder, 'SDSS_RA_deg', SDSS_RA)
pv.SaveParameter_ObjLog(CodeName, FileFolder, 'SDSS_DEC_deg', SDSS_DEC)
pv.SaveParameter_ObjLog(CodeName, FileFolder, 'SDSS_RA_Hour', SDSS_RA_hours)
pv.SaveParameter_ObjLog(CodeName, FileFolder, 'SDSS_DEC_Hour', SDSS_DEC_hours)
pv.SaveParameter_ObjLog(CodeName, FileFolder, 'SDSS_u_magModel', mag_u)
pv.SaveParameter_ObjLog(CodeName, FileFolder, 'SDSS_g_magModel', mag_g)
pv.SaveParameter_ObjLog(CodeName, FileFolder, 'SDSS_r_magModel', mag_r)
pv.SaveParameter_ObjLog(CodeName, FileFolder, 'SDSS_g_magModel', mag_g)
pv.SaveParameter_ObjLog(CodeName, FileFolder, 'URL_SDSS', website)
SDSS.cache_location = FileFolder
spec = SDSS.get_spectra(plate=plate, fiberID=fiberID, mjd=mjd, timeout=60, cache=True, data_release=12)
# print Name, SDSS_RA, SDSS_DEC
#Download the object spectrum
Spec_data = spec[0][2].data
sn_median = Spec_data['SN_MEDIAN_ALL'][0]
z_sdss = Spec_data['Z'][0]
Lines_data = spec[0][3].data
index_Hbeta = (Lines_data['LINENAME'] == Hbeta_label)
Flux_Hbeta = Lines_data['LINEAREA'][index_Hbeta]
Er_Hbeta = Lines_data['LINEAREA_ERR'][index_Hbeta]
Ew_Hbeta = Lines_data['LINEEW'][index_Hbeta]
EwEr_Hbeta = Lines_data['LINEEW_ERR'][index_Hbeta]
#print Name, mjd, plate, fiberID, sn_median, z_sdss, Flux_Hbeta, Ew_Hbeta
def grab_sdss_spectra(radec,
radius=0.1 * u.deg,
outfil=None,
debug=False,
maxsep=None,
timeout=600.,
zmin=None):
""" Grab SDSS spectra
Parameters
----------
radec : tuple
RA, DEC in deg
radius : float, optional (0.1*u.deg)
Search radius -- Astroquery actually makes a box, not a circle
timeout : float, optional
Timeout limit for connection with SDSS
outfil : str ('tmp.fits')
Name of output file for FITS table
maxsep : float (None) :: Mpc
Maximum separation to include
zmin : float (None)
Minimum redshift to include
Returns
-------
tbl : Table
"""
cC = coords.SkyCoord(ra=radec[0], dec=radec[1])
# Query
photoobj_fs = ['ra', 'dec', 'objid', 'run', 'rerun', 'camcol', 'field']
mags = [
'petroMag_u', 'petroMag_g', 'petroMag_r', 'petroMag_i', 'petroMag_z'
]
magsErr = [
'petroMagErr_u', 'petroMagErr_g', 'petroMagErr_r', 'petroMagErr_i',
'petroMagErr_z'
]
phot_catalog = SDSS.query_region(cC,
spectro=True,
radius=radius,
timeout=timeout,
photoobj_fields=photoobj_fs + mags +
magsErr) # Unique
spec_catalog = SDSS.query_region(cC,
spectro=True,
radius=radius,
timeout=timeout) # Duplicates exist
nobj = len(phot_catalog)
#
print('grab_sdss_spectra: Found {:d} sources in the search box.'.format(
nobj))
# Coordinates
cgal = SkyCoord(ra=phot_catalog['ra'] * u.degree,
dec=phot_catalog['dec'] * u.degree)
sgal = SkyCoord(ra=spec_catalog['ra'] * u.degree,
dec=spec_catalog['dec'] * u.degree)
sepgal = cgal.separation(cC) #in degrees
# Check for problems and parse z
zobj = np.zeros(nobj)
idx, d2d, d3d = coords.match_coordinates_sky(cgal, sgal, nthneighbor=1)
if np.max(d2d) > 1. * u.arcsec:
print('No spectral match!')
xdb.set_trace()
else:
zobj = spec_catalog['z'][idx]
idx, d2d, d3d = coords.match_coordinates_sky(cgal, cgal, nthneighbor=2)
if np.min(d2d.to('arcsec')) < 1. * u.arcsec:
print('Two photometric sources with same RA/DEC')
xdb.set_trace()
#xdb.set_trace()
# Cut on Separation
if not maxsep is None:
print('grab_sdss_spectra: Restricting to {:g} Mpc separation.'.format(
maxsep))
sepgal_kpc = cosmo.kpc_comoving_per_arcmin(zobj) * sepgal.to('arcmin')
sepgal_mpc = sepgal_kpc.to('Mpc')
gdg = np.where(sepgal_mpc < (maxsep * u.Unit('Mpc')))[0]
phot_catalog = phot_catalog[gdg]
#xdb.set_trace()
nobj = len(phot_catalog)
print('grab_sdss_spectra: Grabbing data for {:d} sources.'.format(nobj))
# Grab Spectra from SDSS
# Generate output table
attribs = galaxy_attrib()
npix = 5000 #len( spec_hdus[0][1].data.flux )
spec_attrib = [(str('FLUX'), np.float32, (npix, )),
(str('SIG'), np.float32, (npix, )),
(str('WAVE'), np.float64, (npix, ))]
tbl = np.recarray((nobj, ), dtype=attribs + spec_attrib)
tbl['RA'] = phot_catalog['ra']
tbl['DEC'] = phot_catalog['dec']
tbl['TELESCOPE'] = str('SDSS 2.5-M')
# Deal with spectra separately (for now)
npix = 5000 #len( spec_hdus[0][1].data.flux )
for idx, obj in enumerate(phot_catalog):
#print('idx = {:d}'.format(idx))
# Grab spectra (there may be duplicates)
mt = np.where(
sgal.separation(cgal[idx]).to('arcsec') < 1. * u.Unit('arcsec'))[0]
if len(mt) > 1:
# Use BOSS if you have it
mmt = np.where(spec_catalog[mt]['instrument'] == 'BOSS')[0]
if len(mmt) > 0:
mt = mt[mmt[0]]
else:
mt = mt[0]
elif len(mt) == 0:
xdb.set_trace()
else:
mt = mt[0]
# Grab spectra
spec_hdus = SDSS.get_spectra(matches=Table(spec_catalog[mt]))
tbl[idx]['INSTRUMENT'] = spec_catalog[mt]['instrument']
spec = spec_hdus[0][1].data
npp = len(spec.flux)
tbl[idx]['FLUX'][0:npp] = spec.flux
sig = np.zeros(npp)
gdi = np.where(spec.ivar > 0.)[0]
if len(gdi) > 0:
sig[gdi] = np.sqrt(1. / spec.ivar[gdi])
tbl[idx]['SIG'][0:npp] = sig
tbl[idx]['WAVE'][0:npp] = 10.**spec.loglam
# Redshifts
meta = spec_hdus[0][2].data
for attrib in ['Z', 'Z_ERR']:
tbl[idx][attrib] = meta[attrib]
if debug:
sep_to_qso = cgal[idx].separation(cC).to('arcmin')
print('z = {:g}, Separation = {:g}'.format(tbl[idx].Z, sep_to_qso))
xdb.set_trace()
# Fill in rest
tbl[idx].SDSS_MAG = np.array([obj[phot] for phot in mags])
tbl[idx].SDSS_MAGERR = np.array([obj[phot] for phot in magsErr])
# Clip on redshift to excise stars/quasars
if zmin is not None:
gd = np.where(tbl['Z'] > zmin)[0]
tbl = tbl[gd]
# Write to FITS file
if outfil is not None:
prihdr = fits.Header()
prihdr['COMMENT'] = 'SDSS Spectra'
prihdu = fits.PrimaryHDU(header=prihdr)
tbhdu = fits.BinTableHDU(tbl)
thdulist = fits.HDUList([prihdu, tbhdu])
thdulist.writeto(outfil, clobber=True)
print('Wrote SDSS table to {:s}'.format(outfil))
return tbl
from astroquery.simbad import Simbad
result_table = Simbad.query_object("m1")
result_table.pprint(show_unit=True)
from astroquery.sdss import SDSS
from astropy import coordinates as coords
#pos = coords.SkyCoord(packet['candidate']['ra'],packet['candidate']['dec'], unit="deg”) #to use info from avro packet
pos = coords.SkyCoord(244.566202,16.138884, unit="deg”) #to test query
xid = SDSS.query_region(pos, spectro=True)
print(xid)
print(xid.columns)
xid['z']
sp = SDSS.get_spectra(matches=xid)
im = SDSS.get_images(matches=xid, band='r')
objDF[f'{szLabel}_err'] = np.nan
objDF['Comment'] = 'None'
for idx, obj in enumerate(objDF.index.values):
# Object indexing
plate, mjd, fiber = objDF.iloc[idx]['PLATEID':'FIBERID_1']
objLabel = f'{plate}-{mjd}-{fiber}'
spec_name = f'{objLabel}.fits'
output_file = fits_folder / spec_name
print(f'- Obj {idx}/{n_objs}')
try:
spec = SDSS.get_spectra(plate=plate,
fiberID=fiber,
mjd=mjd,
timeout=60,
cache=True,
data_release=16)
spec[0].writeto(output_file, overwrite=True)
except:
spec = None
print(f'-- Astroquery failure: {objLabel}')
if spec is None:
try:
output_file = fits_folder / spec_name
fetch_spec(str(output_file), plate, mjd, fiber)
time.sleep(3)
except:
spec = None
objDF.loc[objLabel, 'Comment'] = 'No result query'
def grab_sdss_spectra(radec, radius=0.1*u.deg, outfil=None,
debug=False, maxsep=None, timeout=600., zmin=None):
""" Grab SDSS spectra
Parameters
----------
radec : tuple
RA, DEC in deg
radius : float, optional (0.1*u.deg)
Search radius -- Astroquery actually makes a box, not a circle
timeout : float, optional
Timeout limit for connection with SDSS
outfil : str ('tmp.fits')
Name of output file for FITS table
maxsep : float (None) :: Mpc
Maximum separation to include
zmin : float (None)
Minimum redshift to include
Returns
-------
tbl : Table
"""
cC = coords.SkyCoord(ra=radec[0], dec=radec[1])
# Query
photoobj_fs = ['ra', 'dec', 'objid', 'run', 'rerun', 'camcol', 'field']
mags = ['petroMag_u', 'petroMag_g', 'petroMag_r', 'petroMag_i', 'petroMag_z']
magsErr = ['petroMagErr_u', 'petroMagErr_g', 'petroMagErr_r', 'petroMagErr_i', 'petroMagErr_z']
phot_catalog = SDSS.query_region(cC,spectro=True,radius=radius, timeout=timeout,
photoobj_fields=photoobj_fs+mags+magsErr) # Unique
spec_catalog = SDSS.query_region(cC,spectro=True, radius=radius, timeout=timeout) # Duplicates exist
nobj = len(phot_catalog)
#
print('grab_sdss_spectra: Found {:d} sources in the search box.'.format(nobj))
# Coordinates
cgal = SkyCoord(ra=phot_catalog['ra']*u.degree, dec=phot_catalog['dec']*u.degree)
sgal = SkyCoord(ra=spec_catalog['ra']*u.degree, dec=spec_catalog['dec']*u.degree)
sepgal = cgal.separation(cC) #in degrees
# Check for problems and parse z
zobj = np.zeros(nobj)
idx, d2d, d3d = coords.match_coordinates_sky(cgal, sgal, nthneighbor=1)
if np.max(d2d) > 1.*u.arcsec:
print('No spectral match!')
xdb.set_trace()
else:
zobj = spec_catalog['z'][idx]
idx, d2d, d3d = coords.match_coordinates_sky(cgal, cgal, nthneighbor=2)
if np.min(d2d.to('arcsec')) < 1.*u.arcsec:
print('Two photometric sources with same RA/DEC')
xdb.set_trace()
#xdb.set_trace()
# Cut on Separation
if not maxsep is None:
print('grab_sdss_spectra: Restricting to {:g} Mpc separation.'.format(maxsep))
sepgal_kpc = cosmo.kpc_comoving_per_arcmin(zobj) * sepgal.to('arcmin')
sepgal_mpc = sepgal_kpc.to('Mpc')
gdg = np.where( sepgal_mpc < (maxsep * u.Unit('Mpc')))[0]
phot_catalog = phot_catalog[gdg]
#xdb.set_trace()
nobj = len(phot_catalog)
print('grab_sdss_spectra: Grabbing data for {:d} sources.'.format(nobj))
# Grab Spectra from SDSS
# Generate output table
attribs = galaxy_attrib()
npix = 5000 #len( spec_hdus[0][1].data.flux )
spec_attrib = [(str('FLUX'), np.float32, (npix,)),
(str('SIG'), np.float32, (npix,)),
(str('WAVE'), np.float64, (npix,))]
tbl = np.recarray( (nobj,), dtype=attribs+spec_attrib)
tbl['RA'] = phot_catalog['ra']
tbl['DEC'] = phot_catalog['dec']
tbl['TELESCOPE'] = str('SDSS 2.5-M')
# Deal with spectra separately (for now)
npix = 5000 #len( spec_hdus[0][1].data.flux )
for idx,obj in enumerate(phot_catalog):
#print('idx = {:d}'.format(idx))
# Grab spectra (there may be duplicates)
mt = np.where( sgal.separation(cgal[idx]).to('arcsec') < 1.*u.Unit('arcsec'))[0]
if len(mt) > 1:
# Use BOSS if you have it
mmt = np.where( spec_catalog[mt]['instrument'] == 'BOSS')[0]
if len(mmt) > 0:
mt = mt[mmt[0]]
else:
mt = mt[0]
elif len(mt) == 0:
xdb.set_trace()
else:
mt = mt[0]
# Grab spectra
spec_hdus = SDSS.get_spectra(matches=Table(spec_catalog[mt]))
tbl[idx]['INSTRUMENT'] = spec_catalog[mt]['instrument']
spec = spec_hdus[0][1].data
npp = len(spec.flux)
tbl[idx]['FLUX'][0:npp] = spec.flux
sig = np.zeros(npp)
gdi = np.where(spec.ivar > 0.)[0]
if len(gdi) > 0:
sig[gdi] = np.sqrt( 1./spec.ivar[gdi] )
tbl[idx]['SIG'][0:npp] = sig
tbl[idx]['WAVE'][0:npp] = 10.**spec.loglam
# Redshifts
meta = spec_hdus[0][2].data
for attrib in ['Z','Z_ERR']:
tbl[idx][attrib] = meta[attrib]
if debug:
sep_to_qso = cgal[idx].separation(cC).to('arcmin')
print('z = {:g}, Separation = {:g}'.format(tbl[idx].Z, sep_to_qso))
xdb.set_trace()
# Fill in rest
tbl[idx].SDSS_MAG = np.array( [obj[phot] for phot in mags])
tbl[idx].SDSS_MAGERR = np.array( [obj[phot] for phot in magsErr])
# Clip on redshift to excise stars/quasars
if zmin is not None:
gd = np.where(tbl['Z'] > zmin)[0]
tbl = tbl[gd]
# Write to FITS file
if outfil is not None:
prihdr = fits.Header()
prihdr['COMMENT'] = 'SDSS Spectra'
prihdu = fits.PrimaryHDU(header=prihdr)
tbhdu = fits.BinTableHDU(tbl)
thdulist = fits.HDUList([prihdu, tbhdu])
thdulist.writeto(outfil,clobber=True)
print('Wrote SDSS table to {:s}'.format(outfil))
return tbl
def sdss_spectra(coo, redshift=0., columns=1):
try:
n_coo = len(coo)
except:
n_coo = 1
coo = [coo]
try:
n_redshift = len(redshift)
except:
n_redshift = 1
redshift = [redshift]
if n_coo > 1 & n_redshift == 1:
redshift = np.ones(n_coo) * redshift[0]
n_col = np.min([n_coo, columns])
n_row = np.ceil(n_coo * 1. / n_col)
fig, ax = plt.subplots(figsize=(16, 6 * n_row / (n_col * 1.)),
sharex=True,
sharey=True)
# fig, ax = plt.subplots(figsize=(20,8*n_coo), sharex=True, sharey=True)
for i in range(len(coo)):
ax = plt.subplot(n_row, n_col, i + 1)
#ax = plt.subplot(n_coo, 1,i+1)
print 'Procesando galaxia ' + str(i)
xid = SDSS.query_region(coo[i], spectro=True)
spec = SDSS.get_spectra(matches=xid)[0][1]
spec_h = spec.header
spec_data = spec.data
loglam = spec_data['loglam'] # Logaritmo de la longitud de onda
flux = spec_data['flux'] # Flujo medido en unidades de Ergs/cm^2/s/AA
window_len = 9
s = np.r_[flux[window_len - 1:0:-1], flux, flux[-1:-window_len:-1]]
w = np.ones(window_len, 'd')
w = eval('np.bartlett(window_len)')
flux_smooth = np.convolve(w / w.sum(), s,
mode='valid')[(window_len - 1) /
2:-(window_len - 1) / 2]
gv = (loglam > np.log10(4000.)) & (loglam < np.log10(8000.))
flux_scale = 80. / np.percentile(flux_smooth[gv], 98)
#ax.plot(10.**loglam, flux*flux_scale, label=xid['instrument'][0], color='black', linewidth=1)
ax.plot(10.**loglam,
flux_smooth * flux_scale,
label=xid['instrument'][0],
color='black',
linewidth=1)
for j in range(len(absorption_lines['name'])):
ax.plot(
absorption_lines['lambda'][j] * np.ones(2) *
(1. + redshift[i]), [0., 1e5],
absorption_lines['color'] + '--')
ax.text(absorption_lines['lambda'][j] * (1. + redshift[i]) +
absorption_lines['offset'][j],
absorption_lines['position'][j] * 100.,
absorption_lines['name'][j],
color=absorption_lines['color'],
alpha=0.7,
fontsize=16 / (n_col * 0.8),
horizontalalignment=absorption_lines['align'][j])
for j in range(len(emission_lines['name'])):
ax.plot(
emission_lines['lambda'][j] * np.ones(2) * (1. + redshift[i]),
[0., 1e5], emission_lines['color'] + '--')
ax.text(emission_lines['lambda'][j] * (1. + redshift[i]) +
emission_lines['offset'][j],
emission_lines['position'][j] * 100.,
emission_lines['name'][j],
color=emission_lines['color'],
alpha=0.7,
fontsize=16 / (n_col * 0.8),
horizontalalignment=emission_lines['align'][j])
if (i % n_col == 0):
ax.set_ylabel(r'Flujo [10$^{-17}$ ergs/cm$^2$/s/$\AA$]',
fontsize=14 / (n_col * 0.8))
if (i >= (n_col * (n_row - 1))):
ax.set_xlabel(r'Longitud de onda [$\AA$]', fontsize=14)
ax.set_title('Galaxia ' + str(i))
ax.set_xlim(3500, 8000)
ax.set_ylim(0., 100.)
fig.subplots_adjust(hspace=0.3, wspace=0.1)
# Get the entire catalog, not just the first 50 rows
Vizier.ROW_LIMIT = 2
catalogs = Vizier.get_catalogs(catalog_list.keys())
# cat = 'J/MNRAS/455/3413/table6'
PMF = catalogs[0]['PMF']
#for i in range(len(PMF)):
# PMF[i] = PMF[i].decode("utf-8")
print("Downloading " + str(len(PMF)) + " spectra now...")
for i in np.arange(len(PMF)):
filename = '../data/spectra-' + str(PMF[i].decode("utf-8")) + '.fits'
if os.path.exists(filename): continue
# print("PMF:", PMF[i].decode("utf-8"))
plate = int(str(PMF[i].decode("utf-8"))[0:4])
mjd = int(str(PMF[i].decode("utf-8"))[5:10])
fiber = int(str(PMF[i].decode("utf-8"))[11:15])
try:
spec = SDSS.get_spectra(plate=plate, mjd=mjd, fiberID=fiber)
spec[0].writeto(filename)
except:
print("Could not download spectra:", plate, mjd, fiber)
pass
print("...finished downloading spectra.")
from astropy.table import Table
import matplotlib.pyplot as plt
from matplotlib.ticker import MultipleLocator
import seaborn as sns
import numpy as np
import matplotlib
from tables import open_file
from astropy.io import fits
sns.set_context('talk')
ML = MultipleLocator(500)
ml = MultipleLocator(100)
MLy = MultipleLocator(1)
mly = MultipleLocator(0.2)
ncat = Table.read('ryota.dat', format='ascii.fixed_width_two_line')
sp = SDSS.get_spectra(matches=ncat)
index = -5
out_pdf = 'ryota_hetdex.pdf'
F = fits.open('ryota_hetdex_spectra.fits')
pdf = matplotlib.backends.backend_pdf.PdfPages(out_pdf)
hdfile = open_file('survey_hdr1.h5')
t = Table(hdfile.root.Survey[:])
cnt = 0
for index in np.arange(len(ncat)):
if (cnt % 3) == 0:
plot_num = 311
fig = plt.figure(figsize=(8.5, 11)) # inches
plt.subplot(plot_num)
plt.plot(10**(sp[index][1].data['loglam']), sp[index][1].data['flux'], lw=1,
alpha=0.5, label='SDSS')
flam = 10**(-0.4 * (ncat[index]['g']-23.9)) * 1e-29 * 3e18 / 5000.**2 * 1e17
def download_spectra(coord_list_url, from_sp, to_sp, save=False):
"""
download_spectra()
Downloads SDSS spectra in a specified range based on a list of coordinates
Parameters
----------
coord_list_url: string
The path for the CSV file that contains the list of coordinates
that was downloaded using SQL, which provides coordinates for spectra
to download. Contains 500,000 rows
from_sp : int
The index from which to download spectra. This enables us to download in
batches and save the spectral data in batches
to_sp : int
The index which specifies the upper limit until which to download spectra.
save : boolean
When True, save the resulting DataFrame into a pickle
When False, don't save
Returns
-------
df: pandas.DataFrame
The DataFrame that contains all downloaded spectral data.
columns: 'flux_list',
'wavelength',
'z',
'ra',
'dec',
'objid'
"""
t_start = time.clock()
coord_list = pd.read_csv(filepath_or_buffer=coord_list_url)
print(f'coord_list = {coord_list}')
ra_list = coord_list["ra"].tolist()
dec_list = coord_list["dec"].tolist()
ra = ra_list[from_sp:to_sp]
dec = dec_list[from_sp:to_sp]
n_errors = 0
df = {}
df['flux_list'] = []
df['wavelength'] = []
df['z'] = []
df['ra'] = []
df['dec'] = []
df['objid'] = []
n_coordinates = len(ra)
number_none = 0
for i in range(n_coordinates):
try:
pos = coords.SkyCoord((ra[i]) * u.deg, (dec[i]) * u.deg,
frame='icrs')
xid = SDSS.query_region(pos, spectro=True) # radius=5 * u.arcsec)
if xid == None:
number_none = number_none + 1
print('xid is None at:', i)
continue
elif xid != None and len(xid) > 1:
xid = Table(xid[0])
sp = SDSS.get_spectra(matches=xid)
df['flux_list'].append(sp[0][1].data['flux'])
df['wavelength'].append(10.**sp[0][1].data['loglam'])
df['z'].append(xid['z'])
df['ra'].append(xid['ra'])
df['dec'].append(xid['dec'])
df['objid'].append(xid['objid'])
print(f'Downloaded: {i}')
except:
print('Failed to download at:', i)
n_errors = n_errors + 1
df = pd.DataFrame(df)
print('df.head()', df.head())
if save:
df.to_pickle('data/sdss/spectra/spectra_' + str(from_sp) + '-' +
str(to_sp) + '.pkl')
t_end = time.clock()
t_delta = t_end - t_start
n_downloads = len(ra) - 1
print("time for " + str(n_downloads) + " stellar objects:", t_delta)
print('DF After Downloading:')
print(df.columns)
print(df)
print(f'Length of df = {len(df)}')
return df
id = empates[0][empates[1].arcsec <= 1.0]
r_Mag = UGC10214_sdss['r'][id]
rH_color = r_Mag - (UGC10214_2mass['h_m'])[empates[1].arcsec <= 1.0]
gr_color = UGC10214_sdss['g'][id] - r_Mag
positivos = (r_Mag > 0) & (rH_color > 0) & (gr_color > 0)
plt.scatter(rH_color[positivos], gr_color[positivos], color='k')
plt.xlabel('r-H')
plt.ylabel('g-r')
i = 1
espectros = []
fibras = []
while i < 641:
try:
espectros.append(SDSS.get_spectra(plate=266,\
fiberID=i, mjd=51630)[0])
fibras.append(i)
i += 1
except:
i += 1
flujo = [(x[1].data['flux']) for x in espectros]
lam = [10**(x[1].data['loglam']) for x in espectros]
import statsmodels.api as sm
lowess = sm.nonparametric.lowess
m = 6563
modelo = models.Gaussian1D(40, m, 100)
modelo.mean.max = m + 200
modelo.mean.min = m - 200
def do_sdss_spectra(catalog):
"""Import spectra from LAMOST."""
task_str = catalog.get_current_task_str()
# Set preferred names, calculate some columns based on imported data,
# sanitize some fields
keys = list(catalog.entries.keys())
fureps = {
'erg/cm2/s/A': 'erg/s/cm^2/Angstrom',
'1E-17 erg/cm^2/s/Ang': 'erg/s/cm^2/Angstrom'
}
c_kms = con.c.cgs.value / 1.0e5
cntsdss = 0
for oname in pbar(keys, task_str):
# Some events may be merged in cleanup process, skip them if
# non-existent.
if (FASTSTARS.RA not in catalog.entries[oname]
or FASTSTARS.DEC not in catalog.entries[oname]):
continue
else:
xid = SDSS.query_region(coord.SkyCoord(
ra=catalog.entries[oname][FASTSTARS.RA][0]['value'],
dec=catalog.entries[oname][FASTSTARS.DEC][0]['value'],
unit=(un.hourangle, un.deg),
frame='icrs'),
spectro=True)
# xid = SDSS.query_region(coord.SkyCoord(
# ra='14:34:06.17',
# dec='+56:30:47.24',
# unit=(un.hourangle, un.deg), frame='icrs'), spectro=True)
if xid is None:
continue
while len(xid) > 1:
notstar = xid['z'].argmax()
xid.remove_row(notstar)
#print(xid)
# star = None
# for row in tab:
# if (row['objType'] == 'Star' and
# row['Class'].lower() in ['star', 'unknown']):
# star = row
# break
# if not star:
# continue
try:
name, source = catalog.new_entry(oname,
bibcode='2015ApJS..219...12A',
srcname='SDSS',
url='http://www.sdss.org/')
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
ffile = ('spec-' + str(xid['specobjid'][0]) + '.fits.gz')
# furl = 'http://dr3.lamost.org/sas/fits/' + vname + '/' + ffile
datafile = os.path.join(catalog.get_current_task_repo(), 'SDSS',
ffile)
if not os.path.exists(datafile):
# Download spectra
try:
sp = SDSS.get_spectra(matches=xid)[0]
except urllib.error.HTTPError:
catalog.log.warning(
'"{}" threw an HTTP 404, must be error upstream. Will likely go away on the next run.'
.format(oname))
continue
# Identify star
# assert sp[2].data['class'][0]=='STAR'
# Write spectra
# print(catalog.entries[oname][FASTSTARS.RA][0]['value'],catalog.entries[oname][FASTSTARS.DEC][0]['value'])
sp.writeto(datafile, overwrite=True)
# open(datafile, 'wb').write(fr.content)
# Open spectra
hdulist = fits.open(datafile)
# sp contains a list of fits datafiles, identify main one
i_primary = 0
i_coadd = 1
i_specobj = 2
assert hdulist[i_primary].name == 'PRIMARY'
assert hdulist[i_coadd].name == 'COADD'
assert (hdulist[i_specobj].name == 'SPECOBJ'
or hdulist[i_specobj].name == 'SPALL')
# xid = SDSS.query_region(coord.SkyCoord(
# ra='12:11:50.27',
# dec='+14:37:16.2',
# unit=(un.hourangle, un.deg), frame='icrs'), spectro=True)
# from SPECOBJ
# print('.'+hdulist[i_specobj].data['ELODIE_SPTYPE'][0]+'.')
if hdulist[i_specobj].data['ELODIE_SPTYPE'][0] != 'unknown':
ST, SCfull = hdulist[i_specobj].data['ELODIE_SPTYPE'][
0][:2], hdulist[i_specobj].data['ELODIE_SPTYPE'][0][2:]
if len(SCfull) > 0:
if 'IV' in SCfull:
SC = 'sg'
elif 'III' in SCfull:
SC = 'g'
elif 'V' in SCfull:
SC = 'd'
elif 'I' in SCfull:
SC = 'Sg'
else:
SC = False
if SC is not False:
catalog.entries[name].add_quantity(
FASTSTARS.STELLAR_CLASS, SC, source=source)
catalog.entries[name].add_quantity(FASTSTARS.SPECTRAL_TYPE,
ST,
source=source)
if hdulist[i_specobj].data['Z'][0] != 0.0:
catalog.entries[name].add_quantity(
FASTSTARS.REDSHIFT,
str(hdulist[i_specobj].data['Z'][0]),
e_value=str(hdulist[i_specobj].data['Z_ERR'][0]),
source=source)
catalog.entries[name].add_quantity(
FASTSTARS.VELOCITY,
pretty_num(float(hdulist[i_specobj].data['Z'][0]) * c_kms,
sig=5),
e_value=pretty_num(float(
hdulist[i_specobj].data['Z_ERR'][0] * c_kms),
sig=5),
source=source)
for oi, obj in enumerate(hdulist[0].header):
if any(x in ['.', '/'] for x in obj):
del (hdulist[0].header[oi])
hdulist[0].verify('silentfix')
hdrkeys = list(hdulist[0].header.keys())
# print(hdrkeys)
# for key in hdulist[0].header.keys():
# print(key, hdulist[0].header[key])
if hdulist[0].header['SIMPLE']:
if 'JD' in hdrkeys:
mjd = str(jd_to_mjd(Decimal(str(hdulist[0].header['JD']))))
elif 'MJD' in hdrkeys:
mjd = str(hdulist[0].header['MJD'])
elif 'DATE-OBS' in hdrkeys:
if 'T' in hdulist[0].header['DATE-OBS']:
dateobs = hdulist[0].header['DATE-OBS'].strip()
elif 'UTC-OBS' in hdrkeys:
dateobs = hdulist[0].header['DATE-OBS'].strip(
) + 'T' + hdulist[0].header['UTC-OBS'].strip()
mjd = str(astrotime(dateobs, format='isot').mjd)
else:
raise ValueError("Couldn't find JD/MJD for spectrum.")
if hdulist[i_coadd].header['NAXIS'] == 2:
waves = [
str(x)
for x in list(10**hdulist[i_coadd].data['loglam'])
]
fluxes = [
str(x) for x in list(hdulist[i_coadd].data['flux'])
]
else:
print('Warning: Skipping FITS spectrum `{}`.'.format(
datafile))
continue
else:
raise ValueError('Non-simple FITS import not yet supported.')
if 'BUNIT' in hdrkeys:
fluxunit = hdulist[0].header['BUNIT']
if fluxunit in fureps:
fluxunit = fureps[fluxunit]
if fluxunit[:3] == '1E-17':
fluxes = [
str(x * 1e-17)
for x in list(hdulist[i_coadd].data['flux'])
]
else:
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
specdict = {
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.WAVELENGTHS: waves,
SPECTRUM.TIME: mjd,
SPECTRUM.U_TIME: 'MJD',
SPECTRUM.FLUXES: fluxes,
SPECTRUM.U_FLUXES: fluxunit,
SPECTRUM.FILENAME: ffile,
SPECTRUM.SOURCE: source
}
if 'TELESCOP' in hdrkeys:
specdict[SPECTRUM.TELESCOPE] = hdulist[0].header['TELESCOP']
if 'INSTRUME' in hdrkeys:
specdict[SPECTRUM.INSTRUMENT] = hdulist[0].header['INSTRUME']
if 'SITENAME' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['SITENAME']
elif 'OBSERVAT' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['OBSERVAT']
if 'OBSERVER' in hdrkeys:
specdict[SPECTRUM.OBSERVER] = hdulist[0].header['OBSERVER']
if 'AIRMASS' in hdrkeys:
specdict[SPECTRUM.AIRMASS] = hdulist[0].header['AIRMASS']
catalog.entries[name].add_spectrum(**specdict)
cntsdss += 1
hdulist.close()
catalog.journal_entries()
print('`{}` have SDSS spectra.'.format(cntsdss))
return
def sdss_spectra(coo, redshift=0., columns=1):
try:
n_coo=len(coo)
except:
n_coo=1
coo=[coo]
try:
n_redshift=len(redshift)
except:
n_redshift=1
redshift=[redshift]
if n_coo>1 & n_redshift==1:
redshift=np.ones(n_coo)*redshift[0]
n_col=np.min([n_coo,columns])
n_row=np.ceil(n_coo*1./n_col)
fig, ax = plt.subplots(figsize=(16,6*n_row/(n_col*1.)), sharex=True, sharey=True)
# fig, ax = plt.subplots(figsize=(20,8*n_coo), sharex=True, sharey=True)
for i in range(len(coo)):
ax = plt.subplot(n_row, n_col, i+1)
#ax = plt.subplot(n_coo, 1,i+1)
print 'Procesando galaxia '+str(i)
xid = SDSS.query_region(coo[i], spectro=True)
spec=SDSS.get_spectra(matches=xid)[0][1]
spec_h=spec.header
spec_data=spec.data
loglam=spec_data['loglam'] # Logaritmo de la longitud de onda
flux=spec_data['flux'] # Flujo medido en unidades de Ergs/cm^2/s/AA
window_len=9
s=np.r_[flux[window_len-1:0:-1],flux,flux[-1:-window_len:-1]]
w=np.ones(window_len,'d')
w=eval('np.bartlett(window_len)')
flux_smooth=np.convolve(w/w.sum(),s,mode='valid')[(window_len-1)/2:-(window_len-1)/2]
gv=(loglam>np.log10(4000.)) & (loglam<np.log10(8000.))
flux_scale=80./np.percentile(flux_smooth[gv],98)
#ax.plot(10.**loglam, flux*flux_scale, label=xid['instrument'][0], color='black', linewidth=1)
ax.plot(10.**loglam, flux_smooth*flux_scale, label=xid['instrument'][0], color='black', linewidth=1)
for j in range(len(absorption_lines['name'])):
ax.plot(absorption_lines['lambda'][j]*np.ones(2)*(1.+redshift[i]), [0., 1e5], absorption_lines['color']+'--')
ax.text(absorption_lines['lambda'][j]*(1.+redshift[i])+absorption_lines['offset'][j], absorption_lines['position'][j]*100., absorption_lines['name'][j], color=absorption_lines['color'], alpha=0.7, fontsize=16/(n_col*0.8), horizontalalignment=absorption_lines['align'][j])
for j in range(len(emission_lines['name'])):
ax.plot(emission_lines['lambda'][j]*np.ones(2)*(1.+redshift[i]), [0., 1e5], emission_lines['color']+'--')
ax.text(emission_lines['lambda'][j]*(1.+redshift[i])+emission_lines['offset'][j], emission_lines['position'][j]*100., emission_lines['name'][j], color=emission_lines['color'], alpha=0.7, fontsize=16/(n_col*0.8), horizontalalignment=emission_lines['align'][j])
if (i % n_col == 0):
ax.set_ylabel(r'Flujo [10$^{-17}$ ergs/cm$^2$/s/$\AA$]', fontsize=14/(n_col*0.8))
if (i >= (n_col*(n_row-1))):
ax.set_xlabel(r'Longitud de onda [$\AA$]', fontsize=14)
ax.set_title('Galaxia '+str(i))
ax.set_xlim(3500,8000)
ax.set_ylim(0.,100.)
fig.subplots_adjust(hspace=0.3, wspace=0.1)
def get_sdss_spectra(outfile = "outfile", N_spec = 5):
from urllib2 import HTTPError
from astroquery.sdss import SDSS
# query = "SELECT TOP 1000 p.objid, p.dec, p.r,p.i, p.run, p.rerun, p.camcol, p.field, s.specobjid, s.class, s.z as redshift FROM PhotoObj AS p JOIN SpecObj AS s ON s.bestobjid = p.objid WHERE p.r BETWEEN 0 AND 17.0 AND s.class = 'QSO' AND s.z BETWEEN 1.0 AND 2.3 AND p.dec >= 15.0"
query = "SELECT TOP "+str(N_spec)+" specObjID, plate, mjd, subClass, fiberID FROM SpecPhoto WHERE (class = 'QSO') AND" \
" (psfmag_r <= 17.0) AND (dec >= 15.0) AND (z BETWEEN 1.0 AND 2.3) AND zwarning = 0 AND" \
" (subClass = 'BROADLINE') AND nChild = 0 AND (mode = 1) AND ((0x10000000) != 0)" \
" AND (bossprimary= 0) AND programname = 'legacy'"
res = SDSS.query_sql(query)
# (subClass = 'BROADLINE') AND
# print(res['subClass'])
spectra = []
var = []
waves = []
mask = []
z = []
# print(res['plate'], res['mjd'], res['fiberID'])
num_skipped = 0
count = 1
n_spec = len(res['specObjID'])
for i in range(n_spec):
# print(res['subClass'][i])
try:
sp = SDSS.get_spectra(plate=res['plate'][i], mjd=res['mjd'][i], fiberID=res['fiberID'][i])[0]
data = (sp[1].data)
wave = (10**data.field('loglam'))
flux = data.field('flux')
err = data.field('ivar')
masking = data.field('and_mask')
mask.append(masking)
z.append(sp[2].data.field('Z'))
spectra.append(flux)
var.append(err)
waves.append(wave)
# print(res['plate'][i],res['mjd'][i], res['fiberID'][i])
# pl.plot(wave, flux)
# pl.show()
count += 1
except HTTPError:
num_skipped += 1
print("%i, %i, %i not found" % (res['plate'][i], res['mjd'][i], res['fiberID'][i]))
continue
except ValueError:
num_skipped += 1
print("%i, %i, %i ValueError" % (res['plate'][i], res['mjd'][i], res['fiberID'][i]))
continue
except TypeError:
num_skipped += 1
print("%i, %i, %i TypeError" % (res['plate'][i], res['mjd'][i], res['fiberID'][i]))
continue
print('Number of spectrum processed: {0} out of {1}'.format(count, n_spec - num_skipped))
print(" %i spectra skipped" % num_skipped)
# exit()
np.savez(outfile,
wave = waves,
spectra=spectra,
var = var,
mask = mask,
plate = res['plate'],
mjd = res['mjd'],
fiberID = res['fiberID'],
z = z
)
def getspectrum(plate, fiber, mjd):
hdulist = SDSS.get_spectra(plate=plate, fiberID=fiber, mjd=mjd)[0]
spectrum = hdulist[1].data
return spectrum
