def run_hostlist(nowise=False,noML=False,nosaga=False,flagged_obs_hosts=False):
"""
For each host in hostlist, create base catalog
default is to read offline Lang WISE catalogs
Parameters
----------
nowise : bool, optional. Turn off WISE catalog read
"""
# RUN EITHER FULL HOST LIST OR JSUT FLAG ZERO HOSTS, default to flag zero
sheet = GoogleSheets('1b3k2eyFjHFDtmHce1xi6JKuj3ATOWYduTBFftx5oPp8', 448084634)
flag0data = sheet.load()
file = SAGA_DROPBOX + '/hosts/submaster_all.ecsv.gz'
submaster = ascii.read(file, format='ecsv')
## print googleflag0.columns
flag0 = np.in1d(submaster['NSAID'],flag0data['NSAID'])
hostdata = submaster[flag0]
# FOR EACH HOST, READ SQL AND CREATE BASE CATALOGS
for host in hostdata:
nid = host['NSAID']
catalog = create_base_catalog(nid, host,nowise,noML,nosaga)
write_base_fits(nid, catalog)
def run_named_hosts(nowise=False,noML=False,nosaga=False):
"""
Run base catalog for SAGA named_hosts host
ACCEPTS ONLY FLAG_ZERO HOSTS FOR MOMENT!
Parameters
----------
nowise : bool, optional. Turn off WISE catalog read
"""
# sheet = GoogleSheets('1b3k2eyFjHFDtmHce1xi6JKuj3ATOWYduTBFftx5oPp8', 448084634)
# hostdata = sheet.load()
file = SAGA_DROPBOX + '/hosts/submaster_all.ecsv.gz'
hostdata = ascii.read(file, format='ecsv')
sheet = GoogleSheets('1GJYuhqfKeuJr-IyyGF_NDLb_ezL6zBiX2aeZFHHPr_s', 0)
named_hosts = sheet.load()
for sagahost in named_hosts:
nid = sagahost['NSA']
msk = hostdata['NSAID'] == nid
if np.sum(msk) == 0:
print nid#,sagahost['SAGA Name']
print 'NO HOST FOUND'
print
else:
catalog = create_base_catalog(nid, hostdata[msk],nowise,noML,nosaga)
write_base_fits(nid, catalog)
def run_query(flagged_obs_hosts=False):
"""
Send casjob queries for each host
Parameters
----------
flagged_obs_hosts : bool, optional
Returns
-------
Downloads host files into sql files located in SAGADIR/hosts
"""
# RUN EITHER FULL HOST LIST OR JSUT FLAG ZERO HOSTS, default to flag zero
if flagged_obs_hosts:
sheet = GoogleSheets('1GJYuhqfKeuJr-IyyGF_NDLb_ezL6zBiX2aeZFHHPr_s', 0)
else:
sheet = GoogleSheets('1b3k2eyFjHFDtmHce1xi6JKuj3ATOWYduTBFftx5oPp8',
448084634)
hostdata = sheet.load()
# FOR EACH HOST, DOWNLOAD SQL QUERY
for host in hostdata:
if flagged_obs_hosts and not host['Host Flag']:
continue
nid = host['NSAID']
if nid == -1:
continue
print nid
# CREATE SQL QUERY
qry = construct_sdss_query(nid, host['RA'], host['Dec'])
print qry
run_casjob(qry, 'sql_nsa{}'.format(nid))
def run_query(flagged_obs_hosts=False):
"""
Send casjob queries for each host
Parameters
----------
flagged_obs_hosts : bool, optional
Returns
-------
Downloads host files into sql files located in SAGADIR/hosts
"""
# RUN EITHER FULL HOST LIST OR JSUT FLAG ZERO HOSTS, default to flag zero
if flagged_obs_hosts:
sheet = GoogleSheets('1GJYuhqfKeuJr-IyyGF_NDLb_ezL6zBiX2aeZFHHPr_s', 0)
nsa_col = 'NSA'
else:
sheet = GoogleSheets('1b3k2eyFjHFDtmHce1xi6JKuj3ATOWYduTBFftx5oPp8', 448084634)
nsa_col = 'NSAID'
hostdata = sheet.load()
# FOR EACH HOST, DOWNLOAD SQL QUERY
for host in hostdata:
if flagged_obs_hosts and not host['Host Flag']:
continue
nid = host[nsa_col]
if nid == -1:
continue
print nid
# CREATE SQL QUERY
qry = construct_sdss_query(nid, host['RA'], host['Dec'])
print qry
run_casjob(qry, 'sql_nsa{}'.format(nid))
def compile_saga_spectra(flagged_obs_hosts=False):
# RUN EITHER FULL HOST LIST OR JUST FLAG ZERO HOSTS, default to flag zero
if flagged_obs_hosts:
sheet = GoogleSheets('1GJYuhqfKeuJr-IyyGF_NDLb_ezL6zBiX2aeZFHHPr_s', 0)
else:
sheet = GoogleSheets('1b3k2eyFjHFDtmHce1xi6JKuj3ATOWYduTBFftx5oPp8',
448084634)
hostdata = sheet.load()
# READ ALL NON-SDSS SPECTRA AND COMBINE INTO SINGLE TABLE
gama_table = read_saga_spectra.read_gama()
mmt_table = read_saga_spectra.read_mmt()
aat_table = read_saga_spectra.read_aat()
aat_table_mz = read_saga_spectra.read_aat_mz()
imacs_table = read_saga_spectra.read_imacs()
wiyn_table = read_saga_spectra.read_wiyn()
deimos_table = read_saga_spectra.read_deimos()
# LEAST TO MOST IMPORTANT
sagaspec = table.vstack([wiyn_table,imacs_table,gama_table,deimos_table,aat_table,mmt_table,aat_table_mz],\
metadata_conflicts='silent')
# WRITE FILE WITH ALL SPECTRA
file = os.path.join(SAGA_DIR, 'data', 'saga_spectra_raw.fits.gz')
write_fits(sagaspec, file)
# COMBINE MULTIPLE SPECTRA
# read_saga_spectra.spec_combine(sagaspec)
spec = sagaspec
sagaspec = read_saga_spectra.find_uniques(spec)
# FOR EACH FLAG ZERO HOST OR NON-FLAG ZERO OBSERVED HOST
# READ BASE SDSS FILE
# KEEP ONLY SDSS SPECTRA AND MATCHES FROM OTHER SOURCES
nhost = 0
for host in hostdata:
flag = host['flag'] # SELECT ONLY NON_FLAG0#
if flag == 0:
nra = host['RA']
ndec = host['Dec']
nsaid = host['NSAID'] # NAME OF BASE SQL FILES
basefile = os.path.join(
SAGA_DIR, 'base_catalogs',
'base_sql_nsa{0}.fits.gz'.format(nsaid))
basetable = Table.read(basefile)
print nsaid
# CALCULATE OBJECT DISTANCE FROM HOST
catsc = SkyCoord(u.Quantity(sagaspec['RA'], u.deg),
u.Quantity(sagaspec['DEC'], u.deg))
hostcoords = SkyCoord(nra * u.deg, ndec * u.deg)
seps = catsc.separation(hostcoords)
rhost = seps.to(u.deg).value
host_spec_objs = rhost < 1.0
# COMBINE SDSS SPECTRA AND SAGA SPECTRA
sdss_table = read_saga_spectra.read_sdss(basetable)
hostspec = table.vstack([sdss_table, sagaspec[host_spec_objs]])
tmp = hostspec
hostspec = read_saga_spectra.find_uniques(tmp)
id2, id1, d = sm.spherematch(hostspec['RA'],
hostspec['DEC'],
basetable['RA'],
basetable['DEC'],
3. / 3600,
nnearest=1)
basetable['TELNAME'][id1] = hostspec['TELNAME'][id2]
basetable['MASKNAME'][id1] = hostspec['MASKNAME'][id2]
basetable['ZQUALITY'][id1] = hostspec['ZQUALITY'][id2]
basetable['SPEC_Z'][id1] = hostspec['SPEC_Z'][id2]
basetable['SPEC_Z_ERR'][id1] = hostspec['SPEC_Z_ERR'][id2]
basetable['SPECOBJID'][id1] = hostspec['specobjid'][id2]
basetable['SPEC_REPEAT'][id1] = hostspec['SPEC_REPEAT'][id2]
m = basetable['SPEC_Z'] != -1
m2 = basetable['TELNAME'][m] == ''
print 'number with telname = ', np.sum(m2)
# COMBINE INTO SINGLE ALLSPEC FILE
if (nhost == 0):
allspec = basetable[m]
if (nhost > 0):
allspec = table.vstack([allspec, basetable[m]]) # APPEND
nhost = nhost + 1
# INITIALIZE SATS ARRAY (3=primary, 2=lowz, 1=satellite)
allspec = saga_tools.fill_sats_array(allspec)
# CLEAN UP REPEAT ENTRY OF SATELLITES
allspec = saga_tools.repeat_sat_cleanup(allspec)
# WRITE ALL SPECTRA TAKEN
file = os.path.join(SAGA_DIR, 'data', 'saga_spectra_dirty.fits.gz')
write_fits(allspec, file)
# WRITE ALL GOOD SPECTRA
# KEEP ALL GOOD SPECTRA WHICH ARE GALAXIES
zql = allspec['ZQUALITY'] >= 3
rml = allspec['REMOVE'] == -1
galonly = allspec['PHOTPTYPE'] == 3
clean = zql & rml & galonly
allclean = allspec[clean]
file = os.path.join(SAGA_DIR, 'data', 'saga_spectra_clean.fits.gz')
write_fits(allclean, file)
# WRITE TEXT FILE OF SATELLITES WHICH CAN BE EDITED
# file = os.path.join(SAGA_DIR, 'data', 'saga_satellites.dat')
# allsats = allclean['SATS' == 1]
# write_satellite_file(allsats,file)
return allspec
def compile_saga_spectra(flagged_obs_hosts=False):
# RUN EITHER FULL HOST LIST OR JUST FLAG ZERO HOSTS, default to flag zero
if flagged_obs_hosts:
sheet = GoogleSheets('1GJYuhqfKeuJr-IyyGF_NDLb_ezL6zBiX2aeZFHHPr_s', 0)
else:
sheet = GoogleSheets('1b3k2eyFjHFDtmHce1xi6JKuj3ATOWYduTBFftx5oPp8', 448084634)
hostdata = sheet.load()
# READ ALL NON-SDSS SPECTRA AND COMBINE INTO SINGLE TABLE
gama_table = read_saga_spectra.read_gama()
mmt_table = read_saga_spectra.read_mmt()
aat_table = read_saga_spectra.read_aat()
imacs_table = read_saga_spectra.read_imacs()
wiyn_table = read_saga_spectra.read_wiyn()
deimos_table = read_saga_spectra.read_deimos()
# LEAST TO MOST IMPORTANT
sagaspec = table.vstack([wiyn_table,imacs_table,gama_table,deimos_table,aat_table,mmt_table],\
metadata_conflicts='silent')
# zq = sagaspec['ZQUALITY'] > 2
# need to write script to resolve repeats properly
# sagaspec=sagaspec[zq]
# FOR EACH FLAG ZERO HOST OR NON-FLAG ZERO OBSERVED HOST
# READ BASE SDSS FILE
# KEEP ONLY SDSS SPECTRA AND MATCHES FROM OTHER SOURCES
nhost = 0
for host in hostdata:
flag = host['flag'] # SELECT ONLY NON_FLAG0#
if flag == 0:
nra = host['RA']
ndec = host['Dec']
nsaid = host['NSAID']# NAME OF BASE SQL FILES
basefile = os.path.join(SAGA_DIR, 'base_catalogs', 'base_sql_nsa{0}.fits.gz'.format(nsaid))
basetable = Table.read(basefile)
print nsaid
# CALCULATE OBJECT DISTANCE FROM HOST
catsc = SkyCoord(u.Quantity(sagaspec['RA'], u.deg), u.Quantity(sagaspec['DEC'], u.deg))
hostcoords = SkyCoord(nra*u.deg, ndec*u.deg)
seps = catsc.separation(hostcoords)
rhost = seps.to(u.deg).value
host_spec_objs = rhost < 1.0
# COMBINE SDSS SPECTRA AND SAGA SPECTRA
sdss_table = read_saga_spectra.read_sdss(basetable)
hostspec = table.vstack([sdss_table,sagaspec[host_spec_objs]])
# MATCH ALL SPECTRA TO BASECATALOG, ADD SPEC DETAILS
id2,id1,d = sm.spherematch(hostspec['RA'], hostspec['DEC'],basetable['RA'], basetable['DEC'],3./3600,nnearest=1)
basetable['TELNAME'][id1] = hostspec['TELNAME'][id2]
basetable['MASKNAME'][id1] = hostspec['MASKNAME'][id2]
basetable['ZQUALITY'][id1] = hostspec['ZQUALITY'][id2]
basetable['SPEC_Z'][id1] = hostspec['SPEC_Z'][id2]
basetable['SPECOBJID'][id1] = hostspec['specobjid'][id2]
basetable['SPEC_REPEAT'][id1] = hostspec['TELNAME'][id2]
m = basetable['SPEC_Z'] != -1
# COMBINE INTO SINGLE ALLSPEC FILE
if (nhost == 0):
allspec = basetable[m]
if (nhost > 0):
allspec = table.vstack([allspec,basetable[m]]) # APPEND
nhost=nhost+1
# INITIALIZE SATS ARRAY (3=primary, 2=lowz, 1=satellite)
allspec = saga_tools.fill_sats_array(allspec)
# CLEAN UP REPEAT ENTRY OF SATELLITES
allspec = saga_tools.repeat_sat_cleanup(allspec)
# HACK FOR THE MOMENT
# m1=allspec['REMOVE'] == 3
# m2=allspec['SATS'] == 1
# m=m1&m2
# allspec['REMOVE'][m] = -1 # keep remove = 3 satelites
# m1=allspec['REMOVE'] == 2
# m2=allspec['SATS'] == 1
# m=m1&m2
# allspec['SATS'][m] = -1 # remove remove=2 satellites
# WRITE ALL SPECTRA TAKEN
file = os.path.join(SAGA_DIR, 'data', 'saga_spectra_dirty.fits.gz')
write_fits(allspec, file)
# WRITE ALL GOOD SPECTRA
# KEEP ALL GOOD SPECTRA WHICH ARE GALAXIES
zql = allspec['ZQUALITY'] >= 3
rml = allspec['REMOVE'] == -1
galonly = allspec['PHOTPTYPE'] == 3
clean = zql & rml & galonly
allclean = allspec[clean]
file = os.path.join(SAGA_DIR, 'data', 'saga_spectra_clean.fits.gz')
write_fits(allclean,file)
return allspec
