def run_hostlist(nowise=False,noML=False,nosaga=False,flagged_obs_hosts=False):
"""
For each host in hostlist, create base catalog
default is to read offline Lang WISE catalogs
Parameters
----------
nowise : bool, optional. Turn off WISE catalog read
"""
# RUN EITHER FULL HOST LIST OR JSUT FLAG ZERO HOSTS, default to flag zero
sheet = GoogleSheets('1b3k2eyFjHFDtmHce1xi6JKuj3ATOWYduTBFftx5oPp8', 448084634)
flag0data = sheet.load()
file = SAGA_DROPBOX + '/hosts/submaster_all.ecsv.gz'
submaster = ascii.read(file, format='ecsv')
## print googleflag0.columns
flag0 = np.in1d(submaster['NSAID'],flag0data['NSAID'])
hostdata = submaster[flag0]
# FOR EACH HOST, READ SQL AND CREATE BASE CATALOGS
for host in hostdata:
nid = host['NSAID']
catalog = create_base_catalog(nid, host,nowise,noML,nosaga)
write_base_fits(nid, catalog)
def run_named_hosts(nowise=False,noML=False,nosaga=False):
"""
Run base catalog for SAGA named_hosts host
ACCEPTS ONLY FLAG_ZERO HOSTS FOR MOMENT!
Parameters
----------
nowise : bool, optional. Turn off WISE catalog read
"""
# sheet = GoogleSheets('1b3k2eyFjHFDtmHce1xi6JKuj3ATOWYduTBFftx5oPp8', 448084634)
# hostdata = sheet.load()
file = SAGA_DROPBOX + '/hosts/submaster_all.ecsv.gz'
hostdata = ascii.read(file, format='ecsv')
sheet = GoogleSheets('1GJYuhqfKeuJr-IyyGF_NDLb_ezL6zBiX2aeZFHHPr_s', 0)
named_hosts = sheet.load()
for sagahost in named_hosts:
nid = sagahost['NSA']
msk = hostdata['NSAID'] == nid
if np.sum(msk) == 0:
print nid#,sagahost['SAGA Name']
print 'NO HOST FOUND'
print
else:
catalog = create_base_catalog(nid, hostdata[msk],nowise,noML,nosaga)
write_base_fits(nid, catalog)
def run_query(flagged_obs_hosts=False):
"""
Send casjob queries for each host
Parameters
----------
flagged_obs_hosts : bool, optional
Returns
-------
Downloads host files into sql files located in SAGADIR/hosts
"""
# RUN EITHER FULL HOST LIST OR JSUT FLAG ZERO HOSTS, default to flag zero
if flagged_obs_hosts:
sheet = GoogleSheets('1GJYuhqfKeuJr-IyyGF_NDLb_ezL6zBiX2aeZFHHPr_s', 0)
else:
sheet = GoogleSheets('1b3k2eyFjHFDtmHce1xi6JKuj3ATOWYduTBFftx5oPp8',
448084634)
hostdata = sheet.load()
# FOR EACH HOST, DOWNLOAD SQL QUERY
for host in hostdata:
if flagged_obs_hosts and not host['Host Flag']:
continue
nid = host['NSAID']
if nid == -1:
continue
print nid
# CREATE SQL QUERY
qry = construct_sdss_query(nid, host['RA'], host['Dec'])
print qry
run_casjob(qry, 'sql_nsa{}'.format(nid))
def run_query(flagged_obs_hosts=False):
"""
Send casjob queries for each host
Parameters
----------
flagged_obs_hosts : bool, optional
Returns
-------
Downloads host files into sql files located in SAGADIR/hosts
"""
# RUN EITHER FULL HOST LIST OR JSUT FLAG ZERO HOSTS, default to flag zero
if flagged_obs_hosts:
sheet = GoogleSheets('1GJYuhqfKeuJr-IyyGF_NDLb_ezL6zBiX2aeZFHHPr_s', 0)
nsa_col = 'NSA'
else:
sheet = GoogleSheets('1b3k2eyFjHFDtmHce1xi6JKuj3ATOWYduTBFftx5oPp8', 448084634)
nsa_col = 'NSAID'
hostdata = sheet.load()
# FOR EACH HOST, DOWNLOAD SQL QUERY
for host in hostdata:
if flagged_obs_hosts and not host['Host Flag']:
continue
nid = host[nsa_col]
if nid == -1:
continue
print nid
# CREATE SQL QUERY
qry = construct_sdss_query(nid, host['RA'], host['Dec'])
print qry
run_casjob(qry, 'sql_nsa{}'.format(nid))
import numpy as np
import os
import re
from astropy.io import ascii
from astropy import table
from astropy.table import Table
from tabulate import tabulate
import pyspherematch as sm
from FileLoader import GoogleSheets
SAGA_DIR = os.environ['SAGA_DIR']
REMOVELIST = GoogleSheets('1Y3nO7VyU4jDiBPawCs8wJQt2s_PIAKRj-HSrmcWeQZo', 1379081675, header_start=1)
############################################################################
def calc_general():
# CALCULATE NUMBERS FOR MW SIM SECTION
r =21
d1 = 0.01*3e5/70.
d2 = 0.0045*3e5/70.
dmin = 5*np.log10(20. *1e6) - 5. #Mpc
dmax = 5*np.log10(45. *1e6) - 5.#Mpc
Mr_min = 21 - dmin
Mr_max = 21 - dmax
print "Min/max survey distance ", d1,d2
def compile_saga_spectra(flagged_obs_hosts=False):
# RUN EITHER FULL HOST LIST OR JUST FLAG ZERO HOSTS, default to flag zero
if flagged_obs_hosts:
sheet = GoogleSheets('1GJYuhqfKeuJr-IyyGF_NDLb_ezL6zBiX2aeZFHHPr_s', 0)
else:
sheet = GoogleSheets('1b3k2eyFjHFDtmHce1xi6JKuj3ATOWYduTBFftx5oPp8',
448084634)
hostdata = sheet.load()
# READ ALL NON-SDSS SPECTRA AND COMBINE INTO SINGLE TABLE
gama_table = read_saga_spectra.read_gama()
mmt_table = read_saga_spectra.read_mmt()
aat_table = read_saga_spectra.read_aat()
aat_table_mz = read_saga_spectra.read_aat_mz()
imacs_table = read_saga_spectra.read_imacs()
wiyn_table = read_saga_spectra.read_wiyn()
deimos_table = read_saga_spectra.read_deimos()
# LEAST TO MOST IMPORTANT
sagaspec = table.vstack([wiyn_table,imacs_table,gama_table,deimos_table,aat_table,mmt_table,aat_table_mz],\
metadata_conflicts='silent')
# WRITE FILE WITH ALL SPECTRA
file = os.path.join(SAGA_DIR, 'data', 'saga_spectra_raw.fits.gz')
write_fits(sagaspec, file)
# COMBINE MULTIPLE SPECTRA
# read_saga_spectra.spec_combine(sagaspec)
spec = sagaspec
sagaspec = read_saga_spectra.find_uniques(spec)
# FOR EACH FLAG ZERO HOST OR NON-FLAG ZERO OBSERVED HOST
# READ BASE SDSS FILE
# KEEP ONLY SDSS SPECTRA AND MATCHES FROM OTHER SOURCES
nhost = 0
for host in hostdata:
flag = host['flag'] # SELECT ONLY NON_FLAG0#
if flag == 0:
nra = host['RA']
ndec = host['Dec']
nsaid = host['NSAID'] # NAME OF BASE SQL FILES
basefile = os.path.join(
SAGA_DIR, 'base_catalogs',
'base_sql_nsa{0}.fits.gz'.format(nsaid))
basetable = Table.read(basefile)
print nsaid
# CALCULATE OBJECT DISTANCE FROM HOST
catsc = SkyCoord(u.Quantity(sagaspec['RA'], u.deg),
u.Quantity(sagaspec['DEC'], u.deg))
hostcoords = SkyCoord(nra * u.deg, ndec * u.deg)
seps = catsc.separation(hostcoords)
rhost = seps.to(u.deg).value
host_spec_objs = rhost < 1.0
# COMBINE SDSS SPECTRA AND SAGA SPECTRA
sdss_table = read_saga_spectra.read_sdss(basetable)
hostspec = table.vstack([sdss_table, sagaspec[host_spec_objs]])
tmp = hostspec
hostspec = read_saga_spectra.find_uniques(tmp)
id2, id1, d = sm.spherematch(hostspec['RA'],
hostspec['DEC'],
basetable['RA'],
basetable['DEC'],
3. / 3600,
nnearest=1)
basetable['TELNAME'][id1] = hostspec['TELNAME'][id2]
basetable['MASKNAME'][id1] = hostspec['MASKNAME'][id2]
basetable['ZQUALITY'][id1] = hostspec['ZQUALITY'][id2]
basetable['SPEC_Z'][id1] = hostspec['SPEC_Z'][id2]
basetable['SPEC_Z_ERR'][id1] = hostspec['SPEC_Z_ERR'][id2]
basetable['SPECOBJID'][id1] = hostspec['specobjid'][id2]
basetable['SPEC_REPEAT'][id1] = hostspec['SPEC_REPEAT'][id2]
m = basetable['SPEC_Z'] != -1
m2 = basetable['TELNAME'][m] == ''
print 'number with telname = ', np.sum(m2)
# COMBINE INTO SINGLE ALLSPEC FILE
if (nhost == 0):
allspec = basetable[m]
if (nhost > 0):
allspec = table.vstack([allspec, basetable[m]]) # APPEND
nhost = nhost + 1
# INITIALIZE SATS ARRAY (3=primary, 2=lowz, 1=satellite)
allspec = saga_tools.fill_sats_array(allspec)
# CLEAN UP REPEAT ENTRY OF SATELLITES
allspec = saga_tools.repeat_sat_cleanup(allspec)
# WRITE ALL SPECTRA TAKEN
file = os.path.join(SAGA_DIR, 'data', 'saga_spectra_dirty.fits.gz')
write_fits(allspec, file)
# WRITE ALL GOOD SPECTRA
# KEEP ALL GOOD SPECTRA WHICH ARE GALAXIES
zql = allspec['ZQUALITY'] >= 3
rml = allspec['REMOVE'] == -1
galonly = allspec['PHOTPTYPE'] == 3
clean = zql & rml & galonly
allclean = allspec[clean]
file = os.path.join(SAGA_DIR, 'data', 'saga_spectra_clean.fits.gz')
write_fits(allclean, file)
# WRITE TEXT FILE OF SATELLITES WHICH CAN BE EDITED
# file = os.path.join(SAGA_DIR, 'data', 'saga_satellites.dat')
# allsats = allclean['SATS' == 1]
# write_satellite_file(allsats,file)
return allspec
def compile_saga_spectra(flagged_obs_hosts=False):
# RUN EITHER FULL HOST LIST OR JUST FLAG ZERO HOSTS, default to flag zero
if flagged_obs_hosts:
sheet = GoogleSheets('1GJYuhqfKeuJr-IyyGF_NDLb_ezL6zBiX2aeZFHHPr_s', 0)
else:
sheet = GoogleSheets('1b3k2eyFjHFDtmHce1xi6JKuj3ATOWYduTBFftx5oPp8', 448084634)
hostdata = sheet.load()
# READ ALL NON-SDSS SPECTRA AND COMBINE INTO SINGLE TABLE
gama_table = read_saga_spectra.read_gama()
mmt_table = read_saga_spectra.read_mmt()
aat_table = read_saga_spectra.read_aat()
imacs_table = read_saga_spectra.read_imacs()
wiyn_table = read_saga_spectra.read_wiyn()
deimos_table = read_saga_spectra.read_deimos()
# LEAST TO MOST IMPORTANT
sagaspec = table.vstack([wiyn_table,imacs_table,gama_table,deimos_table,aat_table,mmt_table],\
metadata_conflicts='silent')
# zq = sagaspec['ZQUALITY'] > 2
# need to write script to resolve repeats properly
# sagaspec=sagaspec[zq]
# FOR EACH FLAG ZERO HOST OR NON-FLAG ZERO OBSERVED HOST
# READ BASE SDSS FILE
# KEEP ONLY SDSS SPECTRA AND MATCHES FROM OTHER SOURCES
nhost = 0
for host in hostdata:
flag = host['flag'] # SELECT ONLY NON_FLAG0#
if flag == 0:
nra = host['RA']
ndec = host['Dec']
nsaid = host['NSAID']# NAME OF BASE SQL FILES
basefile = os.path.join(SAGA_DIR, 'base_catalogs', 'base_sql_nsa{0}.fits.gz'.format(nsaid))
basetable = Table.read(basefile)
print nsaid
# CALCULATE OBJECT DISTANCE FROM HOST
catsc = SkyCoord(u.Quantity(sagaspec['RA'], u.deg), u.Quantity(sagaspec['DEC'], u.deg))
hostcoords = SkyCoord(nra*u.deg, ndec*u.deg)
seps = catsc.separation(hostcoords)
rhost = seps.to(u.deg).value
host_spec_objs = rhost < 1.0
# COMBINE SDSS SPECTRA AND SAGA SPECTRA
sdss_table = read_saga_spectra.read_sdss(basetable)
hostspec = table.vstack([sdss_table,sagaspec[host_spec_objs]])
# MATCH ALL SPECTRA TO BASECATALOG, ADD SPEC DETAILS
id2,id1,d = sm.spherematch(hostspec['RA'], hostspec['DEC'],basetable['RA'], basetable['DEC'],3./3600,nnearest=1)
basetable['TELNAME'][id1] = hostspec['TELNAME'][id2]
basetable['MASKNAME'][id1] = hostspec['MASKNAME'][id2]
basetable['ZQUALITY'][id1] = hostspec['ZQUALITY'][id2]
basetable['SPEC_Z'][id1] = hostspec['SPEC_Z'][id2]
basetable['SPECOBJID'][id1] = hostspec['specobjid'][id2]
basetable['SPEC_REPEAT'][id1] = hostspec['TELNAME'][id2]
m = basetable['SPEC_Z'] != -1
# COMBINE INTO SINGLE ALLSPEC FILE
if (nhost == 0):
allspec = basetable[m]
if (nhost > 0):
allspec = table.vstack([allspec,basetable[m]]) # APPEND
nhost=nhost+1
# INITIALIZE SATS ARRAY (3=primary, 2=lowz, 1=satellite)
allspec = saga_tools.fill_sats_array(allspec)
# CLEAN UP REPEAT ENTRY OF SATELLITES
allspec = saga_tools.repeat_sat_cleanup(allspec)
# HACK FOR THE MOMENT
# m1=allspec['REMOVE'] == 3
# m2=allspec['SATS'] == 1
# m=m1&m2
# allspec['REMOVE'][m] = -1 # keep remove = 3 satelites
# m1=allspec['REMOVE'] == 2
# m2=allspec['SATS'] == 1
# m=m1&m2
# allspec['SATS'][m] = -1 # remove remove=2 satellites
# WRITE ALL SPECTRA TAKEN
file = os.path.join(SAGA_DIR, 'data', 'saga_spectra_dirty.fits.gz')
write_fits(allspec, file)
# WRITE ALL GOOD SPECTRA
# KEEP ALL GOOD SPECTRA WHICH ARE GALAXIES
zql = allspec['ZQUALITY'] >= 3
rml = allspec['REMOVE'] == -1
galonly = allspec['PHOTPTYPE'] == 3
clean = zql & rml & galonly
allclean = allspec[clean]
file = os.path.join(SAGA_DIR, 'data', 'saga_spectra_clean.fits.gz')
write_fits(allclean,file)
return allspec
from astropy.coordinates import SkyCoord
from astropy import units as u
import saga_tools
import fix_byhand
import pyspherematch as sm
from FileLoader import GoogleSheets, FitsTable
# SET-UP DIRECTORIES AND FILES TO BE LOADED
#SAGA_DIR = os.getenv('SAGA_DIR', os.curdir)
SAGA_DIR = os.environ['SAGA_DIR']
SAGA_DROPBOX = os.environ['SAGA_DROPBOX']
REMOVELIST = GoogleSheets('1Y3nO7VyU4jDiBPawCs8wJQt2s_PIAKRj-HSrmcWeQZo', 1379081675, header_start=1)
ADDLIST = GoogleSheets('1Y3nO7VyU4jDiBPawCs8wJQt2s_PIAKRj-HSrmcWeQZo', 286645731, header_start=1)
SAGANAMES = GoogleSheets('1GJYuhqfKeuJr-IyyGF_NDLb_ezL6zBiX2aeZFHHPr_s', 0, header_start=0)
NSACAT = FitsTable(os.path.join(SAGA_DIR, 'cats', 'nsa_v0_1_3.fits'))
SAGACAT = FitsTable(os.path.join(SAGA_DIR, 'data', 'saga_spectra_dirty.fits.gz'))
CLEAN = FitsTable(os.path.join(SAGA_DIR, 'data', 'saga_spectra_clean.fits.gz'))
MLFILE = '/Users/marlageha/Dropbox/SAGA/data/SAGA.objid.noclean.PROBS_WISE_PROBS.sept28.fits.gz'
ML = Table.read(MLFILE)
##################################
def run_hostlist(nowise=False,noML=False,nosaga=False,flagged_obs_hosts=False):
"""
For each host in hostlist, create base catalog
default is to read offline Lang WISE catalogs
