def create_source_catalog(version="2.1.3",
make_continuum=True,
save=True,
dsky=4.0):
global config
detects_line_table = Table.read('detect_hdr{}.fits'.format(version))
# detects = Detections(curated_version=version)
# detects_line_table = detects.return_astropy_table()
# detects_line_table.write('test.tab', format='ascii')
detects_line_table.add_column(
Column(str("line"), name="det_type", dtype=str))
detects_cont = Detections(catalog_type="continuum")
sel1 = detects_cont.remove_bad_amps()
sel2 = detects_cont.remove_meteors()
sel3 = detects_cont.remove_shots()
sel4 = detects_cont.remove_bad_detects()
sel5 = detects_cont.remove_large_gal()
sel6 = detects_cont.throughput > 0.08
sel_field = ((detects_cont.field == "cosmos")
| (detects_cont.field == "dex-fall")
| (detects_cont.field == "dex-spring")
| (detects_cont.field == "egs")
| (detects_cont.field == "goods-n"))
detects_cont_table = detects_cont[sel1 * sel2 * sel3 * sel4 * sel5 * sel6 *
sel_field].return_astropy_table()
detects_cont_table.add_column(
Column(str("cont"), name="det_type", dtype=str))
if make_continuum:
detects_cont_table.write("continuum_" + version + ".fits",
overwrite=True)
detects_cont_table.write("continuum_" + version + ".tab",
overwrite=True,
format='ascii')
dets_all = Detections().refine()
sel_tp = dets_all.throughput > 0.08
sel_field = ((dets_all.field == "cosmos")
| (dets_all.field == "dex-fall")
| (dets_all.field == "dex-spring")
| (dets_all.field == "egs")
| (dets_all.field == "goods-n"))
dets_all_table = dets_all[sel_tp * sel_field].return_astropy_table()
dets_all_table.add_column(Column(str("line"), name="det_type", dtype=str))
agn_tab = Table.read(config.agncat,
format="ascii",
include_names=['detectid', 'flux_LyA'])
# add in continuum sources to match to Chenxu's combined catalog
#detects_cont_table_orig = detects_cont[sel1 * sel2 * sel3].return_astropy_table()
dets_all_table = vstack([dets_all_table, detects_cont_table])
detects_broad_table = join(agn_tab,
dets_all_table,
join_type="inner",
keys=["detectid"])
dets_all.close()
del dets_all_table
global detect_table
detect_table = unique(vstack(
[detects_broad_table, detects_cont_table, detects_line_table]),
keys='detectid')
# add fiber_ratio
fiber_ratio = []
for row in detect_table:
det = row['detectid']
det_type = row['det_type']
try:
fiber_ratio.append(return_fiber_ratio(det, det_type))
except:
fiber_ratio.append(np.nan)
print('fiber_ratio failed for {}'.format(det))
detect_table['fiber_ratio'] = fiber_ratio
detect_table.write('test.fits', overwrite=True)
del detects_cont_table, detects_broad_table
# calculate ebv and av for every detections
all_coords = SkyCoord(ra=detect_table['ra'],
dec=detect_table['dec'],
unit='deg')
sfd = SFDQuery()
ebv = sfd(all_coords)
Rv = 3.1
corr_SF2011 = 2.742 # Landolt V
ext = []
Av = corr_SF2011 * ebv
detect_table['Av'] = Av
detect_table['ebv'] = ebv
detect_table = add_elixer_cat_info(detect_table, version)
#print('Adding 1sigma noise from flux limits')
#p = Pool(24)
#res = p.map(get_flux_noise_1sigma, detect_table['detectid'])
#p.close()
#flim = []
#flim_update = []
#for r in res:
# flim.append(r[0])
# flim_update.append(r[1])
#detect_table['flux_noise_1sigma'] = flim_update
#detect_table['flux_noise_1sigma_orig'] = flim
detect_table.write('test2.fits', overwrite=True)
print("Performing FOF in 3D space with linking length=6 arcsec")
# get fluxes to derive flux-weighted distribution of group
detect_table["gmag"][np.isnan(detect_table["gmag"])] = 27
gmag = detect_table["gmag"] * u.AB
flux = gmag.to(u.Jy).value
sel_line = detect_table['det_type'] == 'line' # remove continuum sources
# first cluster in positional/wavelegnth space in a larger
# linking length
kdtree, r = fof.mktree(detect_table["ra"][sel_line],
detect_table["dec"][sel_line],
detect_table["wave"][sel_line],
dsky=6.0,
dwave=8.0)
t0 = time.time()
print("starting fof ...")
wfriend_lst = fof.frinds_of_friends(kdtree, r, Nmin=2)
t1 = time.time()
wfriend_table = fof.process_group_list(
wfriend_lst,
detect_table["detectid"][sel_line],
detect_table["ra"][sel_line],
detect_table["dec"][sel_line],
detect_table["wave"][sel_line],
detect_table['flux'][sel_line],
)
print("Generating combined table \n")
memberlist = []
friendlist = []
for row in wfriend_table:
friendid = row["id"]
members = np.array(row["members"])
friendlist.extend(friendid * np.ones_like(members))
memberlist.extend(members)
wfriend_table.remove_column("members")
wdetfriend_tab = Table()
wdetfriend_tab.add_column(Column(np.array(friendlist), name="id"))
wdetfriend_tab.add_column(Column(memberlist, name="detectid"))
wdetfriend_all = join(wdetfriend_tab, wfriend_table, keys="id")
wdetfriend_all['wave_group_id'] = wdetfriend_all['id'] + 213000000
wdetfriend_all.rename_column('size', 'wave_group_size')
wdetfriend_all.rename_column('a', 'wave_group_a')
wdetfriend_all.rename_column('b', 'wave_group_b')
wdetfriend_all.rename_column('pa', 'wave_group_pa')
wdetfriend_all.rename_column('icx', 'wave_group_ra')
wdetfriend_all.rename_column('icy', 'wave_group_dec')
wdetfriend_all.rename_column('icz', 'wave_group_wave')
w_keep = wdetfriend_all['detectid', 'wave_group_id', 'wave_group_a',
'wave_group_b', 'wave_group_pa', 'wave_group_ra',
'wave_group_dec', 'wave_group_wave']
print("3D FOF analysis complete in {:3.2f} minutes \n".format(
(t1 - t0) / 60))
print("Performing FOF in 2D space with dlink=3.5 arcsec")
kdtree, r = fof.mktree(
detect_table["ra"],
detect_table["dec"],
np.zeros_like(detect_table["ra"]),
dsky=3.5,
)
t0 = time.time()
print("starting fof ...")
friend_lst = fof.frinds_of_friends(kdtree, r, Nmin=1)
t1 = time.time()
print("FOF analysis complete in {:3.2f} minutes \n".format((t1 - t0) / 60))
friend_table = fof.process_group_list(
friend_lst,
detect_table["detectid"],
detect_table["ra"],
detect_table["dec"],
0.0 * detect_table["wave"],
flux,
)
print("Generating combined table \n")
memberlist = []
friendlist = []
for row in friend_table:
friendid = row["id"]
members = np.array(row["members"])
friendlist.extend(friendid * np.ones_like(members))
memberlist.extend(members)
friend_table.remove_column("members")
detfriend_tab = Table()
detfriend_tab.add_column(Column(np.array(friendlist), name="id"))
detfriend_tab.add_column(Column(memberlist, name="detectid"))
detfriend_all = join(detfriend_tab, friend_table, keys="id")
del detfriend_tab
# match detectids at large linking length if a wave group exists
joinfriend = join(detfriend_all, w_keep, keys='detectid', join_type='left')
grp_by_id = joinfriend.group_by('id')
sum_grp = grp_by_id.groups.aggregate(np.sum)
spatial_id = grp_by_id.groups.keys
spatial_id_to_keep1 = spatial_id[np.isfinite(sum_grp['wave_group_id'])]
# also match if a group member is brighter than gmag=21 (testing this)
sel_bright = detect_table['gmag'] < 21
gmag_join = join(joinfriend, detect_table['detectid', 'gmag'][sel_bright])
spatial_id_to_keep2 = gmag_join['id']
spatial_id_to_keep = vstack([spatial_id_to_keep1, spatial_id_to_keep2])
detfriend_1 = join(unique(spatial_id_to_keep), joinfriend)
# link the rest of the detectids with smaller linking length
keep_row = np.ones(np.size(detect_table), dtype=bool)
for i, det in enumerate(detect_table['detectid']):
if det in detfriend_1['detectid']:
keep_row[i] = 0
print("Performing FOF in 2D space with dlink=1.0 arcsec")
kdtree, r = fof.mktree(
detect_table["ra"][keep_row],
detect_table["dec"][keep_row],
np.zeros_like(detect_table["ra"][keep_row]),
dsky=1.0,
)
t0 = time.time()
print("starting fof ...")
friend_lst = fof.frinds_of_friends(kdtree, r, Nmin=1)
t1 = time.time()
print("Final FOF analysis complete in {:3.2f} minutes \n".format(
(t1 - t0) / 60))
friend_table = fof.process_group_list(
friend_lst,
detect_table["detectid"][keep_row],
detect_table["ra"][keep_row],
detect_table["dec"][keep_row],
0.0 * detect_table["wave"][keep_row],
flux[keep_row],
)
print("Generating combined table \n")
memberlist = []
friendlist = []
for row in friend_table:
friendid = row["id"]
members = np.array(row["members"])
friendlist.extend(friendid * np.ones_like(members))
memberlist.extend(members)
friend_table.remove_column("members")
detfriend_tab = Table()
detfriend_tab.add_column(Column(np.array(friendlist), name="id"))
detfriend_tab.add_column(Column(memberlist, name="detectid"))
detfriend_2 = join(detfriend_tab, friend_table, keys="id")
starting_id_1 = int(version.replace('.', '', 2)) * 10**10
starting_id_2 = starting_id_1 + 10**8
detfriend_1.add_column(Column(detfriend_1["id"] + starting_id_1,
name="source_id"),
index=0)
detfriend_2.add_column(Column(detfriend_2["id"] + starting_id_2,
name="source_id"),
index=0)
detfriend_1.remove_column("id")
detfriend_2.remove_column("id")
detfriend_all = vstack([detfriend_1, detfriend_2])
expand_table = join(detfriend_all, detect_table, keys="detectid")
expand_table['wave_group_id'] = expand_table['wave_group_id'].filled(0)
expand_table.remove_column('detectname')
expand_table.write('test3.fits', overwrite=True)
# combine common wavegroups to the same source_id
sel = expand_table['wave_group_id'] > 0
s_by_id = expand_table[sel].group_by('wave_group_id')
for grp in s_by_id.groups:
sid, ns = np.unique(grp['source_id'], return_counts=True)
wid = grp['wave_group_id'][0]
if np.size(sid) > 1:
sel_wid = expand_table['wave_group_id'] == wid
expand_table['source_id'][sel_wid] = sid[0]
del detfriend_all, detect_table, friend_table
gaia_stars = Table.read(config.gaiacat)
gaia_coords = SkyCoord(ra=gaia_stars["ra"] * u.deg,
dec=gaia_stars["dec"] * u.deg)
src_coords = SkyCoord(ra=expand_table["ra"] * u.deg,
dec=expand_table["dec"] * u.deg)
idx, d2d, d3d = src_coords.match_to_catalog_sky(gaia_coords)
sel = d2d < 5.0 * u.arcsec
gaia_match_name = np.zeros_like(expand_table["source_id"], dtype=int)
gaia_match_name[sel] = gaia_stars["source_id"][idx][sel]
gaia_match_dist = np.zeros_like(expand_table["source_id"], dtype=float)
gaia_match_dist[sel] = d2d[sel].to_value(u.arcsec)
expand_table["gaia_match_id"] = gaia_match_name
expand_table["gaia_match_dist"] = gaia_match_dist
expand_table.rename_column("size", "n_members")
expand_table.rename_column("icx", "ra_mean")
expand_table.rename_column("icy", "dec_mean")
expand_table.sort("source_id")
# fill mask values with nans
for col in expand_table.columns:
try:
expand_table[col] = expand_table[col].filled(np.nan)
print('yes', col)
except:
pass
#print('no', col)
return expand_table
def create_source_catalog(version="2.1.3",
make_continuum=True,
save=True,
dsky=4.0):
global config
detects_line_table = Table.read('detect_hdr{}.fits'.format(version))
# detects = Detections(curated_version=version)
# detects_line_table = detects.return_astropy_table()
# detects_line_table.write('test.tab', format='ascii')
detects_line_table.add_column(
Column(str("line"), name="det_type", dtype=str))
detects_cont = Detections(catalog_type="continuum")
sel1 = detects_cont.remove_bad_amps()
sel2 = detects_cont.remove_meteors()
sel3 = detects_cont.remove_shots()
sel4 = detects_cont.remove_bad_detects()
sel5 = detects_cont.remove_large_gal()
detects_cont_table = detects_cont[sel1 * sel2 * sel3 * sel4 *
sel5].return_astropy_table()
detects_cont_table.add_column(
Column(str("cont"), name="det_type", dtype=str))
if make_continuum:
detects_cont_table.write("continuum_" + version + ".fits",
overwrite=True)
dets_all = Detections().refine()
dets_all_table = dets_all.return_astropy_table()
agn_tab = Table.read(config.agncat,
format="ascii",
include_names=["detectid"])
# add in continuum sources to match to Chenxu's combined catalog
detects_cont_table_orig = detects_cont[sel1 * sel2 *
sel3].return_astropy_table()
dets_all_table = vstack([dets_all_table, detects_cont_table_orig])
detects_broad_table = join(agn_tab,
dets_all_table,
join_type="inner",
keys=["detectid"])
detects_broad_table.add_column(
Column(str("agn"), name="det_type", dtype=str))
dets_all.close()
del dets_all_table
detect_table = vstack(
[detects_line_table, detects_broad_table, detects_cont_table])
del detects_cont_table_orig, detects_cont_table, detects_broad_table
kdtree, r = fof.mktree(
detect_table["ra"],
detect_table["dec"],
np.zeros_like(detect_table["ra"]),
dsky=dsky,
)
t0 = time.time()
print("starting fof ...")
friend_lst = fof.frinds_of_friends(kdtree, r, Nmin=1)
t1 = time.time()
print("FOF analysis complete in {:3.2f} minutes \n".format((t1 - t0) / 60))
# get fluxes to derive flux-weighted distribution of group
detect_table["gmag"][np.isnan(detect_table["gmag"])] = 27
gmag = detect_table["gmag"] * u.AB
flux = gmag.to(u.Jy).value
friend_table = fof.process_group_list(
friend_lst,
detect_table["detectid"],
detect_table["ra"],
detect_table["dec"],
0.0 * detect_table["wave"],
flux,
)
print("Generating combined table \n")
memberlist = []
friendlist = []
for row in friend_table:
friendid = row["id"]
members = np.array(row["members"])
friendlist.extend(friendid * np.ones_like(members))
memberlist.extend(members)
friend_table.remove_column("members")
detfriend_tab = Table()
detfriend_tab.add_column(Column(np.array(friendlist), name="id"))
detfriend_tab.add_column(Column(memberlist, name="detectid"))
detfriend_all = join(detfriend_tab, friend_table, keys="id")
del detfriend_tab
expand_table = join(detfriend_all, detect_table, keys="detectid")
del detfriend_all, detect_table, friend_table
starting_id = int(version.replace('.', '', 2)) * 10**10
expand_table.add_column(Column(expand_table["id"] + starting_id,
name="source_id"),
index=0)
expand_table.remove_column("id")
gaia_stars = Table.read(config.gaiacat)
gaia_coords = SkyCoord(ra=gaia_stars["ra"] * u.deg,
dec=gaia_stars["dec"] * u.deg)
src_coords = SkyCoord(ra=expand_table["ra"] * u.deg,
dec=expand_table["dec"] * u.deg)
idx, d2d, d3d = src_coords.match_to_catalog_sky(gaia_coords)
sel = d2d < 5.0 * u.arcsec
gaia_match_name = np.zeros_like(expand_table["source_id"], dtype=int)
gaia_match_name[sel] = gaia_stars["source_id"][idx][sel]
gaia_match_dist = np.zeros_like(expand_table["source_id"], dtype=float)
gaia_match_dist[sel] = d2d[sel]
expand_table["gaia_match_id"] = gaia_match_name
expand_table["gaia_match_dist"] = gaia_match_dist
expand_table.rename_column("size", "n_members")
expand_table.rename_column("icx", "ra_mean")
expand_table.rename_column("icy", "dec_mean")
expand_table.sort("source_id")
return expand_table