def conesearch(catalog, ra, dec, radius):
'''
Returns the cone search result. It uses an auxiliary
function.
Args:
catalog (string): catalog name according to the available catalog
in the docs.
ra (float): ra coordinate of the point to search in degrees.
dec (float): dec coordinate of the point to search in degrees.
radius (float): radius to search in arcsec.
Returns:
A dictionary containing the cone search result for a single catalog.
'''
# call catsHTM cone search
match, catalog_columns, column_units = cone_search(catalog, ra, dec,
radius, path)
# no results, empty dictionary
if match.size == 0:
return {}
# generate dictionaries with response values
results = format_cone_results(match, catalog_columns, column_units)
# add catalog real name and append to final result
result_with_catname = {}
result_with_catname[catalog_map.get(catalog, catalog)] = results
return result_with_catname
def is_star_in_gaia(self, transient):
"""
match tranient position with GAIA DR2 and uses parallax
and proper motion to evaluate star-likeliness
returns: True (is a star) or False otehrwise.
"""
transient_coords = SkyCoord(transient['ra'],
transient['dec'],
unit='deg')
srcs, colnames, colunits = cone_search('GAIADR2',
transient_coords.ra.rad,
transient_coords.dec.rad,
self.gaia_rs,
catalogs_dir=self.catshtm_path)
my_keys = [
'RA', 'Dec', 'Mag_G', 'PMRA', 'ErrPMRA', 'PMDec', 'ErrPMDec',
'Plx', 'ErrPlx'
]
if len(srcs) > 0:
gaia_tab = Table(srcs, names=colnames)
gaia_tab = gaia_tab[my_keys]
gaia_coords = SkyCoord(gaia_tab['RA'], gaia_tab['Dec'], unit='rad')
# compute distance
gaia_tab['DISTANCE'] = transient_coords.separation(
gaia_coords).arcsec
gaia_tab['DISTANCE_NORM'] = (1.8 + 0.6 * exp(
(20 - gaia_tab['Mag_G']) / 2.05) > gaia_tab['DISTANCE']
) #TODO: vary mag_G exclusion
gaia_tab['FLAG_PROX'] = [
True if x['DISTANCE_NORM'] == True and
(self.gaia_veto_gmag_min <= x['Mag_G'] <=
self.gaia_veto_gmag_max) else False for x in gaia_tab
]
# check for proper motion and parallax conditioned to distance
gaia_tab['FLAG_PMRA'] = abs(
gaia_tab['PMRA'] / gaia_tab['ErrPMRA']) > self.gaia_pm_signif
gaia_tab['FLAG_PMDec'] = abs(
gaia_tab['PMDec'] / gaia_tab['ErrPMDec']) > self.gaia_pm_signif
gaia_tab['FLAG_Plx'] = abs(
gaia_tab['Plx'] / gaia_tab['ErrPlx']) > self.gaia_plx_signif
if (any(gaia_tab['FLAG_PMRA'] == True)
or any(gaia_tab['FLAG_PMDec'] == True)
or any(gaia_tab['FLAG_Plx'] == True)) and any(
gaia_tab['FLAG_PROX'] == True):
return True
return False
def crossmatch(catalog, ra, dec, radius):
'''
This function returns the crossmatch result for a catalog.
Args:
catalog (string): catalog name according to the available catalog
in the docs.
ra (float): ra coordinate of the point to search in degrees.
dec (float): dec coordinate of the point to search in degrees.
radius (float): radius to search in arcsec.
Returns:
A dictionary with the crossmatch result.
'''
# call catsHTM cone search
match, catalog_columns, column_units = cone_search(catalog, ra, dec,
radius, path)
if match.size != 0:
return format_crossmatch_results(match, catalog, ra, dec,
catalog_columns, column_units)
else:
return {}
def _make_sim_transient(inputrow, match_rad=2, search_rad=1.8, min_mag=16, max_mag=20.2, glx_search_scale=600,
glx_jitter=5, stamp_size=55, resid_every=0, rng_state=42):
"""
internal routine to generate transients
:param inputrow: row of dataframe (from label_data.get_images()) generated by iterrows()
:param match_rad: radius for MP cross-matching - should be ~1" for optimal results
:param search_rad: size of radius for initial MP crossmatch - should be ~ size of image
:param min_mag: bright limit for minor planets - remove detections that completely overpower host galaxy
:param max_mag: faint limit for minor planets - should be brighter than normal to ensure decent SNR transients
:param glx_search_scale: maximum distance from each minor planet to choose a galaxy from - coherency of PSF
:param glx_jitter: uniform scatter to apply to galaxy stamp to simulate offset transients
:param stamp_size: size of stamps to extract
:param resid_every: number of galaxy residuals
:param rng_state: testing seed for repeatability
:return:
"""
np.random.seed(rng_state)
_, row = inputrow
ut = int(row['instrument'][-1])
fits_filepath = work_to_storage_filepath(row.filepath)
try:
fits_file = fits.open(fits_filepath)
except FileNotFoundError:
try:
fits_file = fits.open(swap_dataloc(fits_filepath))
except FileNotFoundError:
logger.warning("corresponding file not found")
return
indiv_images_df = get_individual_images(row.relatedimage)
try:
indiv_fits_files = [fits.open(work_to_storage_filepath(row.filepath)) for _, row in indiv_images_df.iterrows()]
except FileNotFoundError:
try:
indiv_fits_files = [fits.open(swap_dataloc(work_to_storage_filepath(row.filepath))) for _, row in
indiv_images_df.iterrows()]
except FileNotFoundError:
logger.warning("Individual FITS files missing: {}".format(work_to_storage_filepath(row.filepath)))
return
mpc_search = query_skybot(row.ra_c, row.dec_c, search_rad, row.jd, maglim=max_mag)
try:
logger.info("found {} MPs in image".format(len(mpc_search)))
mpc_ras = list(mpc_search['RA(h)'])
mpc_decs = list(mpc_search['DE(deg)'])
mpc_coo = SkyCoord(mpc_ras, mpc_decs, unit=(u.hourangle, u.deg))
except TypeError:
logger.warning("no MPs found!")
return
try:
diff_phot_tbl = Table(fits_file[DIFFERENCE_PHOT_EXT].data)
except KeyError:
logger.warning("couldn't find {} extension in {}".format(DIFFERENCE_PHOT_EXT, fits_filepath))
return
if len(diff_phot_tbl) < 2:
logger.info('(almost) empty table found in {}[{}]'.format(fits_filepath, DIFFERENCE_PHOT_EXT))
return
diff_coo = SkyCoord(diff_phot_tbl['ra'], diff_phot_tbl['dec'], unit='deg')
try:
idx, d2d, _ = mpc_coo.match_to_catalog_sky(diff_coo)
except ValueError:
logger.error('failed with {}'.format(fits_filepath))
return
# noinspection PyUnresolvedReferences
matches = diff_phot_tbl[idx[(d2d <= match_rad * u.arcsec)]]
if 'realbogus' not in matches.colnames:
logger.warning('no realbogus column in difference photometry for {}'.format(fits_filepath))
return
# little safety net to avoid spuriously matching to RB=0 dets too often
matches = matches[matches['realbogus'] > 0.01]
logger.info('found {} mpc cross-matches'.format(len(matches)))
if matches is None:
logger.warning("no minor planets found!")
return
# optimise - query GLADE once per image to save time
glade_data, glade_cols, _ = catsHTM.cone_search("GLADE", (np.pi / 180) * row.ra_c, (np.pi / 180) * row.dec_c,
search_rad * 3600,
catalogs_dir=CATSHTM_DIR)
glade_tab = Table(glade_data, names=glade_cols)
glade_tab = glade_tab[np.isfinite(glade_tab["B"])] # clean the NaNs to avoid warnings
glade_tab = glade_tab[glade_tab["B"] < 18] # maglim cut
logger.info("{} GLADE galaxies in frame".format(len(glade_tab)))
results = []
for match in matches:
# remove detections that are too bright and saturate the detector.
if match["mag"] < min_mag:
logger.warning("MP exceeds bright threshold")
continue
glade_coords = SkyCoord(glade_tab["RA"], glade_tab["Dec"], unit=u.rad) # keep in to update table on iter
mp_coords = SkyCoord(match["ra"], match["dec"], unit=u.deg)
nn_dists = mp_coords.separation(glade_coords).to(u.arcsec).value
temp_tab = glade_tab[nn_dists < glx_search_scale]
if len(temp_tab) == 0:
logger.warning("No GLADE galaxies within {} arcsec".format(glx_search_scale))
continue
glx_sel = temp_tab[np.argmin(temp_tab["B"])] # greedily select bright galaxies
try:
glx_stamp, glx_indiv = _get_stamps(fits_file,
(180 / np.pi) * glx_sel["RA"] + np.random.uniform(-glx_jitter, glx_jitter) / 3600,
(180 / np.pi) * glx_sel["Dec"] + np.random.uniform(-glx_jitter, glx_jitter) / 3600,
stamp_size,
indiv_fits_files, return_indiv_stamps=True)
except NoOverlapError:
logger.warning("galaxy located off edge of stamp")
continue
# HACK need to work out what the problem is with this...
except (TypeError, ValueError):
logger.warning("problem with stamp extraction")
continue
try:
mp_stamp, mp_indiv = _get_stamps(fits_file, match["ra"], match["dec"], stamp_size, indiv_fits_files,
return_indiv_stamps=True)
# HACK similarly here
except (TypeError, ValueError):
logger.warning("Problem with stamp extraction")
continue
comb_stamp = mp_stamp + glx_stamp
assert mp_indiv.shape == glx_indiv.shape # crash if the stamps aren't equiv
indiv_img_comb = mp_indiv + glx_indiv
# need to respect addition of p2p stamps correctly.
comb_stamp[:, :, 3] = np.ptp(np.atleast_3d(indiv_img_comb), axis=2) # peak to peak
comb_stamp[:, :, 4] = np.min(np.atleast_3d(indiv_img_comb), axis=2) # min
comb_stamp[:, :, 5] = np.max(np.atleast_3d(indiv_img_comb), axis=2) # max
if comb_stamp is not None:
res = dict(
id=str(uuid.uuid4()),
stamps=comb_stamp,
image_id=row['id'],
x=match['x'],
y=match['y'],
ra=match['ra'],
dec=match['dec'],
mag=match['mag'],
fwhm=match['fwhm'],
realbogus=match['realbogus'],
fits_filepath=fits_filepath,
ut=ut,
ncoadd=len(indiv_fits_files),
label=1, # synthetic real transient
metalabel='syntransient'
)
results.append(res)
# avoid duplicate seed galaxies
glade_tab.remove_row(
np.int(np.argwhere((glade_tab["RA"] == glx_sel["RA"]) & (glade_tab["Dec"] == glx_sel["Dec"]))))
if len(glade_tab) == 0:
logger.info("ran out of galaxies")
return results
if resid_every == 0:
return results
glade_tab.sort("B") # sort in ascending order (i.e. bright end first)
temp_coo = SkyCoord((180/np.pi) * glade_tab["RA"], (180/np.pi) * glade_tab["Dec"], unit=u.deg)
ndets = int(min(resid_every*len(matches), len(matches))) # make sure we never exceed the number of matches
im_wcs = WCS(fits_file[1].header) # parse WCS for computing detector position
for idx, c in enumerate(temp_coo[:ndets]):
try:
glx_resid_stamp = _get_stamps(fits_file,
c.ra.value + np.random.uniform(-glx_jitter/2, glx_jitter/2) / 3600,
c.dec.value + np.random.uniform(-glx_jitter/2, glx_jitter/2) / 3600,
stamp_size, indiv_fits_files)
except NoOverlapError:
logger.warning("galaxy located off edge of stamp")
continue
# the top except should catch any problems with this.
xdet, ydet = im_wcs.all_world2pix([c.ra.value], [c.dec.value], 0)
if glx_resid_stamp is not None:
res = dict(
id=str(uuid.uuid4()),
stamps=glx_resid_stamp,
image_id=row['id'],
x=xdet[0],
y=ydet[0],
ra=c.ra.value,
dec=c.dec.value,
mag=glade_tab["B"][idx],
fwhm=99.99, # this is ill-defined here, since we're not using detections
realbogus=99.99, # similarly. b
fits_filepath=fits_filepath,
ut=ut,
ncoadd=len(indiv_fits_files),
label=0, # galaxy residual
metalabel='glxresid'
)
results.append(res)
logger.info("{} transients and residuals added".format(len(results)))
return results
def cat_search(ra_in, dec_in, srad):
ra_in, dec_in = float(ra_in), float(dec_in)
# give catsHTM rootpath
catsHTM_rootpath = par['catsHTM_rootpath']
srad = Angle(float(srad) * u.arcsec)
cats_srad = srad.arcsec
# Convert center-position into Astropy SkyCoords:
position = skycoord(ra=ra_in * u.degree,
dec=dec_in * u.degree,
frame="icrs")
def get_center_offset(pos):
# get separation between two positions
sep_angle = pos.separation(position)
return sep_angle.arcsec
# all catalogs used to check
all_catalogs = [
'TMASS', 'TMASSxsc', 'AAVSO_VSX', 'AKARI', 'APASS', 'DECaLS', 'FIRST',
'GAIADR1', 'GAIADR2', 'GALEX', 'GLADE', 'IPHAS', 'NEDz', 'PS1',
'PTFpc', 'ROSATfsc', 'SkyMapper', 'SpecSDSS', 'SAGE', 'IRACgc',
'UCAC4', 'UKIDSS', 'VISTAviking', 'VSTkids', 'WISE', 'XMM', 'simbad',
'tns'
]
# define the result table
useful_col = ['catname', 'ra', 'dec', 'offset', 'otype']
all_items_df = pd.DataFrame(columns=useful_col)
for cat in all_catalogs: # loop through all catalogs
if cat == 'simbad':
sb_cols, sb_units, sb_out, sb_success = simbad_check(
ra=position.ra.degree,
dec=position.dec.degree,
srad=srad.arcsec)
if (sb_success == 1) and (len(sb_out) > 0):
itemframe = pd.DataFrame(sb_out, columns=sb_cols)
itemframe['offset'] = [
Angle(float(off) * u.arcsec).arcsec
for off in itemframe['offset'].values
]
itemframe = itemframe[useful_col[1:]]
itemframe[useful_col[1:-1]] = itemframe[
useful_col[1:-1]].astype('float')
itemframe['catname'] = 'simbad'
itemframe = itemframe[useful_col]
itemframe = itemframe.sort_values('offset',
ascending=1).iloc[[0]]
all_items_df = pd.concat([all_items_df, itemframe], axis=0)
# if 'Galaxy' in itemframe.otype.values:
# break
elif cat == 'tns':
tns_cols, tns_units, tns_out, tns_success = tns_check(
ra=position.ra.degree,
dec=position.dec.degree,
srad=srad.arcsec)
tns_cols = [tns_cols[i].lower() for i in range(len(tns_cols))]
if tns_success == True and len(tns_out) > 0:
itemframe = pd.DataFrame(tns_out, columns=tns_cols)
itemframe['skycoord'] = skycoord(itemframe['ra'],
itemframe['dec'],
unit=u.deg,
frame='icrs')
itemframe['offset'] = itemframe['skycoord'].apply(
get_center_offset)
itemframe['otype'] = itemframe['obj. type']
itemframe = itemframe[useful_col[1:]]
itemframe['catname'] = 'tns'
itemframe = itemframe[useful_col]
itemframe = itemframe.sort_values('offset',
ascending=1).iloc[[0]]
all_items_df = pd.concat([all_items_df, itemframe], axis=0)
else:
# make sure the catalog name is string
cat = str(cat)
# cone search with catsHTM
if not cat == 'GLADE':
cat_out, colcell, colunits = catsHTM.cone_search(
cat,
position.ra.radian,
position.dec.radian,
cats_srad,
catalogs_dir=catsHTM_rootpath,
verbose=False)
else:
cat_out, colcell, colunits = catsHTM.cone_search(
cat,
position.ra.radian,
position.dec.radian,
Angle(float(30) * u.arcsec).arcsec,
catalogs_dir=catsHTM_rootpath,
verbose=False)
colcell = [colcell[i].lower() for i in range(len(colcell))]
colunits = [colunits[i].lower() for i in range(len(colunits))]
# create dict for unit of each column
colunits = {colcell[i]: colunits[i] for i in range(len(colcell))}
if len(cat_out) > 0:
# create empty cross-match result table if the cross-match result is not None
itemframe = pd.DataFrame(cat_out, columns=colcell)
# change the unit of RA and Dec as degrees
if colunits['ra'] == 'rad' and colunits['dec'] == 'rad':
itemframe['ra'] = itemframe['ra'].apply(degrees)
itemframe['dec'] = itemframe['dec'].apply(degrees)
elif colunits['ra'] == 'radians' and colunits[
'dec'] == 'radians':
itemframe['ra'] = itemframe['ra'].apply(degrees)
itemframe['dec'] = itemframe['dec'].apply(degrees)
elif colunits['ra'] == 'deg' and colunits['dec'] == 'deg':
itemframe['ra'] = itemframe['ra'].apply(float)
itemframe['dec'] = itemframe['dec'].apply(float)
# create new column for SkyCoord onjects
itemframe['skycoord'] = skycoord(itemframe['ra'],
itemframe['dec'],
unit=u.deg,
frame='icrs')
# get angular difference between the cross-matched objects and the input position
itemframe['offset'] = itemframe['skycoord'].apply(
get_center_offset)
# sort dataframe by offset
itemframe = itemframe.sort_values('offset',
ascending=1).iloc[[0]]
# define type of the crossmatched object
if cat == 'AAVSO_VSX':
itemframe['otype'] = 'VS'
elif cat == 'GLADE':
itemframe['otype'] = 'Galaxy'
else:
itemframe['otype'] = 'Unknown'
# add catalog name
itemframe['catname'] = cat
# select useful col only
itemframe = itemframe[useful_col]
all_items_df = pd.concat([all_items_df, itemframe], axis=0)
if all_items_df.empty:
return all_items_df
else:
all_items_df = all_items_df.sort_values('offset', ascending=1)
all_items_df = all_items_df.reset_index().drop("index", axis=1)
return all_items_df