def main():
obj_loc = [252.7534888, 25.8759437] #Holds ra, dec in degrees
get_ref = False #Get reference image?
get_dif = True #Get difference images?
dist = 0.01 #get all observations within dist from obj_loc (in degrees)
#For the query note the following:
#fid=1->zg, 2->zr, 3->zi filter
#dates are in jd
#obsjd BETWEEN start_jd and stop_jd, or obsjd>, obsjd=, etc.
#seeing<2 for seeing below 2 arcsec
t1 = time.Time(58355, format='mjd').jd
t2 = time.Time(58365, format='mjd').jd
t3 = time.Time(58590, format='mjd').jd
t4 = time.Time(58750, format='mjd').jd
q1 = 'fid=2 and obsjd BETWEEN {} and {}'.format(
t1, t2) #Get r im at peak for this obj
#q2='fid=2 and obsjd BETWEEN {} and {}'.format(t3,t4) #Get r ims just before and during possible late-time detects
#Try to get the 3 interesting observations in 1 go
t = time.Time([58503, 58504, 58543, 58544, 58577, 58578], format='mjd').jd
q1 = 'fid=2 and (obsjd BETWEEN {} and {} or obsjd BETWEEN {} and {} or obsjd BETWEEN {} and {})'.format(
t[0], t[1], t[2], t[3], t[4], t[5])
print(q1)
if get_ref:
zquery = query.ZTFQuery()
zquery.load_metadata(kind='ref', radec=obj_loc)
zquery.download_data()
if get_dif:
zquery = query.ZTFQuery()
#zquery2=query.ZTFQuery()
zquery.load_metadata(radec=obj_loc, size=dist, sql_query=q1)
#zquery2.load_metadata(radec=obj_loc, size=dist, sql_query=q2)
zquery.download_data('scimrefdiffimg.fits.fz')
#zquery2.download_data('scimrefdiffimg.fits.fz')
return
def download_images(radec, size, sql_query, download_data): #Downloads the (meta-)data, and returns the meta-data
#Get the meta-data and sort them to be in chronologically order
refquery = query.ZTFQuery()
refquery.load_metadata(kind='ref', radec=radec)
difquery = query.ZTFQuery()
difquery.load_metadata(radec=radec, size=size, sql_query=sql_query)
difquery.metatable.sort_values(by=['obsjd'], inplace = True)
#get the images if wanted
if download_data:
refquery.download_data()
difquery.download_data('scimrefdiffimg.fits.fz')
return refquery, difquery
def download_metadata(corner_df, metadata_file=None):
"""
Download ZTF metadata corresponding to a set of points in a dataframe. If the metadata
are already downloaded read from the metadata_file instead
Parameters
----------
corner_df : `pd.DataFrame`
a pandas dataframe with columns ra, dec in degrees
metadata_file : string
absolute path to a csv file with the columns `expid` and `rcid`
specifying the unique set of images to download.
Returns
-------
unique_table: `pd.DataFrame`
a dataframe of unique images with their entire metadata as downloaded using
, download_few=download_few `ZTFQuery`.
"""
if metadata_file is None:
zquery = query.ZTFQuery()
dfs = []
for (ra, dec) in corner_df[['ra', 'dec']].values:
zquery.load_metadata(radec=[ra, dec], size=0.01)
dfs.append(zquery.metatable)
meta = pd.concat(dfs)
else:
meta = pd.read_csv(metadata_file)
# Note in many cases these duplicates might already be
# filtered and the drop statement not needed.
return meta.drop_duplicates(subset=['expid', 'rcid'])
def query_metadata(ra,
dec,
username,
password,
start_jd=None,
end_jd=None,
out_csv=None):
"""Use ZTFquery to get more reliable upper limits"""
zquery = query.ZTFQuery()
if start_jd == None and end_jd == None:
zquery.load_metadata(kind='sci',
radec=[str(ra.deg), str(dec.deg)],
size=0.003,
auth=[username, password])
else:
if start_jd != None and end_jd == None:
sql_query = 'obsjd>' + repr(start_jd)
elif start_jd == None and end_jd != None:
sql_query = 'obsjd<' + repr(end_jd)
elif start_jd != None and end_jd != None:
sql_query = 'obsjd<' + repr(end_jd) + '+AND+' + 'obsjd>' + repr(
start_jd)
zquery.load_metadata(kind='sci',
radec=[str(ra.deg), str(dec.deg)],
size=0.003,
sql_query=sql_query,
auth=[username, password])
out = zquery.metatable
final_out = out.sort_values(by=['obsjd'])
if out_csv is not None:
final_out.to_csv(out_csv)
return final_out
def download_images(self):
'''
Get all relevant images, download if needed & wanted,
and list them in chronological order.
Refquery is for the refence images, difquery for the difference images.
Returns:
finallist: list of locations of each image
'''
refquery = query.ZTFQuery()
refquery.load_metadata(kind='ref', radec=[self.ra, self.dec])
difquery = query.ZTFQuery()
difquery.load_metadata(radec=[self.ra, self.dec], size=self.size,
sql_query=self.sql)
difquery.metatable.sort_values(by=['obsjd'], inplace = True)
if self.download_data: #Download images if it is requested.
refquery.download_data()
difquery.download_data('scimrefdiffimg.fits.fz')
finallist = refquery.get_local_data() + difquery.get_local_data(
'scimrefdiffimg.fits.fz')
return finallist
__adv__ = 'download_query.py'
print("ziff download_query is deprecated. Use 'irsa_query.py' instead.")
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--query",
type=str,
default="obsjd BETWEEN 2458554.5 AND 2458564")
parser.add_argument("--overwrite", type=int, default=1)
parser.add_argument("--nprocess", type=int, default=1)
args = parser.parse_args()
in_query = args.query
from ztfquery import query
print(in_query)
zquery = query.ZTFQuery()
zquery.load_metadata(sql_query=in_query)
print(zquery.metatable)
keys = ['sciimg.fits', 'mskimg.fits', 'psfcat.fits']
for _key in keys:
zquery.download_data(_key,
show_progress=True,
notebook=False,
nprocess=args.nprocess,
overwrite=bool(args.overwrite))
# End of download_query.py ========================================================
def select_variability(tbl, hard_reject=[], update_database=False,
read_database=True,
use_forced_phot=False, stacked=False,
baseline=0.02, var_baseline={'g': 6, 'r': 8, 'i': 10},
max_duration_tot=30., max_days_g=1e5, snr=4,
index_rise=-0.0, index_decay=0.0,
path_secrets_db='db_access.csv',
save_plot=False, path_plot='./',
show_plot=False, use_metadata=False,
path_secrets_meta='../kowalski/secrets.csv',
save_csv=False, path_csv='./',
path_forced='./forced_phot/'):
"""
Select candidates based on their duration and on their evolution rate.
---
Parameters
tbl astropy table
table with the photometry of the candidates
from the AVRO packets. It can be created using get_lc_kowalski.py
and it usually includes:
name,ra,dec,jd,magpsf,sigmapsf,filter,magzpsci,magzpsciunc,programid,
field,rcid,pid,sgscore1,sgscore2,sgscore3,distpsnr1,distpsnr2,distpsnr3
hard_reject list of str
list of candidates that have to be ignored
update_database bool
if True, it updates the psql database with the results
read_database bool
if True, it reads light curves from the psql database
use_forced_phot bool
if True, forced ForcePhotZTF photometry will be used;
if False, only alerts will be considered.
stacked bool
if True, the light curve will be stacked nightly in flux
baseline float
min time (days) between first and last detection for the fit
to be performed (e.g.: baseline=2. means that if a transient
is found only on a single night, it will not be fit to find
its rise and decay rates)
var_baseline dict
if no evolution (1-sigma) is measured beyond these thresholds,
the candidate is "rejected". Different filters have different
thresholds, the default being {'g': 6, 'r': 8, 'i': 10}
max_duration_tot float
max time (days) allowed between the first and last detections
in any band
max_days_g float
max time (days) allowed between the first and last detections
in g band
snr float
min signal-to-noise ratio for forced photometry data points;
the maxlike method of ForcePhotZTF tends to underestimate the
errors, so snr=4 is preferred over snr=3.
index_rise float
negative number, minimum rise rate (mag/day) for a candidate to
be selected, if the rise index can be measured at all
(e.g.: index_rise=-0.5 will allow you to select only
those candidates rising faster than 0.5 mag/day).
index_decay float
positive number, minimum decay rate (mag/day) for a candidate to
be selected, if the decay index can be measured at all
(e.g.: index_decay=0.3 will allow you to select only
those candidates decaying faster than 0.3 mag/day).
path_secrets_db str
path to the CSV secrets file to access the psql db.
The file will need to have:
db,host,dbname,port,user,password
path_forced str
path to the directory where forced photometry light curves are stored
save_plot bool
save the plot to a png file
path_plot str
path to the folder where to save the plot
show_plot
display the plot while the code is running.
The code will resume running when the plot is manually closed.
use_metadata bool
if True, use ztfquery to fetch upper limits based onZTF pipeline
non-detection of the transient (for plotting only)
path_secrets_meta str
path to the CSV secrets file to access ztfquery.
The file will need to have:
ztfquery_user, ztfquery_pwd
save_csv bool
if True, the light curve is saved in a CSV file
path_csv str
path to the folder where the light curve will be saved
---
Returns
if update_database=True, it automatically updates the database with the
following information:
duration_tot, duration per band, rise or decay rate (index) per band
selected list of str
list of candidates that meet the input selection criteria
rejected
list of candidates that don't meet the input selection criteria
cantsay
list of candidates without enough information to tell
if they meet the input selection criteria
"""
# Useful definitions, specific for ZTF
candidates = set(tbl["name"])
filters = ['g', 'r', 'i']
filters_id = {'1': 'g', '2': 'r', '3': 'i'}
colors = {'g': 'g', 'r': 'r', 'i': 'y'}
if update_database is True or read_database is True:
# Connect to psql db
con, cur = connect_database(update_database=update_database,
path_secrets_db=path_secrets_db)
if save_plot is True:
if not os.path.isdir(path_plot):
os.makedirs(path_plot)
if save_csv is True:
if not os.path.isdir(path_csv):
os.makedirs(path_csv)
names_select = []
names_reject = []
empty_lc = []
# Get forced phot for all the candidates
if read_database is True and use_forced_phot is True:
str_names = "'" + "','".join(candidates) + "'"
# table name
if stacked is True:
table_name = "lightcurve_stacked"
column_names = "name, jd, flux, flux_unc, mag, mag_unc, \
limmag, filter, zp, ezp, programid, field, ccdid, qid"
else:
table_name = "lightcurve_forced"
column_names = "name, jd, filter, programid, \
field, mag, mag_unc, limmag, zp, ezp, flux_maxlike, flux_maxlike_unc"
# Read the light curve from the database
t_pd = pd.read_sql_query(f"SELECT {column_names} \
from {table_name} \
where name IN ({str_names})", con)
# If the table is empty, return
if t_pd.empty:
print("There is no forced photometry in the database \
for any of the given candidates!")
return None, None, None
t_forced = Table.from_pandas(t_pd)
for name in candidates:
# Is the candidate to be ignored?
if name in hard_reject:
continue
# Check if the forced photometry light curve is available
if use_forced_phot is True:
with_forced_phot = True
if read_database is True:
t = t_forced[t_forced['name'] == name]
else:
# Read the light curve from a file
files = glob.glob(f"{path_forced}/*{name}*maxlike*fits")
if len(files) == 0:
print(f"No forced photometry available for {name}: skipping")
continue
elif len(files) > 1:
print(f"WARNING: more than one light curve found for {name}")
print(f"Using {files[0]}")
filename = files[0]
else:
filename = files[0]
t = Table(fits.open(filename)[1].data)
empty = False
if len(t) == 0:
empty = True
empty_lc.append(name)
print(f"Empty forced photometry light curve for {name}: skipping")
continue
# Keep going with only the alerts?
#t = tbl[tbl['name'] == name]
#t_ul = t[:0].copy()
else:
t_ul = t[t["mag"] > 50]
t = t[t["mag"] < 50]
# Fix the column names
t.rename_column('mag', 'magpsf')
t.rename_column('mag_unc', 'sigmapsf')
if read_database is False:
t.rename_column('jdobs', 'jd')
t_ul.rename_column('jdobs', 'jd')
# Add missing epochs packets
for l in tbl[tbl['name'] == name]:
if len(t_ul) > 0:
min_delta_ul = np.min(np.abs(t_ul['jd'] - l['jd']))
else:
min_delta_ul = np.inf
if len(t) > 0:
min_delta_det = np.min(np.abs(t['jd'] - l['jd']))
else:
min_delta_det = np.inf
# Min time from forced photometry for using alerts
if stacked is True:
# 12 hours for stacked images
min_delta = 12./24.
else:
# 15 seconds for non-stacked light curves
min_delta = 15./24./60./60.
if np.min([min_delta_ul, min_delta_det]) > min_delta:
if stacked is False and read_database is False:
new_row = [l['jd'], l['filter'], np.nan,
np.nan, np.nan, np.nan, 1, l['field'],
l['rcid'], np.nan, np.nan, np.nan,
np.nan, np.nan, np.nan, np.nan, np.nan,
np.nan, np.nan, np.nan, '', '', np.nan,
np.nan, np.nan, np.nan, np.nan,
l['magpsf'], l['sigmapsf'], np.nan]
elif stacked is False and read_database is True:
new_row = [name, l['jd'], l['filter'], 1,
l['field'], l['magpsf'], l['sigmapsf'],
99., 0., 0., 0., 0.]
elif stacked is True and read_database is True:
new_row = [name, l['jd'], np.nan, np.nan,
l['magpsf'], l['sigmapsf'], 99.,
l['filter'], 0., 0., l['programid'], l['field'], 0., 0.]
else:
new_row = [l['jd'], np.nan, np.nan, np.nan, np.nan,
l['magpsf'], l['sigmapsf'], np.nan,
l['filter'], 1]
t.add_row(new_row)
else:
with_forced_phot = False
empty = False
t = tbl[tbl['name'] == name]
t_ul = t[:0].copy()
# Reject those with only upper limits
if len(t) == 0 and empty is False:
names_reject.append(name)
continue
# Determine the light curve starting time
t0 = min(t["jd"])
# Reject if the overall duration is longer than ?? days
# or if there is only 1 detection
try:
if update_database is True:
# FIXME do we want to report only the max duration_tot between alerts, forcephot, and stack?
cur.execute(f"UPDATE candidate SET \
duration_tot = {np.max(t['jd']) - np.min(t['jd'])}\
where name = '{name}'")
if np.max(t['jd']) - np.min(t['jd']) > max_duration_tot or np.max(t['jd']) - np.min(t['jd']) == 0:
names_reject.append(name)
continue
except ValueError:
print("Failed calculating max(t['jd']) - min(t['jd']) > 10.")
pdb.set_trace()
try:
plt.close()
except:
pass
#plt.clf()
plt.figure(figsize=(8,6))
plt.subplot(1, 1, 1)
plotted = False
#print(f"-------- {name}")
for f in filters:
tf = t[t['filter'] == f]
if len(tf) == 0:
continue
if use_metadata is False:
tf_ul = t_ul[t_ul['filter'] == f]
if len(tf_ul) > 0:
tf_ul["jd"] = tf_ul["jd"] - t0
plt.plot(np.array(tf_ul["jd"]), np.array(tf_ul["limmag"]),
colors[f]+'v', markeredgecolor=colors[f],
markerfacecolor='w')
plt.plot([],[], 'kv', label='UL')
# Correct the start time
tf["jd"] = tf["jd"] - t0
#brightest, faintest detections
bright = np.min(tf["magpsf"])
try:
brighterr = tf["sigmapsf"][tf["magpsf"] == bright][0]
except:
print("problems with",
"brighterr = tf['sigmapsf'][tf['magpsf'] == bright][0]")
print(tf)
pdb.set_trace()
continue
bright_jd = tf["jd"][tf["magpsf"] == bright][0]
faint = np.max(tf["magpsf"])
fainterr = tf["sigmapsf"][tf["magpsf"] == faint][0]
faint_jd = tf["jd"][tf["magpsf"] == faint][0]
# First and last detections
first = np.min(tf["jd"])
last = np.max(tf["jd"])
# Add the information regarding the duration in the db
# duration_g is the max number of days between the first
# detection and the last detection in g band.
if update_database is True:
cur.execute(f"UPDATE candidate SET \
duration_{f} = {last-first} \
where name = '{name}'")
time = np.array(tf["jd"])
mag = np.array(tf["magpsf"])
magerr = np.array(tf["sigmapsf"])
plt.errorbar(np.array(tf["jd"][tf['programid']!=1]),
np.array(tf["magpsf"][tf['programid']!=1]),
fmt=colors[f]+'s',
yerr=np.array(tf["sigmapsf"][tf['programid']!=1]),
markeredgecolor='k', markersize=8)
plt.errorbar(np.array(tf["jd"][tf['programid']==1]),
np.array(tf["magpsf"][tf['programid']==1]),
fmt=colors[f]+'o',
yerr=np.array(tf["sigmapsf"][tf['programid']==1]),
markeredgecolor='k', markersize=8)
plt.plot([],[], 'ks', label='programid=2,3')
plt.plot([],[], 'ko', label='programid=1')
# max_days_g is the max number of days between the first
# detection in any band and the last g-band detection
if update_database is True:
cur.execute(f"UPDATE candidate SET \
max_days_{f} = {np.max(tf['jd'])} \
where name = '{name}'")
# SELECT: not enough baseline - no action taken
if np.abs(last-first) < baseline:
continue
# SELECT: no variability between the first and last detection - rejection
if bright+brighterr > faint-fainterr and np.abs(bright_jd - faint_jd) >= var_baseline[f]:
names_reject.append(name)
continue
# SELECT: if a g-band detection is present xx days
# after the first detection, reject
if f == 'g' and np.max(tf['jd']) > max_days_g:
names_reject.append(name)
continue
onlyrise = False
onlyfade = False
if bright_jd < first + baseline:
onlyfade = True
riseorfade = 'fade'
# Some info may be stored that we want to remove
if update_database is True and with_forced_phot is False:
cur.execute(f"UPDATE candidate SET \
index_rise_{f} = NULL \
where name = '{name}'")
elif update_database is True and with_forced_phot is True:
if stacked is True:
column = f"index_rise_stack_{f}"
else:
column = f"index_rise_forced_{f}"
cur.execute(f"UPDATE candidate SET \
{column} = NULL \
where name = '{name}'")
elif bright_jd > last - baseline:
onlyrise = True
riseorfade = 'rise'
if update_database is True and with_forced_phot is False:
cur.execute(f"UPDATE candidate SET \
index_fade_{f} = NULL \
where name = '{name}'")
elif update_database is True and with_forced_phot is True:
if stacked is True:
column = f"index_fade_stack_{f}"
else:
column = f"index_fade_forced_{f}"
cur.execute(f"UPDATE candidate SET \
{column} = NULL \
where name = '{name}'")
# Fit
fitfunc = lambda p, x: p[0] + p[1] * x
errfunc = lambda p, x, y, err: (y - fitfunc(p, x)) / err
pinit = [1.0, -1.0]
if onlyrise or onlyfade:
pfinal, covar, index, amp, indexErr, ampErr = do_fit(errfunc, pinit, time, mag, magerr)
plt.plot(time, fitfunc(pfinal, time), color=colors[f],
label=f"{f}, index= {'{:.2f}'.format(index)}+-{'{:.2f}'.format(indexErr)}")
# Add info to the database
if update_database is True and with_forced_phot is False:
cur.execute(f"UPDATE candidate SET \
index_{riseorfade}_{f} = {index} \
where name = '{name}'")
elif update_database is True and with_forced_phot is True:
if stacked is True:
column = f"index_{riseorfade}_stack_{f}"
else:
column = f"index_{riseorfade}_forced_{f}"
cur.execute(f"UPDATE candidate SET \
{column} = {index} \
where name = '{name}'")
# SELECT: slow evolution over a time longer than the baseline
if ((index > 0 and index <= index_decay) and
(last-first) > var_baseline[f]):
names_reject.append(name)
else:
plotted = True
else:
indexrise = np.where(time <= bright_jd)
indexfade = np.where(time >= bright_jd)
for i, riseorfade in zip([indexrise, indexfade], ['rise', 'fade']):
time_new = time[i[0]]
mag_new = mag[i[0]]
magerr_new = magerr[i[0]]
faint_new = np.max(mag_new)
fainterr_new = magerr_new[np.where(mag_new == faint_new)[0]]
faint_jd_new = time_new[np.where(mag_new == faint_new)[0]]
# Format check
if type(magerr_new) is list or type(magerr_new) is np.ndarray:
magerr_new = magerr_new[0]
if type(fainterr_new) is list or type(fainterr_new) is np.ndarray:
fainterr_new = fainterr_new[0]
if type(faint_jd_new) is list or type(faint_jd_new) is np.ndarray:
faint_jd_new = faint_jd_new[0]
# SELECT: no evolution
try:
if (bright+brighterr > faint_new - fainterr_new and
np.abs(faint_jd_new - bright_jd) >= var_baseline[f]):
names_reject.append(name)
plt.errorbar(time, mag, yerr=magerr,
fmt=colors[f]+'.',
markeredgecolor='k',
markersize=8)
continue
except:
print(bright, brighterr, faint_new, fainterr_new)
pdb.set_trace()
pfinal, covar, index, amp, indexErr, ampErr = do_fit(errfunc, pinit, time_new, mag_new, magerr_new)
plt.plot(time_new, fitfunc(pfinal, time_new),
color=colors[f],
label=f"{f}, index= {'{:.2f}'.format(index)}+-{'{:.2f}'.format(indexErr)}")
if update_database is True and with_forced_phot is False:
cur.execute(f"UPDATE candidate SET \
index_{riseorfade}_{f} = {index} \
where name = '{name}'")
elif update_database is True and with_forced_phot is True:
if stacked is True:
column = f"index_{riseorfade}_stack_{f}"
else:
column = f"index_{riseorfade}_forced_{f}"
cur.execute(f"UPDATE candidate SET \
{column} = NULL \
where name = '{name}'")
# SELECT: slow evolution
if ((index > 0 and index <= index_decay) and
(last-first) > var_baseline[f]):
names_reject.append(name)
else:
plotted = True
if name in names_reject:
plt.close()
continue
if plotted is True:
# The candidate was selected!
names_select.append(name)
if plotted is True and (show_plot is True or save_plot is True):
# The following is for the file naming
if use_forced_phot is True:
forcedbool = 1
else:
forcedbool = 0
if stacked is True:
stackbool = 1
else:
stackbool = 0
if use_metadata:
# Fetch metadata on all the available ZTF images
#start_jd = Time('2018-03-01 00:00:00', format='iso').jd
#end_jd = Time('2030-02-01 04:50:59.998', format='iso').jd
start_jd, end_jd = None, None
# Read the secrets
secrets = ascii.read(path_secrets_meta, format='csv')
username = secrets['ztfquery_user'][0]
password = secrets['ztfquery_pwd'][0]
zquery = query.ZTFQuery()
coords = SkyCoord(ra=np.mean(tbl[tbl['name'] == name]['ra']*u.deg),
dec=np.mean(tbl[tbl['name'] == name]['dec']*u.deg))
metadata = query_metadata(coords.ra, coords.dec, username, password,
zquery, start_jd=start_jd, end_jd=end_jd,
out_csv=None)
t_ul = Table([[],[],[],[],[],[],[],[]],
names=('jd', 'magpsf', 'sigmapsf', 'filter',
'snr', 'limmag', 'seeing', 'programid'),
dtype=('double','f','f','S','f','f','f','int'))
for j, ml, fid, s, pid in zip(metadata['obsjd'],
metadata['maglimit'],
metadata['fid'],
metadata['seeing'],
metadata['pid']):
if not (j in t['jd']):
new_row = [j, 99.9, 99.9, filters_id[str(fid)],
np.nan, ml, s, 0]
t_ul.add_row(new_row)
for f in filters:
tf_ul = t_ul[t_ul['filter'] == f]
if len(tf_ul) > 0:
tf_ul["jd"] = tf_ul["jd"] - t0
plt.plot(np.array(tf_ul["jd"]), np.array(tf_ul["limmag"]),
colors[f]+'v', markeredgecolor=colors[f],
markerfacecolor='w')
plt.plot([],[], 'kv', label='UL')
plt.title(f"{name}")
plt.xlabel('Time [days]', fontsize=18)
plt.ylabel('mag', fontsize=18)
plt.tick_params(axis='both', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
labelsize=16)
handles, labels = plt.gca().get_legend_handles_labels()
by_label = OrderedDict(zip(labels, handles))
plt.legend(by_label.values(), by_label.keys(), fontsize=16)
plt.gca().invert_yaxis()
if save_plot is True:
# Save the image
plt.savefig(f"{path_plot}/lc_{name}_forced{forcedbool}_stacked{stackbool}.png")
if save_csv is True:
# Save the light curve
t_union = vstack([t, t_ul])
ascii.write(t_union,
f"{path_csv}/lc_{name}_forced{forcedbool}_stacked{stackbool}.csv",
format='csv', overwrite=True)
if show_plot is True:
plt.show()
else:
plt.close()
if update_database is True:
# Update the hard_reject flag
names_reject_str = "','".join(list(names_reject))
cur.execute(f"UPDATE candidate SET \
hard_reject = 1 \
where name IN ('{names_reject_str}')")
# Commit the changes
con.commit()
# Close the connection
cur.close()
con.close()
print(f"{len(set(empty_lc))} empty light curves")
print(f"Select {set(names_select)}")
print(f"Reject {set(names_reject)}")
cantsay = list(n for n in candidates if
not (n in names_select) and not (n in names_reject))
print(f"Cannot say {set(cantsay)}")
print(f"{len(set(names_reject))}/{len(candidates)} objects rejected")
print(f"{len(set(names_select))}/{len(candidates)} objects selected")
print(f"{len(set(cantsay))}/{len(candidates)} objects cannot say")
return names_select, names_reject, cantsay
def download_images_diffpsf_refdiff(targetdir,
ra1,
dec1,
start_jd=None,
open_check=True):
'''
Download subtracted images and psf images from irsa
This function will create two folders to store fits files downloaded from ipac:
difference images go to --> targetdir/images_refdiff/
psf images go to --> targetdir/images_diffpsf/
ra1, dec1 are the coordinates of the target
set start_jd = None if you want all images in ZTF history
set open_check = True, the function will try to opeb all files in the final step.
Sometimes (although very seldom) the fits file can be broken.
'''
try:
os.stat(targetdir)
except:
os.mkdir(targetdir)
subdir1 = os.path.dirname(targetdir + '/images_diffpsf/')
try:
os.stat(subdir1)
except:
os.mkdir(subdir1)
subdir2 = os.path.dirname(targetdir + '/images_refdiff/')
try:
os.stat(subdir2)
except:
os.mkdir(subdir2)
############## Get metadata ocf all images at this location ################
zquery = query.ZTFQuery()
print('\n')
print("Querying for metadata...")
if start_jd == None:
zquery.load_metadata(kind='sci', radec=[ra1, dec1], size=0.003)
else:
zquery.load_metadata(kind='sci',
radec=[ra1, dec1],
size=0.003,
sql_query='obsjd>' + repr(start_jd))
out = zquery.metatable
final_out = out.sort_values(by=['obsjd'])
final_out.to_csv(targetdir + '/irsafile.csv')
urls, dl_loc = zquery.download_data(nodl=True)
urls = np.array(urls)
dl_loc = np.array(dl_loc)
print('Trying to download %d images from irsa...' % len(urls))
for i in range(len(urls)):
if i % 50 == 0:
print('In progress: %d in %d' % (i, len(urls)))
_url = urls[i]
_url_diffpsf = _url.split('sciimg')[0] + 'diffimgpsf.fits'
_url_ref = _url.split('sciimg')[0] + 'scimrefdiffimg.fits.fz'
fitsfile1 = _url_diffpsf.split('/')[-1]
fitsfile2 = _url_ref.split('/')[-1]
savename1 = subdir1 + '/' + fitsfile1.split('.fits')[0] + '.fits'
savename2 = subdir2 + '/' + fitsfile2.split('.fits')[0] + '.fits'
if os.path.isfile(savename1) == False:
download_single_url(_url_diffpsf,
fileout=savename1,
cookies=None,
verbose=False)
if os.path.isfile(savename2) == False:
download_single_url(_url_ref,
fileout=savename2,
cookies=None,
verbose=False)
################## we do not have access to some images ###################
original_names = []
for i in range(len(urls)):
original_names.append(urls[i].split('/')[-1].split('_sciimg.fits')[0])
original_names = np.array(original_names)
argo = np.argsort(original_names)
original_names = original_names[argo]
durls = glob.glob(subdir1 + '/*.fits')
downloaded_names = []
for i in range(len(durls)):
downloaded_names.append(
durls[i].split('/')[-1].split('_diffimgpsf.fits')[0])
downloaded_names = np.array(downloaded_names)
argd = np.argsort(downloaded_names)
downloaded_names = downloaded_names[argd]
ix_why = np.in1d(original_names, downloaded_names)
print('%d images in %d we do not have data:' %
(np.sum(~ix_why), len(urls)))
print('\n')
###################### check if files can be opened #######################
if open_check == True:
print('checking if all files can be opened...')
imgdir = targetdir + '/images_refdiff/'
psfdir = targetdir + '/images_diffpsf/'
imgfiles = np.array(glob.glob(imgdir + '*.fits'))
arg = np.argsort(imgfiles)
imgfiles = imgfiles[arg]
psffiles = np.array(glob.glob(psfdir + '*.fits'))
arg = np.argsort(psffiles)
psffiles = psffiles[arg]
n = len(imgfiles)
if len(imgfiles) != len(psffiles):
imgstrings = [x.split('/')[-1][:-20] for x in imgfiles]
psfstrings = [x.split('/')[-1][:-16] for x in psffiles]
imgstrings = np.array(imgstrings)
psfstrings = np.array(psfstrings)
if len(imgstrings) > len(psfstrings):
ix = np.in1d(imgstrings, psfstrings)
for x in imgfiles[~ix]:
os.remove(x)
else:
ix = np.in1d(psfstrings, imgstrings)
for x in psffiles[~ix]:
os.remove(x)
imgdir = targetdir + '/images_refdiff/'
psfdir = targetdir + '/images_diffpsf/'
imgfiles = np.array(glob.glob(imgdir + '*.fits'))
arg = np.argsort(imgfiles)
imgfiles = imgfiles[arg]
psffiles = np.array(glob.glob(psfdir + '*.fits'))
arg = np.argsort(psffiles)
psffiles = psffiles[arg]
n = len(imgfiles)
for i in range(n):
if i % 50 == 0:
print('In progress: %d in %d...' % (i, n))
imgpath = imgfiles[i]
psfpath = psffiles[i]
try:
fits.open(imgpath)[1].header
fits.open(imgpath)[1].data
fits.open(psfpath)[0].data
except:
print('file broken, remove %s, %s' % (imgpath, psffiles[i]))
os.remove(imgfiles[i])
os.remove(psffiles[i])
print('\n')
def get_finder(ra,
dec,
name,
rad=0.01,
target_mag=np.nan,
starlist=None,
print_starlist=True,
telescope="P200",
main_comment="",
minmag=15,
maxmag=18.5,
figdir=None):
"""
Aim: Generate finder chart
Parameters:
ra: float or string.
eg: ra="18h24m25.36s", 210.437583
dec: float or string, type must be consistent with ra.
eg: dec="+44d07m50.0s", 46.215583
name: ZTFname or host name.
eg: name="18aaslhxt_h", "ZTF18abclfee"
rad: search radius in the unit of degree
target_mag: magnitude of the target at time of observation,
default shoudl be r band
starlist: name of the starlist
eg: starlist="/Users/yuhanyao/Desktop/observation/20190428_LRIS/starlist"
References:
Keck starlist format: https://www2.keck.hawaii.edu/observing/starlist.html
First field is the target name in columns 1-16 (tabs and spaces allowed). Maximum length is 15 characters.
Next three space-separated tokens (beginning in column 17) are RA in the form HH MM SS.SS (including an arbitrary number of decimal places).
Next three space-separated tokens are Dec in the form (-)DD MM SS.S (again, to an arbitrary number of decimal places). Note: if the Dec is between 0 and -1, the DD field MUST be -00).
Next token is equinox in the form YYYY.YY (no parentheses; arbitrary number of decimal places), with <=1950 meaning FK4 and 2000 meaning FK5 and APP meaning apparent.
"""
print('Using search radius of %.1f arcsec.' % (rad * 3600))
name = str(name)
#assert len(name) in [12, 11]
assert type(ra) == type(dec)
if type(ra) == str:
c1 = SkyCoord(ra, dec, frame='icrs')
ra = c1.ra.degree
dec = c1.dec.degree
ra = float(ra)
dec = float(dec)
# prepare to print starlist
if telescope == "Keck":
name_length = 16 # maximum length of name is 15
commentchar = "#"
separator = ""
rah_length = 11
elif telescope == "P200":
name_length = 20
commentchar = "!"
separator = "|"
rah_length = 12
#Write to the starlist if the name of the starlist was provided.
rah, dech = deg2hour(ra, dec, sep=" ")
if (not starlist is None):
s_target_mag = "yyao: (r={0:.1f})".format(target_mag)
if telescope == "Keck":
with open(starlist, "a") as f:
f.write("{:s}{:s} {:s} 2000.0 {:s} {:s} {:s} {:s} \n".format(
name.ljust(name_length), rah.ljust(rah_length), dech,
commentchar, main_comment, separator, s_target_mag))
f.close()
elif telescope == "P200":
with open(starlist, "a") as f:
f.write("{:s}{:s} {:s} 2000.0 {:s} {:s} {:s} {:s} \n".format(
name.ljust(name_length), rah.ljust(rah_length), dech,
commentchar, main_comment, separator, s_target_mag))
f.close()
# Get metadata of all images at this location
print("Querying for metadata...")
zquery = query.ZTFQuery()
zquery.load_metadata(radec=[ra, dec], size=rad)
out = zquery.metatable
# Do you need to use a reference image?
need_ref = len(out) == 0
if need_ref or name[:4] == "ZTFJ":
print("Using a reference image")
imfile, catfile = choose_ref(zquery, ra, dec)
else:
print("Using a science image")
imfile, catfile = choose_sci(zquery, out, name, ra, dec)
# get the cutout
inputf = pyfits.open(imfile)
im = inputf[0].data
inputf.close()
head = fits.getheader(imfile)
# Get the x and y position of the target,
# as per the IPAC catalog
wcs = astropy.wcs.WCS(head)
world = np.array([[ra, dec]], np.float_)
target_pix = wcs.wcs_world2pix(world, 0)[0]
xpos = target_pix[0]
ypos = target_pix[1]
# adjust counts
im[np.isnan(im)] = 0
im[im > 30000] = 30000
# extract 600x600 region around the position of the target
width = 600
height = 600
xmax = xpos + width / 2
xmin = xpos - width / 2
ymax = ypos + height / 2
ymin = ypos - height / 2
plt.figure(figsize=(8, 6))
plt.set_cmap('gray_r')
smoothedimage = gaussian_filter(im, 1.3)
# pad the image
im_padded = np.pad(smoothedimage, 300, mode='constant', constant_values=0)
# If it's a reference image, you have to flip it up/down and left/right
if need_ref:
croppedimage = np.fliplr(
np.flipud(im_padded[int(ymin) + 300:int(ymax) + 300,
int(xmin) + 300:int(xmax) + 300]))
# If it's a science image, you just flip it up/down
else:
croppedimage = np.flipud(im_padded[int(ymin) + 300:int(ymax) + 300,
int(xmin) + 300:int(xmax) + 300])
plt.imshow(
croppedimage,
origin='lower', # convention for IPAC images
vmin=np.percentile(im.flatten(), 10),
vmax=np.percentile(im.flatten(), 99.0))
# Mark target: should just be the center of the image, now
# horizontal line
plt.plot([300 + 5, 300 + 20], [300, 300], 'g-', lw=2)
# vertical line
plt.plot([300, 300], [300 + 5, 300 + 20], 'g-', lw=2)
# and the offset of the original coordinate system with the new coordinates
offset_x = xpos - 300
offset_y = ypos - 300
# Choose offset stars
cat = pyfits.open(catfile)[1].data
zp = pyfits.open(catfile)[0].header['MAGZP']
sep_pix = np.sqrt(
(xpos-cat['xpos'])**2 + \
(ypos-cat['ypos'])**2)
# should be separated by at least 10 pixels
crit_a = np.logical_and(sep_pix > 10, cat['flags'] == 0)
crit_b = np.logical_and(cat['chi'] < 2, cat['snr'] > 10)
crit_c = cat['sharp'] < 0.3
crit_ab = np.logical_and(crit_a, crit_b)
crit = np.logical_and(crit_ab, crit_c)
# should be bright
mag_crit = np.logical_and(cat['mag'] + zp >= minmag,
cat['mag'] + zp <= maxmag)
choose_ind = np.where(np.logical_and(crit, mag_crit))
# mark the closest three stars
nref = 3
order = np.argsort(sep_pix[choose_ind])
cols = ['orange', 'purple', 'red']
for ii in np.arange(nref):
ref_xpos_original = cat['xpos'][choose_ind][order][ii] - offset_x
ref_ypos_original = cat['ypos'][choose_ind][order][ii] - offset_y
# transform to flipped plot
if need_ref:
ref_xpos = 600 - ref_xpos_original
ref_ypos = 600 - ref_ypos_original
else:
ref_xpos = ref_xpos_original
ref_ypos = 600 - ref_ypos_original
plt.plot([ref_xpos + 5, ref_xpos + 20], [ref_ypos, ref_ypos],
c=cols[ii],
ls='-',
lw=2)
plt.plot([ref_xpos, ref_xpos], [ref_ypos + 5, ref_ypos + 20],
c=cols[ii],
ls='-',
lw=2)
refra = cat['ra'][choose_ind][order][ii]
refdec = cat['dec'][choose_ind][order][ii]
if telescope == 'Keck':
refrah, refdech = deg2hour(refra, refdec, sep=" ")
elif telescope == 'P200':
refrah, refdech = deg2hour(refra, refdec, sep=":")
else:
print("I don't recognize this telescope")
refmag = cat['mag'][choose_ind][order][ii] + zp
dra, ddec = get_offset(refra, refdec, ra, dec)
offsetnum = 0.2
plt.text(1.02,
0.60 - offsetnum * ii,
'Ref S%s, mag %s' % ((ii + 1), np.round(refmag, 1)),
transform=plt.axes().transAxes,
fontweight='bold',
color=cols[ii])
plt.text(1.02,
0.55 - offsetnum * ii,
'%s %s' % (refrah, refdech),
color=cols[ii],
transform=plt.axes().transAxes)
plt.text(1.02,
0.50 - offsetnum * ii,
str(np.round(ddec, 2)) + "'' N, " + str(np.round(dra, 2)) +
"'' E",
color=cols[ii],
transform=plt.axes().transAxes)
# Print starlist for telescope
if telescope == 'Keck':
# Target name is columns 1-16
# RA must begin in 17, separated by spaces
print(
"{:s}{:s} {:s} 2000.0 {:s} raoffset={:.2f} decoffset={:.2f} {:s} r={:.1f} "
.format((name + "_S%s" % (ii + 1)).ljust(name_length),
refrah.ljust(rah_length), refdech, separator, dra,
ddec, commentchar, refmag))
elif telescope == 'P200':
print(
"{:s}{:s} {:s} 2000.0 {:s} raoffset={:.2f} decoffset={:.2f} r={:.1f} \n"
.format((name + "_S%s" % (ii + 1)).ljust(name_length),
refrah.ljust(rah_length), refdech, commentchar, dra,
ddec, refmag))
# and save to file if starlist name is provided
if (not starlist is None):
if telescope == "Keck":
with open(starlist, "a") as f:
f.write(
"{:s}{:s} {:s} 2000.0 {:s} raoffset={:.2f} decoffset={:.2f} {:s} r={:.1f} \n"
.format((name + "_S%s" % (ii + 1)).ljust(name_length),
refrah.ljust(rah_length), refdech, commentchar,
dra, ddec, separator, refmag))
f.close()
elif telescope == "P200":
with open(starlist, "a") as f:
f.write(
"{:s}{:s} {:s} 2000.0 {:s} raoffset={:.2f} decoffset={:.2f} r={:.1f} \n"
.format((name + "_S%s" % (ii + 1)).ljust(name_length),
refrah.ljust(rah_length), refdech, commentchar,
dra, ddec, refmag))
f.close()
# Plot compass
plt.plot([width - 10, height - 40], [10, 10], 'k-', lw=2)
plt.plot([width - 10, height - 10], [10, 40], 'k-', lw=2)
plt.annotate("N",
xy=(width - 20, 40),
xycoords='data',
xytext=(-4, 5),
textcoords='offset points')
plt.annotate("E",
xy=(height - 40, 20),
xycoords='data',
xytext=(-12, -5),
textcoords='offset points')
# Get rid of axis labels
plt.gca().get_xaxis().set_visible(False)
plt.gca().get_yaxis().set_visible(False)
# Set size of window (leaving space to right for ref star coords)
plt.subplots_adjust(right=0.65, left=0.05, top=0.99, bottom=0.05)
# List name, coords, mag of the target
plt.text(1.02,
0.85,
name,
transform=plt.axes().transAxes,
fontweight='bold')
# Can't print mag, because we don't know how bright the target is
#plt.text(1.02, 0.80, "%s"%mag, transform=plt.axes().transAxes, fontweight='bold')
plt.text(1.02,
0.80,
"%.5f %.5f" % (ra, dec),
transform=plt.axes().transAxes)
rah, dech = deg2hour(ra, dec)
plt.text(1.02, 0.75, rah + " " + dech, transform=plt.axes().transAxes)
if figdir == None:
plt.savefig("finder_chart_%s.png" % name)
else:
plt.savefig(figdir + "finder_chart_%s.png" % name)
plt.close()
def __init__(self, target=None):
""" """
self._zquery = query.ZTFQuery()
if target is not None:
self.set_name(target)
def get_forced_phot(name,ra,dec,jdobs):
"""
Uses Yuhan's code to perform forced photometry
for a given RA and Dec and central JD (day of the transient)
Over a window of +/- 10 days
Parameters
----------
name: name of the source
ra: ra in decimal degrees
dec: dec in decimal degrees
"""
start_jd = jdobs-200
end_jd = jdobs+100
zquery = query.ZTFQuery()
zquery.load_metadata(
radec=[ra, dec], size=0.0001,
sql_query="obsjd>%s and obsjd<%s" %(start_jd,end_jd))
zquery.download_data("scimrefdiffimg.fits.fz")
zquery.download_data("diffimgpsf.fits")
print("finished downloading data")
filefracday = zquery.metatable['filefracday'].values
jd = []
flux = []
eflux = []
mag = []
emag = []
filt = []
# Normalize all fluxes to the same zp
ZP = 25
for ffd in filefracday:
ffd = str(ffd)
fpath = 'Data/sci/' + ffd[0:4] + '/' + ffd[4:8] + '/' + ffd[8:]
# Check if directory is empty
if len(glob.glob(fpath + "/*")) > 0:
imgpath = glob.glob(fpath + "/*.fits.fz")[0]
psfpath = glob.glob(fpath + "/*diffimgpsf.fits")[0]
pobj = ZTFphot(name, ra, dec, imgpath, psfpath)
pobj.load_source_cutout()
pobj.load_bkg_cutout()
pobj.get_scr_cor_fn()
pobj.fit_psf()
jd.append(pobj.obsjd)
zp = pobj.zp
new_flux = pobj.Fpsf * 10**(-0.4*(zp-ZP))
flux.append(new_flux)
new_eflux = pobj.eFpsf * 10**(-0.4*(zp-ZP))
eflux.append(new_eflux)
mag.append(pobj.mag)
emag.append(pobj.mag_unc)
filt.append(pobj.filter)
jd = np.array(jd)
flux = np.array(flux)
eflux = np.array(eflux)
mag = np.array(mag)
emag =np.array(emag)
filt = np.array(filt)
return filt,jd,flux,eflux,mag,emag
def query_ipac(name):
'''
To query ipac for reference exposure epoch, you need to send the filterid, fieldid, ccdid, and qid
This can firstly be downloaded from ipac using ztfquery
Please see
https://irsa.ipac.caltech.edu/TAP/sync?query=select+column_name,description,unit,ucd,utype,datatype,principal,indexed+from+TAP_SCHEMA.columns+where+table_name=%27ztf.ztf_current_meta_ref%27+order+by+column_index&format=html
of the description of IPAC reference image columns
'''
# Step 0. As usual, generate directory to store data
cwd = os.getcwd()
targetdir = cwd + '/' + name + '/'
try:
os.stat(targetdir)
except:
os.mkdir(targetdir)
try:
os.stat(targetdir + 'lightcurves/')
except:
os.mkdir(targetdir + 'lightcurves/')
# Step 1. ipac does not know ZTF name, so let's get the coordinate first
ra1, dec1 = get_pos(name)
np.savetxt(targetdir + '/coo_marshal.reg', [ra1, dec1])
# Step 2. download infomation about all images that has ever covered this coordinate
zquery = query.ZTFQuery()
print("Querying for metadata...")
# note: the unit of size is [degree], you may want to change it
zquery.load_metadata(kind='sci', radec=[ra1, dec1], size=0.003)
out = zquery.metatable
final_out = out.sort_values(by=['obsjd'])
final_out.to_csv(targetdir + '/irsafile.csv')
# Step 3. get reference epoch for every row in the file `irsafile.csv`
s = requests.Session()
s.post(
'https://irsa.ipac.caltech.edu/account/signon/login.do?josso_cmd=login',
data={
'josso_username': DEFAULT_AUTH_ipac[0],
'josso_password': DEFAULT_AUTH_ipac[1]
})
mylc = Table([
final_out['field'].values, final_out['ccdid'].values,
final_out['fid'].values, final_out['qid'].values,
final_out['obsjd'].values
],
names=['field', 'ccdid', 'fid', 'qid', 'obsjd'])
# For each unique ccd-quarant (fcqf id), we only want to query once to save time
fcqf = mylc['field'] * 10000 + mylc['ccdid'] * 100 + mylc[
'qid'] * 10 + mylc['fid']
mylc['fcqf'] = fcqf
fcq_uniq = []
for x in mylc['fcqf']:
if x not in fcq_uniq:
fcq_uniq.append(x)
fcq_uniq = np.array(fcq_uniq)
jdref_start = np.zeros(len(mylc))
jdref_end = np.zeros(len(mylc))
for j in range(len(fcq_uniq)):
fcqnow = fcq_uniq[j]
temp1 = fcqnow - fcqnow % 10000
fieldnow = np.int(temp1 / 10000)
temp2 = fcqnow - temp1
temp3 = temp2 - temp2 % 100
ccdidnow = np.int(temp3 / 100)
temp4 = temp2 - temp3
qidnow = np.int((temp4 - temp4 % 10) / 10)
filteridnow = temp4 - qidnow * 10
if filteridnow == 1:
fltidnow = 'zg'
elif filteridnow == 2:
fltidnow = 'zr'
elif filteridnow == 3:
fltidnow = 'zi'
url = 'https://irsa.ipac.caltech.edu/ibe/search/ztf/products/ref?WHERE=field=' +\
'%d'%(fieldnow)+'%20AND%20ccdid='+'%d'%(ccdidnow) +\
'%20AND%20qid='+'%d'%(qidnow)+\
'%20AND%20filtercode=%27'+'%s'%(fltidnow)+'%27'
r = requests.get(url, cookies=s.cookies)
stringnow = r.content
stnow = stringnow.decode("utf-8")
tbnowj = asci.read(stnow)
if len(tbnowj) == 0:
print('no reference image: fcqf id = %d' % fcqnow)
else:
t0 = tbnowj['startobsdate'].data.data[0]
t1 = tbnowj['endobsdate'].data.data[0]
tstart = Time(t0.split(' ')[0] + 'T' + t0.split(' ')[1][:-3],
format='isot',
scale='utc')
tend = Time(t1.split(' ')[0] + 'T' + t1.split(' ')[1][:-3],
format='isot',
scale='utc')
ind = mylc['fcqf'] == fcqnow
jdref_start[ind] = tstart.jd
jdref_end[ind] = tend.jd
mylc['jdref_start'] = jdref_start
mylc['jdref_end'] = jdref_end
mylc.write(targetdir + 'lightcurves/' + '/ipac_info_' + name + '.csv')
