def main(obj, test, magnitude):
print('Got here.')
properties = p.object_params_fors2(obj)
synth_path = properties['data_dir'] + 'synthetic/'
u.mkdir_check(synth_path)
synth_path = synth_path + 'frb_position/'
u.mkdir_check(synth_path)
now = time.Time.now()
now.format = 'isot'
test = str(now) + '_' + test
test_path = synth_path + test + '/'
print(properties['filters'])
ph.insert_synthetic_at_frb(obj=obj,
test_path=test_path,
filters=properties['filters'],
magnitudes=[magnitude, magnitude],
add_path=True)
def main(obj, test, curves_path):
properties = p.object_params_fors2(obj)
if curves_path[-1] != '/':
curves_path += '/'
mag_table_file = filter(lambda file: file[-4:] == '.csv',
os.listdir(curves_path)).__next__()
mag_table = table.Table.read(curves_path + mag_table_file)
filters = mag_table.colnames.copy()
filters.remove('model')
synth_path = properties['data_dir'] + 'synthetic/'
u.mkdir_check(synth_path)
synth_path = synth_path + 'frb_position/'
u.mkdir_check(synth_path)
now = time.Time.now()
now.format = 'isot'
synth_path += f'sn_models_{now}/'
u.mkdir_check(synth_path)
for row in mag_table:
model = row['model']
magnitudes = []
for f in filters:
magnitudes.append(row[f])
test_spec = test + '_' + model
test_path = synth_path + test_spec + '/'
ph.insert_synthetic_at_frb(obj=obj,
test_path=test_path,
filters=filters,
magnitudes=magnitudes,
add_path=False)
for f in filters:
p.add_output_path(obj=obj,
key=f[0] + '_subtraction_image_synth_frb_sn_models',
path=synth_path)
def main(epoch, origin, destination):
print("\nExecuting Python script pipeline_fors2/4.1-insert_test_synth.py, with:")
print(f"\tepoch {epoch}")
print(f"\torigin directory {origin}")
print(f"\tdestination directory {destination}")
print()
epoch_params = p.object_params_fors2(obj=epoch)
outputs = p.object_output_params(obj=epoch, instrument='FORS2')
data_dir = epoch_params['data_dir']
insert = epoch_params['test_synths']
origin_path = data_dir + "analysis/sextractor/" + origin
destination_path = data_dir + destination
u.mkdir_check(destination_path)
u.mkdir_check(destination_path + "science/")
u.mkdir_check(destination_path + "backgrounds/")
filters = outputs['filters']
for fil in filters:
f = fil[0]
path_fil_output = destination_path + "science/" + fil + "/"
path_fil_input = origin_path + fil + "/"
u.mkdir_check(path_fil_output)
u.mkdir_check(destination_path + "backgrounds/" + fil)
zeropoint, _, airmass, _, extinction, _ = ph.select_zeropoint(obj=epoch,
filt=fil,
instrument='fors2',
outputs=outputs)
print(path_fil_input)
# print(os.listdir(path_fil_input))
for fits_file in filter(lambda f: f.endswith("_norm.fits"), os.listdir(path_fil_input)):
print(fits_file)
path_fits_file_input = path_fil_input + fits_file
path_fits_file_output = path_fil_output + fits_file
path_psf_model = path_fits_file_input.replace(".fits", "_psfex.psf")
try:
ph.insert_point_sources_to_file(file=path_fits_file_input,
x=array(insert["ra"]),
y=array(insert["dec"]),
mag=insert[f"{f}_mag"],
output=path_fits_file_output,
zeropoint=zeropoint,
extinction=extinction,
airmass=airmass,
world_coordinates=True,
psf_model=path_psf_model
)
except ValueError:
ph.insert_point_sources_to_file(file=path_fits_file_input,
x=array(insert["ra"]),
y=array(insert["dec"]),
mag=insert[f"{f}_mag"],
output=path_fits_file_output,
zeropoint=zeropoint,
extinction=extinction,
airmass=airmass,
world_coordinates=True,
fwhm=fits.open(path_psf_model)[1].header['PSF_FWHM']
)
if os.path.isfile(origin_path + epoch + '.log'):
copyfile(origin_path + epoch + '.log', destination_path + epoch + ".log")
u.write_log(path=destination_path + epoch + ".log", action=f'Divided by exposure time.')
def main(data_dir, data_title, origin, destination, all_synths):
print("\nExecuting Python script pipeline_fors2/5-background_subtract.py, with:")
print(f"\tepoch {data_title}")
print(f"\torigin directory {origin}")
print(f"\tdestination directory {destination}")
print()
methods = ["ESO backgrounds only", "SExtractor backgrounds only", "polynomial fit", "Gaussian fit", "median value"]
if all_synths:
frame = 56
method = "polynomial fit"
degree = 5
do_mask = True
local = True
global_sub = False
trim_image = False
recorrect_subbed = True
eso_back = False
else:
frame = 200
# frame_arcsec = 30 * units.arcsec
# frame_deg = frame_arcsec.to(units.deg)
eso_back = False
_, method = u.select_option(message="Please select the background subtraction method.", options=methods,
default="polynomial fit")
degree = None
if method == "polynomial fit":
degree = u.user_input(message=f"Please enter the degree of {method} to use:", typ=int, default=3)
elif method == "ESO backgrounds only":
eso_back = True
do_mask = False
if method not in ["ESO backgrounds only", "SExtractor backgrounds only", "median value"]:
do_mask = u.select_yn(message="Mask sources using SExtractor catalogue?", default=True)
if method in ["polynomial fit", "Gaussian fit"]:
local = u.select_yn(message="Use a local fit?", default=True)
else:
local = False
global_sub = False
trim_image = False
recorrect_subbed = False
if local:
global_sub = u.select_yn(message="Subtract local fit from entire image?", default="n")
if not global_sub:
trim_image = u.select_yn(message="Trim images to subtracted region?", default="y")
recorrect_subbed = u.select_yn(message="Re-normalise background of subtracted region?", default="y")
# if not eso_back and method != "SExtractor backgrounds only":
# eso_back = u.select_yn(message="Subtract ESO Reflex fitted backgrounds first?", default=False)
outputs = p.object_output_params(data_title, instrument='FORS2')
data_dir = u.check_trailing_slash(data_dir)
destination = u.check_trailing_slash(destination)
destination = data_dir + destination
u.mkdir_check_nested(destination)
origin = u.check_trailing_slash(origin)
science_origin = data_dir + origin + "science/"
print(science_origin)
filters = outputs['filters']
frb_params = p.object_params_frb(obj=data_title[:-2])
epoch_params = p.object_params_fors2(obj=data_title)
background_origin_eso = ""
if eso_back:
background_origin_eso = data_dir + "/" + origin + "/backgrounds/"
if method == "SExtractor backgrounds only":
background_origin = f"{data_dir}{origin}backgrounds_sextractor/"
elif method == "polynomial fit":
background_origin = f"{destination}backgrounds/" # f"{destination}backgrounds_{method.replace(' ', '')}_degree_{degree}_local_{local}_globalsub_{global_sub}/"
else:
background_origin = f"{destination}backgrounds/" # f"{destination}backgrounds_{method.replace(' ', '')}_local_{local}_globalsub_{global_sub}/"
trimmed_path = ""
if trim_image:
trimmed_path = f"{data_dir}{origin}trimmed_to_background/"
u.mkdir_check_nested(trimmed_path)
ra = frb_params["burst_ra"]
dec = frb_params["burst_dec"]
if all_synths:
ras = epoch_params["test_synths"]["ra"]
decs = epoch_params["test_synths"]["dec"]
else:
ras = [ra]
decs = [dec]
for fil in filters:
trimmed_path_fil = ""
if trim_image:
trimmed_path_fil = f"{trimmed_path}{fil}/"
u.mkdir_check(trimmed_path_fil)
background_fil_dir = f"{background_origin}{fil}/"
u.mkdir_check_nested(background_fil_dir)
science_destination_fil = f"{destination}science/{fil}/"
u.mkdir_check_nested(science_destination_fil)
files = os.listdir(science_origin + fil + "/")
for file_name in files:
if file_name.endswith('.fits'):
new_file = file_name.replace("norm", "bg_sub")
new_path = f"{science_destination_fil}/{new_file}"
print("NEW_PATH:", new_path)
science = science_origin + fil + "/" + file_name
# First subtract ESO Reflex background images
# frame = (frame_deg / f.get_pixel_scale(file=science, astropy_units=True)[1]).to(f.pix).value
if eso_back:
background_eso = background_origin_eso + fil + "/" + file_name.replace("SCIENCE_REDUCED",
"PHOT_BACKGROUND_SCI")
ff.subtract_file(file=science, sub_file=background_eso, output=new_path)
science_image = new_path
if method != "ESO backgrounds only":
print(ra, dec)
print("Science image:", science)
science_image = fits.open(science)
print("Science file:", science_image)
wcs_this = WCS(header=science_image[0].header)
if method == "SExtractor backgrounds only":
background = background_origin + fil + "/" + file_name + "_back.fits"
print("Background image:", background)
else:
if method == "median value":
print(science_image[0].data.shape)
_, background_value, _ = sigma_clipped_stats(science_image[0].data)
background = deepcopy(science_image)
background[0].data = np.full(shape=science_image[0].data.shape, fill_value=background_value)
background_path = background_origin + fil + "/" + file_name.replace("SCIENCE_REDUCED",
"PHOT_BACKGROUND_MEDIAN")
# Next do background fitting.
else:
background = deepcopy(science_image)
background[0].data = np.zeros(background[0].data.shape)
background_path = background_origin + fil + "/" + file_name.replace("SCIENCE_REDUCED",
"PHOT_BACKGROUND_FITTED")
for i, ra in enumerate(ras):
dec = decs[i]
x, y = wcs_this.all_world2pix(ra, dec, 0)
print(x, y)
bottom, top, left, right = ff.subimage_edges(data=science_image[0].data, x=x, y=y,
frame=frame)
if do_mask:
# Produce a pixel mask that roughly masks out the true sources in the image so that
# they don't get fitted.
mask_max = 10
_, pixel_scale = ff.get_pixel_scale(science_image)
sextractor = Table.read(
f"{data_dir}analysis/sextractor/4-divided_by_exp_time/{fil}/{file_name.replace('.fits', '_psf-fit.cat')}",
format='ascii.sextractor')
weights = np.ones(shape=science_image[0].data.shape)
for obj in filter(
lambda o: left < o["X_IMAGE"] < right and bottom < o["Y_IMAGE"] < top,
sextractor):
mask_rad = min(int(obj["A_WORLD"] * obj["KRON_RADIUS"] / pixel_scale), mask_max)
x_prime = int(np.round(obj["X_IMAGE"]))
y_prime = int(np.round(obj["Y_IMAGE"]))
weights[y_prime - mask_rad:y_prime + mask_rad,
x_prime - mask_rad:x_prime + mask_rad] = 0.0
plt.imshow(weights, origin="lower")
plt.savefig(
background_origin + fil + "/" + file_name.replace("norm.fits", "mask.png"))
else:
weights = None
background_this = fit_background_fits(image=science_image,
model_type=method[:method.find(" ")],
deg=degree, local=local,
global_sub=global_sub,
centre_x=x, centre_y=y, frame=frame,
weights=weights)
background[0].data += background_this[0].data
if recorrect_subbed:
offset = get_median_background(image=science,
ra=epoch_params["renormalise_centre_ra"],
dec=epoch_params["renormalise_centre_dec"], frame=50,
show=False,
output=new_path[
:new_path.find("bg_sub")] + "renorm_patch_")
print("RECORRECT_SUBBED:", recorrect_subbed)
print("SUBTRACTING FROM BACKGROUND:", offset)
print(bottom, top, left, right)
print(background[0].data[bottom:top, left:right].shape)
print(np.median(background[0].data[bottom:top, left:right]))
background[0].data[bottom:top, left:right] -= offset
print(np.median(background[0].data[bottom:top, left:right]))
if trim_image:
print("TRIMMED_PATH_FIL:", trimmed_path_fil)
science_image = ff.trim_file(path=science_image, left=left, right=right, top=top,
bottom=bottom,
new_path=trimmed_path_fil + file_name.replace(
"norm.fits",
"trimmed_to_back.fits"))
print("Science after trim:", science_image)
background = ff.trim_file(path=background, left=left, right=right, top=top,
bottom=bottom,
new_path=background_path)
print("Writing background to:")
print(background_path)
background.writeto(background_path, overwrite=True)
print("SCIENCE:", science_image)
print("BACKGROUND:", background)
subbed = ff.subtract_file(file=science_image, sub_file=background, output=new_path)
# # TODO: check if regions overlap
#
# plt.hist(subbed[0].data[int(y - frame + 1):int(y + frame - 1),
# int(x - frame + 1):int(x + frame - 1)].flatten(),
# bins=10)
# plt.savefig(new_path[:new_path.find("bg_sub")] + "histplot.png")
# plt.close()
copyfile(data_dir + "/" + origin + "/" + data_title + ".log", destination + data_title + ".log")
u.write_log(path=destination + data_title + ".log",
action=f'Backgrounds subtracted using 4-background_subtract.py with method {method}\n')
def main(data_title: 'str'):
print("\nExecuting Python script pipeline_fors2/1-initial.py, with:")
print(f"\tepoch {data_title}")
print()
epoch_params = p.object_params_fors2(obj=data_title)
data_dir = epoch_params['data_dir']
output_dir = data_dir + "/0-data_with_raw_calibs/"
# Write tables of fits files to main directory; firstly, science images only:
table = fits_table(input_path=output_dir,
output_path=data_dir + data_title + "_fits_table_science.csv",
science_only=True)
# Then including all calibration files
table_full = fits_table(input_path=output_dir,
output_path=data_dir + data_title + "_fits_table_all.csv",
science_only=False)
fits_table_all(input_path=output_dir,
output_path=data_dir + data_title + "_fits_table_detailled.csv",
science_only=False)
# Clear output files for fresh start.
u.rm_check(data_dir + '/output_values.yaml')
u.rm_check(data_dir + '/output_values.json')
# Collect list of filters used:
filters = []
columns = []
for j in [1, 2, 3, 4, 5]:
column = 'filter' + str(j)
for name in table[column]:
if name != 'free':
if name not in filters:
filters.append(name)
columns.append(column)
# Collect pointings of standard-star observations.
std_ras = []
std_decs = []
std_pointings = []
# TODO: This is a horrible way to do this. Take the time to find a better one.
for ra in table_full[table_full['object'] == 'STD']['ref_ra']:
if ra not in std_ras:
std_ras.append(ra)
for dec in table_full[table_full['object'] == 'STD']['ref_dec']:
if dec not in std_decs:
std_decs.append(dec)
for i, ra in enumerate(std_ras):
std_pointings.append(f'RA{ra}_DEC{std_decs[i]}')
print(std_ras)
print(std_decs)
print(std_pointings)
# Collect and save some stats on those filters:
param_dict = {}
exp_times = []
ns_exposures = []
param_dict['filters'] = filters
param_dict['object'] = table['object'][0]
param_dict['obs_name'] = table['obs_name'][0]
mjd = param_dict['mjd_obs'] = float(table['mjd_obs'][0])
for i, f in enumerate(filters):
f_0 = f[0]
exp_time = table['exp_time'][table[columns[i]] == f]
exp_times.append(exp_time)
airmass_col = table['airmass'][table[columns[i]] == f]
n_frames = sum(table[columns[i]] == f)
n_exposures = n_frames / 2
ns_exposures.append(n_exposures)
airmass = float(np.nanmean(airmass_col))
param_dict[f_0 + '_exp_time_mean'] = float(np.nanmean(exp_time))
param_dict[f_0 + '_exp_time_err'] = float(2 * np.nanstd(exp_time))
param_dict[f_0 + '_airmass_mean'] = airmass
param_dict[f_0 + '_airmass_err'] = float(
max(np.nanmax(airmass_col) - airmass, airmass - np.nanmin(airmass_col)))
param_dict[f_0 + '_n_frames'] = float(n_frames)
param_dict[f_0 + '_n_exposures'] = float(n_exposures)
param_dict[f_0 + '_mjd_obs'] = float(np.nanmean(table['mjd_obs'][table[columns[i]] == f]))
std_filter_dir = f'{data_dir}calibration/std_star/{f}/'
u.mkdir_check(std_filter_dir)
print(f'Copying {f} calibration data to std_star folder...')
# Sort the STD files by filter, and within that by pointing.
for j, ra in enumerate(std_ras):
at_pointing = False
pointing = std_pointings[j]
pointing_dir = std_filter_dir + pointing + '/'
for file in \
table_full[
(table_full['object'] == 'STD') &
(table_full['ref_ra'] == ra) &
(table_full[columns[i]] == f)]['identifier']:
at_pointing = True
u.mkdir_check(pointing_dir)
shutil.copyfile(output_dir + file, pointing_dir + file)
if at_pointing:
for file in table_full[table_full['object'] == 'BIAS']['identifier']:
shutil.copyfile(output_dir + file, pointing_dir + file)
for file in table_full[(table_full['object'] == 'FLAT,SKY') & (table_full[columns[i]] == f)][
'identifier']:
shutil.copyfile(output_dir + file, pointing_dir + file)
p.add_output_values(obj=data_title, params=param_dict)
if "new_epoch" in data_dir:
mjd = f"MJD{int(float(mjd))}"
new_data_dir = data_dir.replace("new_epoch", mjd)
p.add_epoch_param(obj=data_title, params={"data_dir": new_data_dir})
def main(obj, test, n, mag_lower, mag_upper, colour_upper, colour_lower):
properties = p.object_params_fors2(obj)
output = p.object_output_params(obj=obj, instrument='FORS2')
paths = p.object_output_paths(obj)
burst_properties = p.object_params_frb(obj[:-2])
synth_path = properties['data_dir'] + 'synthetic/'
u.mkdir_check(synth_path)
synth_path = synth_path + 'random/'
u.mkdir_check(synth_path)
now = time.Time.now()
now.format = 'isot'
test = str(now) + '_' + test
test_path = synth_path + test + '/'
u.mkdir_check(test_path)
filters = {}
bluest = None
bluest_lambda = np.inf
for f in output['filters']:
filter_properties = p.filter_params(f=f, instrument='FORS2')
filters[f] = filter_properties
lambda_eff = filter_properties['lambda_eff']
if lambda_eff < bluest_lambda:
bluest_lambda = lambda_eff
bluest = f
# Insert random sources in the bluest filter.
f_0 = bluest[0]
output_properties = p.object_output_params(obj)
fwhm = output_properties[f_0 + '_fwhm_pix']
zeropoint, _, airmass, _ = ph.select_zeropoint(obj,
bluest,
instrument='fors2')
base_path = paths[f_0 + '_subtraction_image']
output_path = test_path + f_0 + '_random_sources.fits'
_, sources = ph.insert_random_point_sources_to_file(file=base_path,
fwhm=fwhm,
output=output_path,
n=n,
airmass=airmass,
zeropoint=zeropoint)
p.add_output_path(obj=obj,
key=f_0 + '_subtraction_image_synth_random',
path=output_path)
# Now insert sources at the same positions in other filters, but with magnitudes randomised.
for f in filters:
if f != bluest:
f_0 = f[0]
output_properties = p.object_output_params(obj)
fwhm = output_properties[f_0 + '_fwhm_pix']
zeropoint, _, airmass, _ = ph.select_zeropoint(obj,
f,
instrument='fors2')
base_path = paths[f_0 + '_subtraction_image']
mag = np.random.uniform(mag_lower, mag_upper, size=n)
output_path = test_path + f_0 + '_random_sources.fits'
ph.insert_point_sources_to_file(file=base_path,
fwhm=fwhm,
output=output_path,
x=sources['x_0'],
y=sources['y_0'],
mag=mag,
airmass=airmass,
zeropoint=zeropoint)
p.add_output_path(obj=obj,
key=f_0 + '_subtraction_image_synth_random',
path=output_path)
def main(origin_dir, output_dir, data_title, sextractor_path):
print("\nExecuting Python script pipeline_fors2/3-trim.py, with:")
print(f"\tepoch {data_title}")
print(f"\torigin directory {origin_dir}")
print(f"\toutput directory {output_dir}")
print()
# If this is None, we don't want the SExtractor components to be performed.
if sextractor_path is not None:
if not os.path.isdir(sextractor_path):
os.mkdir(sextractor_path)
do_sextractor = True
print(os.getcwd())
else:
do_sextractor = False
if not os.path.isdir(output_dir):
os.mkdir(output_dir)
if not os.path.isdir(output_dir + "backgrounds/"):
os.mkdir(output_dir + "backgrounds/")
if not os.path.isdir(output_dir + "science/"):
os.mkdir(output_dir + "science/")
wdir = origin_dir + "backgrounds/"
epoch_params = p.object_params_fors2(obj=data_title)
outputs = p.object_output_params(obj=data_title)
fils = outputs["filters"]
edged = False
up_left = 0
up_right = 0
up_bottom = 0
up_top = 0
dn_left = 0
dn_right = 0
dn_bottom = 0
dn_top = 0
for fil in fils:
print(output_dir + "backgrounds/" + fil)
if not os.path.isdir(output_dir + "backgrounds/" + fil):
os.mkdir(output_dir + "backgrounds/" + fil)
print('HERE:')
print(wdir + fil)
files = os.listdir(wdir + fil)
files.sort()
if not edged:
# Find borders of noise frame using backgrounds.
# First, make sure that the background we're using is for the top chip.
i = 0
while f.get_chip_num(wdir + fil + "/" + files[i]) != 1:
i += 1
up_left, up_right, up_bottom, up_top = f.detect_edges(wdir + fil +
"/" +
files[i])
# Ditto for the bottom chip.
i = 0
while f.get_chip_num(wdir + fil + "/" + files[i]) != 2:
i += 1
dn_left, dn_right, dn_bottom, dn_top = f.detect_edges(wdir + fil +
"/" +
files[i])
up_left = up_left + 5
up_right = up_right - 5
up_top = up_top - 5
dn_left = dn_left + 5
dn_right = dn_right - 5
dn_bottom = dn_bottom + 5
print('Upper chip:')
print(up_left, up_right, up_top, up_bottom)
print('Lower:')
print(dn_left, dn_right, dn_top, dn_bottom)
edged = True
for i, file in enumerate(files):
print(f'{i} {file}')
for i, file in enumerate(files):
new_path = output_dir + "backgrounds/" + fil + "/" + file.replace(
".fits", "_trim.fits")
# Add GAIN and SATURATE keywords to headers.
path = wdir + fil + "/" + file
print(f'{i} {file}')
# Split the files into upper CCD and lower CCD
if f.get_chip_num(path) == 1:
print('Upper Chip:')
f.trim_file(path,
left=up_left,
right=up_right,
top=up_top,
bottom=up_bottom,
new_path=new_path)
elif f.get_chip_num(path) == 2:
print('Lower Chip:')
f.trim_file(path,
left=dn_left,
right=dn_right,
top=dn_top,
bottom=dn_bottom,
new_path=new_path)
else:
raise ValueError(
'Invalid chip ID; could not trim based on upper or lower chip.'
)
# Repeat for science images
wdir = origin_dir + "science/"
fils = os.listdir(wdir)
for fil in fils:
print(output_dir + "science/" + fil)
if do_sextractor:
if not os.path.isdir(sextractor_path + fil):
os.mkdir(sextractor_path + fil)
if not os.path.isdir(output_dir + "science/" + fil):
os.mkdir(output_dir + "science/" + fil)
files = os.listdir(wdir + fil)
files.sort()
for i, file in enumerate(files):
print(f'{i} {file}')
for i, file in enumerate(files):
# Split the files into upper CCD and lower CCD, with even-numbered being upper and odd-numbered being lower
new_file = file.replace(".fits", "_trim.fits")
new_path = output_dir + "science/" + fil + "/" + new_file
path = wdir + fil + "/" + file
f.change_header(file=path, key='GAIN', value=0.8)
f.change_header(file=path, key='SATURATE', value=65535.)
if f.get_chip_num(path) == 1:
print('Upper Chip:')
f.trim_file(path,
left=up_left,
right=up_right,
top=up_top,
bottom=up_bottom,
new_path=new_path)
if do_sextractor:
copyfile(new_path, sextractor_path + fil + "/" + new_file)
elif f.get_chip_num(path) == 2:
print('Lower Chip:')
f.trim_file(path,
left=dn_left,
right=dn_right,
top=dn_top,
bottom=dn_bottom,
new_path=new_path)
try:
copyfile(origin_dir + data_title + ".log",
output_dir + data_title + ".log")
except FileNotFoundError:
print("Previous log not found.")
u.write_log(path=output_dir + data_title + ".log",
action='Edges trimmed using 3-trim.py\n')
def main(obj, astrometry_path, sextractor_path, template, show):
print("\nExecuting Python script pipeline_fors2/8-astrometry.py, with:")
print(f"\tepoch {obj}")
print(f"\tastrometry path {astrometry_path}")
print(f"\tsextractor path {sextractor_path}")
print(f"\ttemplate {template}")
print(f"\tshow {show}")
print()
files = os.listdir(astrometry_path)
template_file = template[0] + '_astrometry.fits'
params = p.object_params_fors2(obj)
for file in files:
if 'coadded.fits' in file:
astrometry_file = file.replace('_coadded', '_astrometry')
coadd = fits.open(astrometry_path + file)
header = coadd[0].header
if os.path.isfile(astrometry_path + astrometry_file):
astrometry = fits.open(astrometry_path + astrometry_file, mode='update')
astrometry[0].header['AIRMASS'] = header['AIRMASS']
astrometry[0].header['FILTER'] = header['FILTER']
astrometry[0].header['OBJECT'] = header['OBJECT']
gain = header['GAIN']
astrometry[0].header['GAIN'] = gain
astrometry[0].header['EXPTIME'] = header['EXPTIME']
astrometry[0].header['RDNOISE'] = 0
astrometry[0].header['MJD-OBS'] = header['MJD-OBS']
ff.add_log(astrometry, 'Refined astrometric solution using Astrometry.net')
coadd.close()
astrometry.close()
print()
p.add_output_path(obj=obj, key=file[0] + '_astrometry_image',
path=astrometry_path + astrometry_file)
if file != template_file:
shutil.copy(astrometry_path + astrometry_file,
astrometry_path + astrometry_file.replace('.fits', '_orig.fits'))
shutil.copy(astrometry_path + template_file,
astrometry_path + template_file.replace('.fits', '_orig.fits'))
ff.align(comparison=astrometry_path + astrometry_file,
template=astrometry_path + template_file,
comparison_output=astrometry_path + astrometry_file,
template_output=astrometry_path + template_file)
else:
print(f'No astrometry file corresponding to {file} found.')
# TODO: This will break if more than two filters have been used on this observation.
# Rewrite align to work with one template and multiple comparisons.
if params['astrometry_tweak']:
tweak_final(epoch=obj, sextractor_path=sextractor_path,
destination=astrometry_path, instrument='FORS2',
show=show, output_suffix='astrometry_tweaked', input_suffix='astrometry', stars_only=True,
specific_star=False,
path_add='tweaked_image', manual=False)
tweak_final(epoch=obj, sextractor_path=sextractor_path,
destination=astrometry_path, instrument='FORS2',
show=show, output_suffix='astrometry_tweaked_hg', input_suffix='astrometry', stars_only=True,
specific_star=True,
path_add='tweaked_image_hg', manual=False)
if params['manual_astrometry']:
tweak_final(epoch=obj, sextractor_path=sextractor_path,
destination=astrometry_path, instrument='FORS2',
show=show, output_suffix='astrometry_tweaked_manual', input_suffix='astrometry', stars_only=True,
path_add='tweaked_image_manual', manual=True)
def main(epoch, test_name, sex_x_col, sex_y_col, sex_ra_col, sex_dec_col,
sex_flux_col, stars_only, show_plots, mag_range_sex_lower,
mag_range_sex_upper, pix_tol, instrument):
print("\nExecuting Python script pipeline_fors2/9-zeropoint.py, with:")
print(f"\tepoch {epoch}")
print()
properties = p.object_params_fors2(epoch)
outputs = p.object_output_params(obj=epoch, instrument='fors2')
proj_paths = p.config
output = properties['data_dir'] + '9-zeropoint/'
mkdir_check(output)
output_std = output + 'std/'
mkdir_check(output_std)
mkdir_check(output_std)
std_path = properties['data_dir'] + 'calibration/std_star/'
filters = outputs['filters']
std_field_path = proj_paths['top_data_dir'] + "std_fields/"
std_fields = list(
filter(lambda path: os.path.isdir(path), os.listdir(std_field_path)))
print('Standard fields with available catalogues:')
print(std_fields)
print()
cat_names = photometry_catalogues
for fil in filters:
print('Doing filter', fil)
f = fil[0]
# Obtain zeropoints from FRB field, if data is available.
print(
f"\nDetermining science-field zeropoints for {epoch}, filter {fil}:\n"
)
if f"{f}_zeropoints" in outputs:
zeropoints = outputs[f"{f}_zeropoints"]
else:
zeropoints = {}
zeropoints["science_field"] = {}
for cat_name in cat_names:
zeropoints["science_field"][
cat_name], _ = photometry.zeropoint_science_field(
epoch=epoch,
instrument=instrument,
test_name=None,
sex_x_col='XPSF_IMAGE',
sex_y_col='YPSF_IMAGE',
sex_ra_col='ALPHAPSF_SKY',
sex_dec_col='DELTAPSF_SKY',
sex_flux_col='FLUX_PSF',
stars_only=True,
star_class_col='CLASS_STAR',
star_class_tol=0.95,
show_plots=False,
mag_range_sex_lower=-100. * units.mag,
mag_range_sex_upper=100. * units.mag,
pix_tol=5. * units.pixel,
separate_chips=True,
cat_name=cat_name)
# Obtain zeropoints from available standard fields and available data.
print(
f"\nDetermining standard-field zeropoints for {epoch}, filter {fil}\n"
)
zeropoints["standard_field"] = {}
fil_path = std_path + fil + '/'
if os.path.isdir(fil_path):
fields = filter(lambda d: os.path.isdir(fil_path + d),
os.listdir(fil_path))
output_path_fil_std = output_std + '/' + fil + '/'
mkdir_check(output_path_fil_std)
for field in fields:
zeropoints["standard_field"][field] = {}
ra = float(field[field.find("RA") + 2:field.find("_")])
dec = float(field[field.find("DEC") + 3:])
print("Looking for photometry data in field " + field + ":")
mkdir_check(std_field_path + field)
field_path = fil_path + field + '/'
output_path = output_path_fil_std + field + '/'
std_cat_path = std_field_path + field + '/'
std_properties = p.load_params(field_path + 'params.yaml')
use_sex_star_class = std_properties['use_sex_star_class']
# Cycle through the three catalogues used to determine zeropoint, in order of preference.
for cat_name in cat_names:
# Check for photometry on-disk in the relevant catalogue; if none present, attempt to retrieve from
# online archive.
print(f"In {cat_name}:")
output_path_cat = output_path + cat_name
cat_path = f"{std_cat_path}{cat_name}/{cat_name}.csv"
if not (os.path.isdir(std_cat_path + cat_name)
) or os.path.isfile(cat_path):
print(
"None found on disk. Attempting retrieval from archive..."
)
if update_std_photometry(ra=ra, dec=dec,
cat=cat_name) is None:
print("\t\tNo data found in archive.")
zeropoints["standard_field"][field][
cat_name] = None
continue
column_names = cat_columns(cat=cat_name, f=f)
cat_ra_col = column_names['ra']
cat_dec_col = column_names['dec']
cat_mag_col = column_names['mag_psf']
if not use_sex_star_class:
star_class_col = column_names['class_star']
else:
star_class_col = 'CLASS_STAR'
cat_type = 'csv'
sextractor_path = field_path + 'sextractor/_psf-fit.cat'
image_path = field_path + '3-trimmed/standard_trimmed_img_up.fits'
star_class_tol = std_properties['star_class_tol']
now = time.Time.now()
now.format = 'isot'
test_name = str(now) + '_' + test_name
mkdir_check(properties['data_dir'] +
'/analysis/zeropoint/')
mkdir_check(output_path)
exp_time = ff.get_exp_time(image_path)
print('SExtractor catalogue path:', sextractor_path)
print('Image path:', image_path)
print('Catalogue name:', cat_name)
print('Catalogue path:', cat_path)
print('Class star column:', star_class_col)
print('Output:', output_path_cat)
print('Exposure time:', exp_time)
print("Use sextractor class star:", use_sex_star_class)
zeropoints["standard_field"][field][
cat_name] = photometry.determine_zeropoint_sextractor(
sextractor_cat=sextractor_path,
image=image_path,
cat_path=cat_path,
cat_name=cat_name,
output_path=output_path_cat,
show=show_plots,
cat_ra_col=cat_ra_col,
cat_dec_col=cat_dec_col,
cat_mag_col=cat_mag_col,
sex_ra_col=sex_ra_col,
sex_dec_col=sex_dec_col,
sex_x_col=sex_x_col,
sex_y_col=sex_y_col,
dist_tol=pix_tol,
flux_column=sex_flux_col,
mag_range_sex_upper=mag_range_sex_upper,
mag_range_sex_lower=mag_range_sex_lower,
stars_only=stars_only,
star_class_tol=star_class_tol,
star_class_col=star_class_col,
exp_time=exp_time,
y_lower=0,
cat_type=cat_type,
)
output_dict = {f + '_zeropoints': zeropoints}
p.add_output_values(obj=epoch, instrument='FORS2', params=output_dict)
outputs = p.object_output_params(obj=epoch, instrument='fors2')
output_path_final = output + "collated/"
mkdir_check(output_path_final)
print("Collating zeropoints...")
for fil in filters:
print(f"For {fil}:")
f = fil[0]
output_path_final_f = output_path_final + fil + "/"
mkdir_check(output_path_final_f)
zeropoints = outputs[f"{f}_zeropoints"]
airmass_sci = outputs[f"{f}_airmass_mean"]
airmass_sci_err = outputs[f"{f}_airmass_err"]
extinction = outputs[f"{f}_extinction"]
extinction_err = outputs[f"{f}_extinction_err"]
zeropoint_tbl = table.Table(
dtype=[("type", 'S15'), ("field", 'S25'), (
"cat", 'S10'), ("zeropoint",
float), ("zeropoint_err",
float), ("airmass",
float), ("airmass_err",
float),
("extinction",
float), ("extinction_err",
float), ("zeropoint_ext_corr",
float), ("zeropoint_ext_corr_err",
float), ("n_matches", float)])
if f"provided" in zeropoints:
print(f"\tProvided:")
zeropoint_prov = zeropoints["provided"]
zeropoint_tbl.add_row([
"provided", "N/A", "N/A", zeropoint_prov["zeropoint"],
zeropoint_prov["zeropoint_err"], 0.0, 0.0, extinction,
extinction_err, zeropoint_prov["zeropoint"],
zeropoint_prov["zeropoint_err"], 0
])
print(f"\tScience field:")
for cat_name in zeropoints["science_field"]:
print(f"\t\t{cat_name}")
zeropoint_sci = zeropoints["science_field"][cat_name]
if zeropoint_sci is not None:
zeropoint_corrected = zeropoint_sci[
"zeropoint"] + extinction * airmass_sci
print("\t\t\t", zeropoint_sci["zeropoint"], extinction,
airmass_sci)
zeropoint_corrected_err = zeropoint_sci[
"zeropoint_err"] + error_product(
value=extinction * airmass_sci,
measurements=[extinction, airmass_sci],
errors=[extinction_err, airmass_sci_err])
zp_cat = table.Table.read(zeropoint_sci["matches_cat_path"])
zeropoint_tbl.add_row([
"science_field", epoch[:-2], cat_name,
zeropoint_sci["zeropoint"], zeropoint_sci["zeropoint_err"],
airmass_sci, airmass_sci_err, extinction, extinction_err,
zeropoint_corrected, zeropoint_corrected_err,
len(zp_cat)
])
plt.scatter(zp_cat["mag_cat"], zp_cat["mag"], c='green')
plt.errorbar(zp_cat["mag_cat"],
zp_cat["mag"],
yerr=zp_cat["mag_err"],
linestyle="None",
c='black')
plt.plot(zp_cat["mag_cat"],
zp_cat["mag_cat"] - zeropoint_sci["zeropoint"],
c="violet")
plt.title(f"{fil}: science-field, {cat_name}")
plt.figtext(
0, 0.01,
f'zeropoint - kX: {zeropoint_sci["zeropoint"]} ± {zeropoint_sci["zeropoint_err"]}\n'
f'zeropoint: {zeropoint_corrected} ± {zeropoint_corrected_err}\n'
f'num stars: {len(zp_cat)}')
plt.gca().set_aspect('equal', adjustable='box')
plt.savefig(output_path_final_f +
f"zeropoint_science_{cat_name}.pdf")
plt.show()
plt.close()
print("\tStandard fields:")
for field in zeropoints["standard_field"]:
print(f"\t\t{field}")
for cat_name in zeropoints["standard_field"][field]:
print(f"\t\t\t{cat_name}")
zeropoint_std = zeropoints["standard_field"][field][cat_name]
if zeropoint_std is not None:
zeropoint_corrected = zeropoint_std[
"zeropoint"] + extinction * zeropoint_std["airmass"]
print("\t\t\t\t", zeropoint_std["zeropoint"], extinction,
zeropoint_std["airmass"])
zeropoint_corrected_err = zeropoint_std[
"zeropoint_err"] + error_product(
value=extinction * zeropoint_std["airmass"],
measurements=[
extinction, zeropoint_std["airmass"]
],
errors=[extinction_err, 0.0])
zp_cat = table.Table.read(
zeropoint_std["matches_cat_path"])
zeropoint_tbl.add_row([
"standard_field", field, cat_name,
zeropoint_std["zeropoint"],
zeropoint_std["zeropoint_err"],
zeropoint_std["airmass"], 0.0, extinction,
extinction_err, zeropoint_corrected,
zeropoint_corrected_err,
len(zp_cat)
])
plt.scatter(zp_cat["mag_cat"], zp_cat["mag"], c='green')
plt.errorbar(zp_cat["mag_cat"],
zp_cat["mag"],
yerr=zp_cat["mag_err"],
linestyle="None",
c='black')
plt.plot(zp_cat["mag_cat"],
zp_cat["mag_cat"] - zeropoint_std["zeropoint"],
c="violet")
plt.title(f"{fil}: standard-field {field}, {cat_name}")
plt.gca().set_aspect('equal', adjustable='box')
plt.figtext(
0, 0.01,
f'zeropoint - kX: {zeropoint_std["zeropoint"]} ± {zeropoint_std["zeropoint_err"]}\n'
f'zeropoint: {zeropoint_corrected} ± {zeropoint_corrected_err}\n'
f'num stars: {len(zp_cat)}')
plt.savefig(output_path_final_f +
f"zeropoint_standard_{field}_{cat_name}.pdf")
plt.show()
plt.close()
zeropoint_tbl["selection_index"] = zeropoint_tbl[
"n_matches"] / zeropoint_tbl["zeropoint_ext_corr_err"]
best_arg = np.argmax(zeropoint_tbl["selection_index"])
print("Best zeropoint:")
best_zeropoint = zeropoint_tbl[best_arg]
print(best_zeropoint)
zeropoints = outputs[f + '_zeropoints']
zeropoints['best'] = {
"zeropoint": float(best_zeropoint['zeropoint']),
"zeropoint_err": float(best_zeropoint["zeropoint_err"]),
"airmass": float(best_zeropoint["airmass"]),
"airmass_err": float(best_zeropoint["airmass_err"]),
"type": str(best_zeropoint["type"]),
"catalogue": str(best_zeropoint["cat"])
}
output_dict = {f + '_zeropoints': zeropoints}
p.add_output_values(obj=epoch, instrument='FORS2', params=output_dict)
zeropoint_tbl.write(output_path_final_f + "zeropoints.csv",
format="ascii.csv",
overwrite=True)
def main(data_title, sextractor_path, origin, destination):
print(
"\nExecuting Python script pipeline_fors2/4-divide_by_exp_time.py, with:"
)
print(f"\tepoch {data_title}")
print(f"\tsextractor path {sextractor_path}")
print(f"\torigin directory {origin}")
print(f"\tdestination directory {destination}")
print()
properties = p.object_params_fors2(data_title)
outputs = p.object_output_params(obj=data_title, instrument='FORS2')
data_dir = properties['data_dir']
if sextractor_path is not None:
if not os.path.isdir(sextractor_path):
os.mkdir(sextractor_path)
do_sextractor = True
else:
do_sextractor = False
origin_path = data_dir + origin
destination_path = data_dir + destination
u.mkdir_check(destination_path)
u.mkdir_check(destination_path + "science/")
u.mkdir_check(destination_path + "backgrounds/")
filters = outputs['filters']
for fil in filters:
u.mkdir_check(destination_path + "science/" + fil)
u.mkdir_check(destination_path + "backgrounds/" + fil)
if do_sextractor:
if not os.path.isdir(sextractor_path + fil):
os.mkdir(sextractor_path + fil)
files = os.listdir(origin_path + "science/" + fil + "/")
for file_name in files:
if file_name[-5:] == '.fits':
science_origin = origin_path + "science/" + fil + "/" + file_name
science_destination = destination_path + "science/" + fil + "/" + file_name.replace(
"trim", "norm")
background_origin = origin_path + "backgrounds/" + fil + "/" + file_name.replace(
"SCIENCE_REDUCED", "PHOT_BACKGROUND_SCI")
background_destination = destination_path + "backgrounds/" + fil + "/" + \
file_name.replace("SCIENCE_REDUCED", "PHOT_BACKGROUND_SCI").replace("trim",
"norm")
print(science_origin)
# Divide by exposure time to get an image in counts/second.
ff.divide_by_exp_time(file=science_origin,
output=science_destination)
ff.divide_by_exp_time(file=background_origin,
output=background_destination)
if do_sextractor:
copyfile(
science_destination, sextractor_path + fil + "/" +
file_name.replace("trim", "norm"))
if os.path.isfile(origin_path + data_title + '.log'):
copyfile(origin_path + data_title + '.log',
destination_path + data_title + ".log")
u.write_log(path=destination_path + data_title + ".log",
action=f'Divided by exposure time.')
def main(data_title: 'str', delete_output: bool = True):
print(
"\nExecuting Python script pipeline_fors2/2-sort_after_esoreflex.py, with:"
)
print(f"\tepoch {data_title}")
print()
eso_dir = p.config['esoreflex_output_dir']
if os.path.isdir(eso_dir):
obj_params = p.object_params_fors2(data_title)
data_dir = obj_params['data_dir']
destination = data_dir + "2-sorted/"
date = None
output_values = p.object_output_params(obj=data_title,
instrument='FORS2')
mjd = int(output_values['mjd_obs'])
obj = output_values['object']
print(
f"Looking for data with object '{obj}' and MJD of observation {mjd} inside {eso_dir}"
)
# Look for files with the appropriate object and MJD, as recorded in output_values
# List directories in eso_output_dir; these are dates on which data was reduced using ESOReflex.
eso_dirs = filter(lambda d: os.path.isdir(eso_dir + "/" + d),
os.listdir(eso_dir))
for directory in eso_dirs:
directory += "/"
# List directories within 'reduction date' directories.
# These should represent individual images reduced.
print(f"Searching {eso_dir + directory}")
eso_subdirs = filter(
lambda d: os.path.isdir(eso_dir + directory + d) and "FORS2" in
d, os.listdir(eso_dir + directory))
for subdirectory in eso_subdirs:
subdirectory += "/"
subpath = eso_dir + "/" + directory + subdirectory
print(f"\tSearching {subpath}")
# Get the files within the image directory.
files = filter(lambda d: os.path.isfile(subpath + d),
os.listdir(subpath))
for file_name in files:
# Retrieve the target object name from the fits file.
file_path = subpath + file_name
file = fits.open(file_path)
file_obj = ff.get_object(file)
file_mjd = int(ff.get_header_attribute(file, 'MJD-OBS'))
file_filter = ff.get_filter(file)
# Check the object name and observation date against those of the epoch we're concerned with.
if file_obj == obj and file_mjd == mjd:
# Check which type of file we have.
date = directory[:-1]
if file_name[-28:] == "PHOT_BACKGROUND_SCI_IMG.fits":
file_destination = f"{destination}/backgrounds/"
suffix = "PHOT_BACKGROUND_SCI_IMG.fits"
elif file_name[-25:] == "OBJECT_TABLE_SCI_IMG.fits":
file_destination = f"{destination}/obj_tbls/"
suffix = "OBJECT_TABLE_SCI_IMG.fits"
elif file_name[-24:] == "SCIENCE_REDUCED_IMG.fits":
file_destination = f"{destination}/science/"
suffix = "SCIENCE_REDUCED_IMG.fits"
else:
file_destination = f"{destination}/sources/"
suffix = "SOURCES_SCI_IMG.fits"
# Make this directory, if it doesn't already exist.
u.mkdir_check(file_destination)
# Make a subdirectory by filter.
file_destination += file_filter + "/"
u.mkdir_check(file_destination)
# Title new file.
file_destination += f"{data_title}_{subdirectory[:-1]}_{suffix}"
# Copy file to new location.
print(
f"Copying: {file_path} to \n\t {file_destination}")
file.writeto(file_destination, overwrite=True)
if delete_output and os.path.isfile(file_destination):
os.remove(file_path)
if not os.path.isfile(f"{destination}/{data_title}.log"):
sh.copy(f"{data_dir}0-data_with_raw_calibs/{data_title}.log",
f"{destination}/{data_title}.log")
u.write_log(path=destination + data_title + ".log",
action=f'Data reduced with ESOReflex.',
date=date)
u.write_log(path=destination + data_title + ".log",
action='Files sorted using 2-sort_after_esoreflex.sh')
else:
print(f"ESO output directory '{eso_dir}' not found.")
exit(1)
