def main(data_dir, data_title, destination, fil, object):
table = (destination + '/' + fil, science_only=False)
fil = fil.replace('/', '')
header_file = destination + '/' + fil + '_template.hdr'
header_stream = open(header_file, 'r')
header = header_stream.readlines()
header_stream.close()
# Transfer and/or modify header information
params = p.object_output_params(data_title, instrument='IMACS')
airmass = table['airmass'].mean()
saturate = table['saturate'].mean()
old_gain = table['gain'].mean()
n_frames = params[fil + '_n_exposures']
gain = gain_median_combine(old_gain=old_gain, n_frames=n_frames)
header.insert(-1, f'AIRMASS = {airmass}\n')
header.insert(-1, f'FILTER = {fil}\n')
header.insert(-1, f'OBJECT = {object}\n')
header.insert(-1, f'EXPTIME = 1.\n')
header.insert(-1, f'GAIN = {gain}\n')
header.insert(-1, f'SATURATE= {saturate}\n')
header.insert(-1, f'MJD-OBS = {float(np.nanmean(table["mjd_obs"]))}\n')
os.remove(header_file)
with open(header_file, 'w') as file:
file.writelines(header)
p.add_output_path(obj=data_title, key=fil + '_subtraction_image',
path=destination + '/' + fil + '_coadded.fits', instrument='IMACS')
def main(ob, path):
print("\nExecuting Python script pipeline_fors2/9-esorex_zeropoint_prep.py, with:")
print(f"\tepoch {ob}")
print(f"\tcalibration data path {path}")
print()
output = object_output_params(obj=ob, instrument='FORS2')
filters = output['filters']
for f in filters:
mkdir_check(path + '/' + f)
def main(obj, test, mag_min, mag_max, increment, instrument):
properties = p.object_params_instrument(obj, instrument=instrument)
output = p.object_output_params(obj=obj, instrument=instrument)
paths = p.object_output_paths(obj=obj, instrument=instrument)
synth_path = properties['data_dir'] + 'synthetic/'
u.mkdir_check(synth_path)
synth_path = synth_path + 'frb_position/'
u.mkdir_check(synth_path)
now = time.Time.now()
now.format = 'isot'
synth_path += f'range_{now}/'
u.mkdir_check(synth_path)
filters = output['filters']
for magnitude in np.arange(mag_min, mag_max, increment):
magnitudes = []
for i in range(len(filters)):
magnitudes.append(magnitude)
test_spec = test + '_' + str(u.round_to_sig_fig(magnitude, 4))
test_path = synth_path + test_spec + '/'
ph.insert_synthetic_at_frb(obj=properties,
test_path=test_path,
filters=filters,
magnitudes=magnitudes,
add_path=False,
psf=True,
output_properties=output,
instrument=instrument,
paths=paths)
p.add_output_path(obj=obj,
key='subtraction_image_synth_frb_range',
path=synth_path,
instrument=instrument)
def main(obj, test, n, filter_dist, instrument, limit):
properties = p.object_params_instrument(obj, instrument=instrument)
burst_properties = p.object_params_frb(obj=obj[:-2])
output = p.object_output_params(obj=obj, instrument=instrument)
paths = p.object_output_paths(obj=obj, instrument=instrument)
z = burst_properties['z']
mjd_burst = burst_properties['mjd_burst']
ebv_mw = burst_properties['dust_ebv']
mjd_obs = properties['mjd']
synth_path = properties['data_dir'] + 'synthetic/'
u.mkdir_check(synth_path)
synth_path = synth_path + 'sn_random/'
u.mkdir_check(synth_path)
epoch = mjd_obs - mjd_burst
f_0 = filter_dist[0]
hg_ra = burst_properties['hg_ra']
hg_dec = burst_properties['hg_dec']
burst_ra = burst_properties['burst_ra']
burst_dec = burst_properties['burst_dec']
image_path = paths[f_0 + '_' + properties['subtraction_image']]
image = fits.open(image_path)
wcs_info = wcs.WCS(image[0].header)
burst_x, burst_y = wcs_info.all_world2pix(burst_ra, burst_dec, 0)
now = time.Time.now()
now.format = 'isot'
synth_path += test + '_' + str(now) + '/'
u.mkdir_check(synth_path)
filters = output['filters']
psf_models = []
for i, f in enumerate(filters):
sn.register_filter(f=f, instrument=instrument)
psf_model = fits.open(paths[f[0] + '_psf_model'])
psf_models.append(psf_model)
for i in range(n):
test_spec = test + '_' + str(i)
test_path = synth_path + test_spec + '/'
magnitudes = []
mags_filters, model, x, y, tbl = sn.random_light_curves_type_ia(
filters=filters,
image=image,
hg_ra=hg_ra,
hg_dec=hg_dec,
z=z,
ebv_mw=ebv_mw,
output_path=test_path,
output_title=test,
limit=limit,
x=burst_x,
y=burst_y,
ra=burst_ra,
dec=burst_dec)
days = mags_filters['days']
for f in filters:
magnitude = sn.magnitude_at_epoch(epoch=epoch,
days=days,
mags=mags_filters[f])
print(f, 'mag:', magnitude)
magnitudes.append(magnitude)
ph.insert_synthetic(x=float(x),
y=float(y),
obj=properties,
test_path=test_path,
filters=filters,
magnitudes=magnitudes,
suffix='sn_random_random_ia',
extra_values=tbl,
paths=paths,
output_properties=output,
psf_models=psf_models,
instrument=instrument)
p.add_output_path(obj=obj,
key='subtraction_image_synth_sn_random_ia',
path=synth_path,
instrument=instrument)
def main(epoch, origin, destination):
print("\nExecuting Python script pipeline_fors2/4.1-insert_test_synth.py, with:")
print(f"\tepoch {epoch}")
print(f"\torigin directory {origin}")
print(f"\tdestination directory {destination}")
print()
epoch_params = p.object_params_fors2(obj=epoch)
outputs = p.object_output_params(obj=epoch, instrument='FORS2')
data_dir = epoch_params['data_dir']
insert = epoch_params['test_synths']
origin_path = data_dir + "analysis/sextractor/" + origin
destination_path = data_dir + destination
u.mkdir_check(destination_path)
u.mkdir_check(destination_path + "science/")
u.mkdir_check(destination_path + "backgrounds/")
filters = outputs['filters']
for fil in filters:
f = fil[0]
path_fil_output = destination_path + "science/" + fil + "/"
path_fil_input = origin_path + fil + "/"
u.mkdir_check(path_fil_output)
u.mkdir_check(destination_path + "backgrounds/" + fil)
zeropoint, _, airmass, _, extinction, _ = ph.select_zeropoint(obj=epoch,
filt=fil,
instrument='fors2',
outputs=outputs)
print(path_fil_input)
# print(os.listdir(path_fil_input))
for fits_file in filter(lambda f: f.endswith("_norm.fits"), os.listdir(path_fil_input)):
print(fits_file)
path_fits_file_input = path_fil_input + fits_file
path_fits_file_output = path_fil_output + fits_file
path_psf_model = path_fits_file_input.replace(".fits", "_psfex.psf")
try:
ph.insert_point_sources_to_file(file=path_fits_file_input,
x=array(insert["ra"]),
y=array(insert["dec"]),
mag=insert[f"{f}_mag"],
output=path_fits_file_output,
zeropoint=zeropoint,
extinction=extinction,
airmass=airmass,
world_coordinates=True,
psf_model=path_psf_model
)
except ValueError:
ph.insert_point_sources_to_file(file=path_fits_file_input,
x=array(insert["ra"]),
y=array(insert["dec"]),
mag=insert[f"{f}_mag"],
output=path_fits_file_output,
zeropoint=zeropoint,
extinction=extinction,
airmass=airmass,
world_coordinates=True,
fwhm=fits.open(path_psf_model)[1].header['PSF_FWHM']
)
if os.path.isfile(origin_path + epoch + '.log'):
copyfile(origin_path + epoch + '.log', destination_path + epoch + ".log")
u.write_log(path=destination_path + epoch + ".log", action=f'Divided by exposure time.')
def main(data_dir, data_title, origin, destination, all_synths):
print("\nExecuting Python script pipeline_fors2/5-background_subtract.py, with:")
print(f"\tepoch {data_title}")
print(f"\torigin directory {origin}")
print(f"\tdestination directory {destination}")
print()
methods = ["ESO backgrounds only", "SExtractor backgrounds only", "polynomial fit", "Gaussian fit", "median value"]
if all_synths:
frame = 56
method = "polynomial fit"
degree = 5
do_mask = True
local = True
global_sub = False
trim_image = False
recorrect_subbed = True
eso_back = False
else:
frame = 200
# frame_arcsec = 30 * units.arcsec
# frame_deg = frame_arcsec.to(units.deg)
eso_back = False
_, method = u.select_option(message="Please select the background subtraction method.", options=methods,
default="polynomial fit")
degree = None
if method == "polynomial fit":
degree = u.user_input(message=f"Please enter the degree of {method} to use:", typ=int, default=3)
elif method == "ESO backgrounds only":
eso_back = True
do_mask = False
if method not in ["ESO backgrounds only", "SExtractor backgrounds only", "median value"]:
do_mask = u.select_yn(message="Mask sources using SExtractor catalogue?", default=True)
if method in ["polynomial fit", "Gaussian fit"]:
local = u.select_yn(message="Use a local fit?", default=True)
else:
local = False
global_sub = False
trim_image = False
recorrect_subbed = False
if local:
global_sub = u.select_yn(message="Subtract local fit from entire image?", default="n")
if not global_sub:
trim_image = u.select_yn(message="Trim images to subtracted region?", default="y")
recorrect_subbed = u.select_yn(message="Re-normalise background of subtracted region?", default="y")
# if not eso_back and method != "SExtractor backgrounds only":
# eso_back = u.select_yn(message="Subtract ESO Reflex fitted backgrounds first?", default=False)
outputs = p.object_output_params(data_title, instrument='FORS2')
data_dir = u.check_trailing_slash(data_dir)
destination = u.check_trailing_slash(destination)
destination = data_dir + destination
u.mkdir_check_nested(destination)
origin = u.check_trailing_slash(origin)
science_origin = data_dir + origin + "science/"
print(science_origin)
filters = outputs['filters']
frb_params = p.object_params_frb(obj=data_title[:-2])
epoch_params = p.object_params_fors2(obj=data_title)
background_origin_eso = ""
if eso_back:
background_origin_eso = data_dir + "/" + origin + "/backgrounds/"
if method == "SExtractor backgrounds only":
background_origin = f"{data_dir}{origin}backgrounds_sextractor/"
elif method == "polynomial fit":
background_origin = f"{destination}backgrounds/" # f"{destination}backgrounds_{method.replace(' ', '')}_degree_{degree}_local_{local}_globalsub_{global_sub}/"
else:
background_origin = f"{destination}backgrounds/" # f"{destination}backgrounds_{method.replace(' ', '')}_local_{local}_globalsub_{global_sub}/"
trimmed_path = ""
if trim_image:
trimmed_path = f"{data_dir}{origin}trimmed_to_background/"
u.mkdir_check_nested(trimmed_path)
ra = frb_params["burst_ra"]
dec = frb_params["burst_dec"]
if all_synths:
ras = epoch_params["test_synths"]["ra"]
decs = epoch_params["test_synths"]["dec"]
else:
ras = [ra]
decs = [dec]
for fil in filters:
trimmed_path_fil = ""
if trim_image:
trimmed_path_fil = f"{trimmed_path}{fil}/"
u.mkdir_check(trimmed_path_fil)
background_fil_dir = f"{background_origin}{fil}/"
u.mkdir_check_nested(background_fil_dir)
science_destination_fil = f"{destination}science/{fil}/"
u.mkdir_check_nested(science_destination_fil)
files = os.listdir(science_origin + fil + "/")
for file_name in files:
if file_name.endswith('.fits'):
new_file = file_name.replace("norm", "bg_sub")
new_path = f"{science_destination_fil}/{new_file}"
print("NEW_PATH:", new_path)
science = science_origin + fil + "/" + file_name
# First subtract ESO Reflex background images
# frame = (frame_deg / f.get_pixel_scale(file=science, astropy_units=True)[1]).to(f.pix).value
if eso_back:
background_eso = background_origin_eso + fil + "/" + file_name.replace("SCIENCE_REDUCED",
"PHOT_BACKGROUND_SCI")
ff.subtract_file(file=science, sub_file=background_eso, output=new_path)
science_image = new_path
if method != "ESO backgrounds only":
print(ra, dec)
print("Science image:", science)
science_image = fits.open(science)
print("Science file:", science_image)
wcs_this = WCS(header=science_image[0].header)
if method == "SExtractor backgrounds only":
background = background_origin + fil + "/" + file_name + "_back.fits"
print("Background image:", background)
else:
if method == "median value":
print(science_image[0].data.shape)
_, background_value, _ = sigma_clipped_stats(science_image[0].data)
background = deepcopy(science_image)
background[0].data = np.full(shape=science_image[0].data.shape, fill_value=background_value)
background_path = background_origin + fil + "/" + file_name.replace("SCIENCE_REDUCED",
"PHOT_BACKGROUND_MEDIAN")
# Next do background fitting.
else:
background = deepcopy(science_image)
background[0].data = np.zeros(background[0].data.shape)
background_path = background_origin + fil + "/" + file_name.replace("SCIENCE_REDUCED",
"PHOT_BACKGROUND_FITTED")
for i, ra in enumerate(ras):
dec = decs[i]
x, y = wcs_this.all_world2pix(ra, dec, 0)
print(x, y)
bottom, top, left, right = ff.subimage_edges(data=science_image[0].data, x=x, y=y,
frame=frame)
if do_mask:
# Produce a pixel mask that roughly masks out the true sources in the image so that
# they don't get fitted.
mask_max = 10
_, pixel_scale = ff.get_pixel_scale(science_image)
sextractor = Table.read(
f"{data_dir}analysis/sextractor/4-divided_by_exp_time/{fil}/{file_name.replace('.fits', '_psf-fit.cat')}",
format='ascii.sextractor')
weights = np.ones(shape=science_image[0].data.shape)
for obj in filter(
lambda o: left < o["X_IMAGE"] < right and bottom < o["Y_IMAGE"] < top,
sextractor):
mask_rad = min(int(obj["A_WORLD"] * obj["KRON_RADIUS"] / pixel_scale), mask_max)
x_prime = int(np.round(obj["X_IMAGE"]))
y_prime = int(np.round(obj["Y_IMAGE"]))
weights[y_prime - mask_rad:y_prime + mask_rad,
x_prime - mask_rad:x_prime + mask_rad] = 0.0
plt.imshow(weights, origin="lower")
plt.savefig(
background_origin + fil + "/" + file_name.replace("norm.fits", "mask.png"))
else:
weights = None
background_this = fit_background_fits(image=science_image,
model_type=method[:method.find(" ")],
deg=degree, local=local,
global_sub=global_sub,
centre_x=x, centre_y=y, frame=frame,
weights=weights)
background[0].data += background_this[0].data
if recorrect_subbed:
offset = get_median_background(image=science,
ra=epoch_params["renormalise_centre_ra"],
dec=epoch_params["renormalise_centre_dec"], frame=50,
show=False,
output=new_path[
:new_path.find("bg_sub")] + "renorm_patch_")
print("RECORRECT_SUBBED:", recorrect_subbed)
print("SUBTRACTING FROM BACKGROUND:", offset)
print(bottom, top, left, right)
print(background[0].data[bottom:top, left:right].shape)
print(np.median(background[0].data[bottom:top, left:right]))
background[0].data[bottom:top, left:right] -= offset
print(np.median(background[0].data[bottom:top, left:right]))
if trim_image:
print("TRIMMED_PATH_FIL:", trimmed_path_fil)
science_image = ff.trim_file(path=science_image, left=left, right=right, top=top,
bottom=bottom,
new_path=trimmed_path_fil + file_name.replace(
"norm.fits",
"trimmed_to_back.fits"))
print("Science after trim:", science_image)
background = ff.trim_file(path=background, left=left, right=right, top=top,
bottom=bottom,
new_path=background_path)
print("Writing background to:")
print(background_path)
background.writeto(background_path, overwrite=True)
print("SCIENCE:", science_image)
print("BACKGROUND:", background)
subbed = ff.subtract_file(file=science_image, sub_file=background, output=new_path)
# # TODO: check if regions overlap
#
# plt.hist(subbed[0].data[int(y - frame + 1):int(y + frame - 1),
# int(x - frame + 1):int(x + frame - 1)].flatten(),
# bins=10)
# plt.savefig(new_path[:new_path.find("bg_sub")] + "histplot.png")
# plt.close()
copyfile(data_dir + "/" + origin + "/" + data_title + ".log", destination + data_title + ".log")
u.write_log(path=destination + data_title + ".log",
action=f'Backgrounds subtracted using 4-background_subtract.py with method {method}\n')
def main(field, destination, epoch, instrument, template_instrument,
comparison_type):
if destination[-1] != '/':
destination = destination + '/'
p.refresh_params_frbs()
types = ['normal', 'synth_random', 'synth_frb']
if comparison_type not in types:
raise ValueError(comparison_type +
' is not a valid synthetic argument; choose from ' +
str(types))
if comparison_type == 'normal':
comparison_type = ''
else:
comparison_type = '_' + comparison_type
params = p.object_params_frb(field)
u.mkdir_check(f'{params["data_dir"]}subtraction/')
destination_path = f'{params["data_dir"]}subtraction/{destination}/'
u.mkdir_check(destination_path)
comparison_title = f'{field}_{epoch}'
comparison_paths = p.object_output_paths(obj=comparison_title,
instrument=instrument)
comparison_params = p.object_params_instrument(obj=comparison_title,
instrument=instrument)
params = p.object_params_frb(field)
template_epoch = params['template_epoch_' + template_instrument.lower()]
template_title = f'{field}_{template_epoch}'
template_paths = p.object_output_paths(obj=template_title,
instrument=template_instrument)
template_outputs = p.object_output_params(obj=template_title,
instrument=template_instrument)
template_params = p.object_params_instrument(
obj=template_title, instrument=template_instrument)
filters = params['filters']
for f in filters:
values = {}
f_0 = f[0]
destination_path_filter = f'{destination_path}{f}/'
u.mkdir_check(destination_path_filter)
# COMPARISON IMAGE:
comparison_image_name = comparison_params[
'subtraction_image'] + comparison_type
# Get path to comparison image from parameter .yaml file
if f'{f_0}_{comparison_image_name}' in comparison_paths:
comparison_origin = comparison_paths[
f'{f_0}_{comparison_image_name}']
elif f'{f_0.lower()}_{comparison_image_name}' in comparison_paths:
comparison_origin = comparison_paths[
f'{f_0.lower()}_{comparison_image_name}']
else:
raise ValueError(
f'{f_0.lower()}_{comparison_image_name} not found in {comparison_title} paths'
)
comparison_destination = f'{comparison_title}_comparison.fits'
if comparison_type != '':
shutil.copyfile(
comparison_origin.replace('.fits',
'.csv'), destination_path_filter +
comparison_destination.replace('.fits', '.csv'))
print('Copying comparison image')
print('From:')
print('\t', comparison_origin)
print('To:')
print(f'\t {destination_path}{f}/{comparison_destination}')
shutil.copy(
comparison_params['data_dir'] + 'output_values.yaml',
f'{destination_path}{f}/{comparison_title}_comparison_output_values.yaml'
)
shutil.copy(
comparison_params['data_dir'] + 'output_values.json',
f'{destination_path}{f}/{comparison_title}_comparison_output_values.json'
)
shutil.copy(comparison_origin,
f'{destination_path}{f}/{comparison_destination}')
values['comparison_file'] = comparison_origin
# TEMPLATE IMAGE
if template_instrument != 'FORS2' and template_instrument != 'XSHOOTER':
f_0 = f_0.lower()
template_image_name = template_params[
'subtraction_image'] + comparison_type
if f'{f_0}_{template_image_name}' in template_paths:
template_origin = template_paths[f'{f_0}_{template_image_name}']
elif f'{f_0.lower()}_{template_image_name}' in template_paths:
template_origin = template_paths[
f'{f_0.lower()}_{template_image_name}']
else:
raise ValueError(
f'{f_0.lower()}_{template_image_name} not found in {template_title} paths'
)
fwhm_template = template_outputs[f_0 + '_fwhm_pix']
template_destination = f'{template_title}_template.fits'
print('Copying template')
print('From:')
print('\t', template_origin)
print('To:')
print(f'\t {destination_path}{f}/{template_destination}')
shutil.copy(
template_params['data_dir'] + 'output_values.yaml',
f'{destination_path}{f}/{template_title}_template_output_values.yaml'
)
shutil.copy(
template_params['data_dir'] + 'output_values.json',
f'{destination_path}{f}/{template_title}_template_output_values.json'
)
shutil.copy(template_origin,
f'{destination_path}{f}/{template_destination}')
values['template_file'] = comparison_origin
p.add_params(f'{destination_path}{f}/output_values.yaml', values)
def main(obj, test, n, filter_dist, sn_type, instrument, limit):
properties = p.object_params_instrument(obj, instrument=instrument)
burst_properties = p.object_params_frb(obj=obj[:-2])
output = p.object_output_params(obj=obj, instrument=instrument)
paths = p.object_output_paths(obj=obj, instrument=instrument)
z = burst_properties['z']
mjd_burst = burst_properties['mjd_burst']
ebv_mw = burst_properties['dust_ebv']
mjd_obs = properties['mjd']
synth_path = properties['data_dir'] + 'synthetic/'
u.mkdir_check(synth_path)
synth_path = synth_path + 'sn_random/'
u.mkdir_check(synth_path)
epoch = mjd_obs - mjd_burst
f_0 = filter_dist[0]
hg_ra = burst_properties['hg_ra']
hg_dec = burst_properties['hg_dec']
image_path = paths[f_0 + '_' + properties['subtraction_image']]
image = fits.open(image_path)
now = time.Time.now()
now.format = 'isot'
test += sn_type + '_'
synth_path += test + str(now) + '/'
u.mkdir_check(synth_path)
filters = output['filters']
if 'FRB190608' in obj:
limit = 45
psf_models = []
for f in filters:
sn.register_filter(f=f)
psf_model = fits.open(paths[f[0] + '_psf_model'])
psf_models.append(psf_model)
for i in range(n):
test_spec = test + str(i)
test_path = synth_path + test_spec + '/'
magnitudes = []
mags_filters, model, x, y, tbl = sn.random_light_curves(
sn_type=sn_type,
filters=filters,
image=image,
hg_ra=hg_ra,
hg_dec=hg_dec,
z=z,
ebv_mw=ebv_mw,
output_path=test_path,
output_title=test,
limit=limit)
days = mags_filters['days']
for f in filters:
magnitude = sn.magnitude_at_epoch(epoch=epoch,
days=days,
mags=mags_filters[f])
print(f, 'mag:', magnitude)
magnitudes.append(magnitude)
ph.insert_synthetic(x=x,
y=y,
obj=properties,
test_path=test_path,
filters=filters,
magnitudes=magnitudes,
suffix='sn_random_random_' + sn_type,
extra_values=tbl,
paths=paths,
output_properties=output,
psf_models=psf_models,
instrument=instrument)
p.add_output_path(obj=obj,
key='subtraction_image_synth_sn_random_' + sn_type,
path=synth_path,
instrument=instrument)
def main(obj, test, n, mag_lower, mag_upper, colour_upper, colour_lower):
properties = p.object_params_fors2(obj)
output = p.object_output_params(obj=obj, instrument='FORS2')
paths = p.object_output_paths(obj)
burst_properties = p.object_params_frb(obj[:-2])
synth_path = properties['data_dir'] + 'synthetic/'
u.mkdir_check(synth_path)
synth_path = synth_path + 'random/'
u.mkdir_check(synth_path)
now = time.Time.now()
now.format = 'isot'
test = str(now) + '_' + test
test_path = synth_path + test + '/'
u.mkdir_check(test_path)
filters = {}
bluest = None
bluest_lambda = np.inf
for f in output['filters']:
filter_properties = p.filter_params(f=f, instrument='FORS2')
filters[f] = filter_properties
lambda_eff = filter_properties['lambda_eff']
if lambda_eff < bluest_lambda:
bluest_lambda = lambda_eff
bluest = f
# Insert random sources in the bluest filter.
f_0 = bluest[0]
output_properties = p.object_output_params(obj)
fwhm = output_properties[f_0 + '_fwhm_pix']
zeropoint, _, airmass, _ = ph.select_zeropoint(obj,
bluest,
instrument='fors2')
base_path = paths[f_0 + '_subtraction_image']
output_path = test_path + f_0 + '_random_sources.fits'
_, sources = ph.insert_random_point_sources_to_file(file=base_path,
fwhm=fwhm,
output=output_path,
n=n,
airmass=airmass,
zeropoint=zeropoint)
p.add_output_path(obj=obj,
key=f_0 + '_subtraction_image_synth_random',
path=output_path)
# Now insert sources at the same positions in other filters, but with magnitudes randomised.
for f in filters:
if f != bluest:
f_0 = f[0]
output_properties = p.object_output_params(obj)
fwhm = output_properties[f_0 + '_fwhm_pix']
zeropoint, _, airmass, _ = ph.select_zeropoint(obj,
f,
instrument='fors2')
base_path = paths[f_0 + '_subtraction_image']
mag = np.random.uniform(mag_lower, mag_upper, size=n)
output_path = test_path + f_0 + '_random_sources.fits'
ph.insert_point_sources_to_file(file=base_path,
fwhm=fwhm,
output=output_path,
x=sources['x_0'],
y=sources['y_0'],
mag=mag,
airmass=airmass,
zeropoint=zeropoint)
p.add_output_path(obj=obj,
key=f_0 + '_subtraction_image_synth_random',
path=output_path)
def main(origin_dir, output_dir, data_title, sextractor_path):
print("\nExecuting Python script pipeline_fors2/3-trim.py, with:")
print(f"\tepoch {data_title}")
print(f"\torigin directory {origin_dir}")
print(f"\toutput directory {output_dir}")
print()
# If this is None, we don't want the SExtractor components to be performed.
if sextractor_path is not None:
if not os.path.isdir(sextractor_path):
os.mkdir(sextractor_path)
do_sextractor = True
print(os.getcwd())
else:
do_sextractor = False
if not os.path.isdir(output_dir):
os.mkdir(output_dir)
if not os.path.isdir(output_dir + "backgrounds/"):
os.mkdir(output_dir + "backgrounds/")
if not os.path.isdir(output_dir + "science/"):
os.mkdir(output_dir + "science/")
wdir = origin_dir + "backgrounds/"
epoch_params = p.object_params_fors2(obj=data_title)
outputs = p.object_output_params(obj=data_title)
fils = outputs["filters"]
edged = False
up_left = 0
up_right = 0
up_bottom = 0
up_top = 0
dn_left = 0
dn_right = 0
dn_bottom = 0
dn_top = 0
for fil in fils:
print(output_dir + "backgrounds/" + fil)
if not os.path.isdir(output_dir + "backgrounds/" + fil):
os.mkdir(output_dir + "backgrounds/" + fil)
print('HERE:')
print(wdir + fil)
files = os.listdir(wdir + fil)
files.sort()
if not edged:
# Find borders of noise frame using backgrounds.
# First, make sure that the background we're using is for the top chip.
i = 0
while f.get_chip_num(wdir + fil + "/" + files[i]) != 1:
i += 1
up_left, up_right, up_bottom, up_top = f.detect_edges(wdir + fil +
"/" +
files[i])
# Ditto for the bottom chip.
i = 0
while f.get_chip_num(wdir + fil + "/" + files[i]) != 2:
i += 1
dn_left, dn_right, dn_bottom, dn_top = f.detect_edges(wdir + fil +
"/" +
files[i])
up_left = up_left + 5
up_right = up_right - 5
up_top = up_top - 5
dn_left = dn_left + 5
dn_right = dn_right - 5
dn_bottom = dn_bottom + 5
print('Upper chip:')
print(up_left, up_right, up_top, up_bottom)
print('Lower:')
print(dn_left, dn_right, dn_top, dn_bottom)
edged = True
for i, file in enumerate(files):
print(f'{i} {file}')
for i, file in enumerate(files):
new_path = output_dir + "backgrounds/" + fil + "/" + file.replace(
".fits", "_trim.fits")
# Add GAIN and SATURATE keywords to headers.
path = wdir + fil + "/" + file
print(f'{i} {file}')
# Split the files into upper CCD and lower CCD
if f.get_chip_num(path) == 1:
print('Upper Chip:')
f.trim_file(path,
left=up_left,
right=up_right,
top=up_top,
bottom=up_bottom,
new_path=new_path)
elif f.get_chip_num(path) == 2:
print('Lower Chip:')
f.trim_file(path,
left=dn_left,
right=dn_right,
top=dn_top,
bottom=dn_bottom,
new_path=new_path)
else:
raise ValueError(
'Invalid chip ID; could not trim based on upper or lower chip.'
)
# Repeat for science images
wdir = origin_dir + "science/"
fils = os.listdir(wdir)
for fil in fils:
print(output_dir + "science/" + fil)
if do_sextractor:
if not os.path.isdir(sextractor_path + fil):
os.mkdir(sextractor_path + fil)
if not os.path.isdir(output_dir + "science/" + fil):
os.mkdir(output_dir + "science/" + fil)
files = os.listdir(wdir + fil)
files.sort()
for i, file in enumerate(files):
print(f'{i} {file}')
for i, file in enumerate(files):
# Split the files into upper CCD and lower CCD, with even-numbered being upper and odd-numbered being lower
new_file = file.replace(".fits", "_trim.fits")
new_path = output_dir + "science/" + fil + "/" + new_file
path = wdir + fil + "/" + file
f.change_header(file=path, key='GAIN', value=0.8)
f.change_header(file=path, key='SATURATE', value=65535.)
if f.get_chip_num(path) == 1:
print('Upper Chip:')
f.trim_file(path,
left=up_left,
right=up_right,
top=up_top,
bottom=up_bottom,
new_path=new_path)
if do_sextractor:
copyfile(new_path, sextractor_path + fil + "/" + new_file)
elif f.get_chip_num(path) == 2:
print('Lower Chip:')
f.trim_file(path,
left=dn_left,
right=dn_right,
top=dn_top,
bottom=dn_bottom,
new_path=new_path)
try:
copyfile(origin_dir + data_title + ".log",
output_dir + data_title + ".log")
except FileNotFoundError:
print("Previous log not found.")
u.write_log(path=output_dir + data_title + ".log",
action='Edges trimmed using 3-trim.py\n')
def main(field, destination, epoch, instrument, template_instrument,
comparison_type):
if destination[-1] != '/':
destination = destination + '/'
# p.refresh_params_frbs()
# types = ['multi_frb_range', 'multi_sn_models', 'multi_sn_random', 'multi_sn_random_ia', 'multi_sn_random_ib']
# if comparison_type not in types:
# raise ValueError(comparison_type + ' is not a valid multi-synthetic argument; choose from ' + str(types))
comparison_type = '_' + comparison_type
type_suffix = comparison_type[7:]
params = p.object_params_frb(field)
u.mkdir_check(f'{params["data_dir"]}subtraction/')
destination_path = f'{params["data_dir"]}subtraction/{destination}/'
u.mkdir_check(destination_path)
comparison_title = f'{field}_{epoch}'
comparison_paths = p.object_output_paths(obj=comparison_title,
instrument=instrument)
comparison_origin_top = comparison_paths['subtraction_image_synth_' +
type_suffix]
comparison_tests = os.listdir(comparison_origin_top)
comparison_params = p.object_params_instrument(obj=comparison_title,
instrument=instrument)
params = p.object_params_frb(field)
template_epoch = params['template_epoch_' + template_instrument.lower()]
template_title = f'{field}_{template_epoch}'
template_paths = p.object_output_paths(obj=template_title,
instrument=template_instrument)
template_outputs = p.object_output_params(obj=template_title,
instrument=template_instrument)
template_params = p.object_params_instrument(
obj=template_title, instrument=template_instrument)
filters = params['filters']
for test in comparison_tests:
destination_test = destination_path + test + '/'
origin_test = comparison_origin_top + test + '/'
u.mkdir_check(destination_test)
test_files = os.listdir(origin_test)
for f in filters:
for file in filter(lambda fil: fil[:2] != f + '_',
os.listdir(origin_test)):
shutil.copyfile(origin_test + file, destination_test + file)
values = {}
f_0 = f[0]
destination_path_filter = f'{destination_test}{f}/'
u.mkdir_check(destination_path_filter)
# COMPARISON IMAGE:
# Get path to comparison image from parameter .yaml file
comparison_origin = origin_test + filter(
lambda file: file[0] == f_0 and file[-5:] == '.fits',
test_files).__next__()
comparison_destination = f'{comparison_title}_comparison.fits'
shutil.copyfile(comparison_origin,
destination_path_filter + comparison_destination)
shutil.copyfile(
comparison_origin.replace('.fits',
'.csv'), destination_path_filter +
comparison_destination.replace('.fits', '.csv'))
print('Copying comparison image from:')
print('\t', comparison_origin)
print('To:')
print(f'\t {destination_test}{f}/{comparison_destination}')
shutil.copy(
comparison_params['data_dir'] + 'output_values.yaml',
f'{destination_test}{f}/{comparison_title}_comparison_output_values.yaml'
)
shutil.copy(
comparison_params['data_dir'] + 'output_values.json',
f'{destination_test}{f}/{comparison_title}_comparison_output_values.json'
)
shutil.copy(comparison_origin,
f'{destination_test}{f}/{comparison_destination}')
values['comparison_file'] = comparison_origin
if template_instrument != 'FORS2':
# TEMPLATE IMAGE
if template_instrument != 'XSHOOTER':
f_0 = f_0.lower()
template_image_name = template_params['subtraction_image']
if f'{f_0}_{template_image_name}' in template_paths:
template_origin = template_paths[
f'{f_0}_{template_image_name}']
elif f'{f_0.lower()}_{template_image_name}' in template_paths:
template_origin = template_paths[
f'{f_0.lower()}_{template_image_name}']
else:
raise ValueError(
f'{f_0.lower()}_{template_image_name} not found in {template_title} paths'
)
fwhm_template = template_outputs[f_0 + '_fwhm_pix']
template_destination = f'{template_title}_template.fits'
print('Copying template from:')
print('\t', template_origin)
print('To:')
print(f'\t {destination_test}{f}/{template_destination}')
shutil.copy(
template_params['data_dir'] + 'output_values.yaml',
f'{destination_test}{f}/{template_title}_template_output_values.yaml'
)
shutil.copy(
template_params['data_dir'] + 'output_values.json',
f'{destination_test}{f}/{template_title}_template_output_values.json'
)
shutil.copy(template_origin,
f'{destination_test}{f}/{template_destination}')
values['template_file'] = comparison_origin
p.add_params(f'{destination_test}{f}/output_values.yaml',
values)
def tweak_final(sextractor_path: str,
destination: str,
epoch: str,
instrument: str,
show: bool,
tolerance: float = 10.,
output_suffix: str = 'astrometry',
input_suffix='coadded',
stars_only: bool = False,
path_add: str = 'subtraction_image',
manual: bool = False,
specific_star: bool = False):
"""
A wrapper for tweak, to interface with the .yaml param files and provide offsets to all filters used in an
observation.
:param sextractor_path: Path to SExtractor-generated catalogue.
:param destination: Directory to write tweaked image to.
:param epoch: The epoch number of the observation to be tweaked.
:param instrument: The instrument on which the observation was taken.
:param show: Plot matches onscreen?
:param tolerance: Tolerance, in pixels, within which matches will be accepted.
:param output_suffix: String to append to filename of output file.
:param input_suffix: Suffix appended to filenmae of input file.
:param stars_only: Only match using stars, determined using SExtractor's 'class_star' output.
:param path_add: Key under which to add the output path in the 'output_paths.yaml' file.
:param manual: Use manual offset?
:param specific_star: Use a specific star to tweak? This means, instead of finding the closest matches for many
stars, alignment is attempted with a single star nearest the given position.
:return: None
"""
u.mkdir_check(destination)
properties = p.object_params_instrument(obj=epoch, instrument=instrument)
frb_properties = p.object_params_frb(obj=epoch[:-2])
cat_name = properties['cat_field_name']
manual = properties['manual_astrometry'] and manual
cat_path = frb_properties['data_dir'] + cat_name.upper(
) + "/" + cat_name.upper() + ".csv"
if cat_path is not None:
outputs = p.object_output_params(obj=epoch, instrument=instrument)
filters = outputs['filters']
param_dict = {}
for filt in filters:
print(filt)
f = filt[0]
if manual:
offset_x = properties[f + '_offset_x']
offset_y = properties[f + '_offset_y']
else:
offset_x = None
offset_y = None
if specific_star:
burst_properties = p.object_params_frb(epoch[:-2])
star_ra = burst_properties['alignment_ra']
star_dec = burst_properties['alignment_dec']
else:
star_ra = star_dec = None
param = tweak(
sextractor_path=sextractor_path + f + '_psf-fit.cat',
destination=destination + f + '_' + output_suffix + '.fits',
image_path=destination + f + '_' + input_suffix + '.fits',
cat_path=cat_path,
cat_name=cat_name,
tolerance=tolerance,
show=show,
stars_only=stars_only,
manual=manual,
offset_x=offset_x,
offset_y=offset_y,
specific_star=specific_star,
star_ra=star_ra,
star_dec=star_dec)
for par in param:
param_dict[f + '_' + par] = param[par]
p.add_output_path(obj=epoch,
key=f + '_' + path_add,
path=destination + f + '_' + output_suffix +
'.fits',
instrument=instrument)
p.add_params(properties['data_dir'] + 'output_values.yaml',
params=param_dict)
else:
print('No catalogue found for this alignment.')
def main(obj, test_name, sex_x_col, sex_y_col, sex_ra_col, sex_dec_col,
sex_flux_col, get_sextractor_names, mag_tol, stars_only,
star_class_col, star_class_tol, show_plots, mag_range_sex_lower,
mag_range_sex_upper, pix_tol, separate_chips, write):
print(separate_chips)
sextractor_names = p.sextractor_names_psf() # None to auto-detect
properties = p.object_params_des(obj=obj)
outputs = p.object_output_params(obj=obj, instrument='des')
paths = p.object_output_paths(obj=obj)
cat_name = 'DES'
cat_path = properties['data_dir'] + 'des_objects.csv'
output = properties['data_dir'] + '/3-zeropoint/'
mkdir_check(output)
output = output + 'field/'
mkdir_check(output)
for fil in properties['filters']:
f_0 = fil[0]
f_up = f_0.upper()
output_path = output + f_0
cat_zeropoint = 0.
cat_zeropoint_err = 0.
now = time.Time.now()
now.format = 'isot'
test_name = str(now) + '_' + test_name
mkdir_check(output_path)
output_path = output_path + '/' + f_0 + '/'
mkdir_check(output_path)
output_path = output_path + '/' + test_name + '/'
mkdir_check(output_path)
if not os.path.isdir(output_path):
os.mkdir(output_path)
output_path = output_path + test_name + '/'
if not os.path.isdir(output_path):
os.mkdir(output_path)
image_path = properties[
'data_dir'] + '2-sextractor/' + f_0 + '_cutout.fits'
sextractor_path = properties[
'data_dir'] + '2-sextractor/' + f_0 + '_psf-fit.cat'
# We override here because there's no need to separate the chips if we're using the DES image.
separate_chips = False
# pix_tol = 1.
mag_range_lower = 16
mag_range_upper = 25
exp_time = 1.
print('SExtractor catalogue path:', sextractor_path)
print('Image path:', image_path)
print('Catalogue path:', cat_path)
print('Output:', output_path + test_name)
print()
print(cat_zeropoint)
up = photometry.determine_zeropoint_sextractor(
sextractor_cat=sextractor_path,
image=image_path,
image_name='DES',
cat_path=cat_path,
cat_name=cat_name,
output_path=output_path + "/chip_1/",
show=show_plots,
cat_ra_col='RA',
cat_dec_col='DEC',
cat_mag_col='WAVG_MAG_PSF_' + f_up,
sex_ra_col=sex_ra_col,
sex_dec_col=sex_dec_col,
sex_x_col=sex_x_col,
sex_y_col=sex_y_col,
dist_tol=pix_tol,
mag_tol=mag_tol,
flux_column=sex_flux_col,
mag_range_cat_upper=mag_range_upper,
mag_range_cat_lower=mag_range_lower,
mag_range_sex_upper=mag_range_sex_upper,
mag_range_sex_lower=mag_range_sex_lower,
stars_only=stars_only,
star_class_tol=star_class_tol,
star_class_col=star_class_col,
exp_time=exp_time,
get_sextractor_names=get_sextractor_names,
sextractor_names=sextractor_names,
cat_type='csv',
cat_zeropoint=cat_zeropoint,
cat_zeropoint_err=cat_zeropoint_err,
)
if write:
update_dict = {
f_0 + '_zeropoint_derived':
float(up['zeropoint_sub_outliers']),
f_0 + '_zeropoint_derived_err': float(up['zeropoint_err'])
}
p.add_output_values(obj=obj, params=update_dict, instrument='des')
def main(epoch, test_name, sex_x_col, sex_y_col, sex_ra_col, sex_dec_col,
sex_flux_col, stars_only, show_plots, mag_range_sex_lower,
mag_range_sex_upper, pix_tol, instrument):
print("\nExecuting Python script pipeline_fors2/9-zeropoint.py, with:")
print(f"\tepoch {epoch}")
print()
properties = p.object_params_fors2(epoch)
outputs = p.object_output_params(obj=epoch, instrument='fors2')
proj_paths = p.config
output = properties['data_dir'] + '9-zeropoint/'
mkdir_check(output)
output_std = output + 'std/'
mkdir_check(output_std)
mkdir_check(output_std)
std_path = properties['data_dir'] + 'calibration/std_star/'
filters = outputs['filters']
std_field_path = proj_paths['top_data_dir'] + "std_fields/"
std_fields = list(
filter(lambda path: os.path.isdir(path), os.listdir(std_field_path)))
print('Standard fields with available catalogues:')
print(std_fields)
print()
cat_names = photometry_catalogues
for fil in filters:
print('Doing filter', fil)
f = fil[0]
# Obtain zeropoints from FRB field, if data is available.
print(
f"\nDetermining science-field zeropoints for {epoch}, filter {fil}:\n"
)
if f"{f}_zeropoints" in outputs:
zeropoints = outputs[f"{f}_zeropoints"]
else:
zeropoints = {}
zeropoints["science_field"] = {}
for cat_name in cat_names:
zeropoints["science_field"][
cat_name], _ = photometry.zeropoint_science_field(
epoch=epoch,
instrument=instrument,
test_name=None,
sex_x_col='XPSF_IMAGE',
sex_y_col='YPSF_IMAGE',
sex_ra_col='ALPHAPSF_SKY',
sex_dec_col='DELTAPSF_SKY',
sex_flux_col='FLUX_PSF',
stars_only=True,
star_class_col='CLASS_STAR',
star_class_tol=0.95,
show_plots=False,
mag_range_sex_lower=-100. * units.mag,
mag_range_sex_upper=100. * units.mag,
pix_tol=5. * units.pixel,
separate_chips=True,
cat_name=cat_name)
# Obtain zeropoints from available standard fields and available data.
print(
f"\nDetermining standard-field zeropoints for {epoch}, filter {fil}\n"
)
zeropoints["standard_field"] = {}
fil_path = std_path + fil + '/'
if os.path.isdir(fil_path):
fields = filter(lambda d: os.path.isdir(fil_path + d),
os.listdir(fil_path))
output_path_fil_std = output_std + '/' + fil + '/'
mkdir_check(output_path_fil_std)
for field in fields:
zeropoints["standard_field"][field] = {}
ra = float(field[field.find("RA") + 2:field.find("_")])
dec = float(field[field.find("DEC") + 3:])
print("Looking for photometry data in field " + field + ":")
mkdir_check(std_field_path + field)
field_path = fil_path + field + '/'
output_path = output_path_fil_std + field + '/'
std_cat_path = std_field_path + field + '/'
std_properties = p.load_params(field_path + 'params.yaml')
use_sex_star_class = std_properties['use_sex_star_class']
# Cycle through the three catalogues used to determine zeropoint, in order of preference.
for cat_name in cat_names:
# Check for photometry on-disk in the relevant catalogue; if none present, attempt to retrieve from
# online archive.
print(f"In {cat_name}:")
output_path_cat = output_path + cat_name
cat_path = f"{std_cat_path}{cat_name}/{cat_name}.csv"
if not (os.path.isdir(std_cat_path + cat_name)
) or os.path.isfile(cat_path):
print(
"None found on disk. Attempting retrieval from archive..."
)
if update_std_photometry(ra=ra, dec=dec,
cat=cat_name) is None:
print("\t\tNo data found in archive.")
zeropoints["standard_field"][field][
cat_name] = None
continue
column_names = cat_columns(cat=cat_name, f=f)
cat_ra_col = column_names['ra']
cat_dec_col = column_names['dec']
cat_mag_col = column_names['mag_psf']
if not use_sex_star_class:
star_class_col = column_names['class_star']
else:
star_class_col = 'CLASS_STAR'
cat_type = 'csv'
sextractor_path = field_path + 'sextractor/_psf-fit.cat'
image_path = field_path + '3-trimmed/standard_trimmed_img_up.fits'
star_class_tol = std_properties['star_class_tol']
now = time.Time.now()
now.format = 'isot'
test_name = str(now) + '_' + test_name
mkdir_check(properties['data_dir'] +
'/analysis/zeropoint/')
mkdir_check(output_path)
exp_time = ff.get_exp_time(image_path)
print('SExtractor catalogue path:', sextractor_path)
print('Image path:', image_path)
print('Catalogue name:', cat_name)
print('Catalogue path:', cat_path)
print('Class star column:', star_class_col)
print('Output:', output_path_cat)
print('Exposure time:', exp_time)
print("Use sextractor class star:", use_sex_star_class)
zeropoints["standard_field"][field][
cat_name] = photometry.determine_zeropoint_sextractor(
sextractor_cat=sextractor_path,
image=image_path,
cat_path=cat_path,
cat_name=cat_name,
output_path=output_path_cat,
show=show_plots,
cat_ra_col=cat_ra_col,
cat_dec_col=cat_dec_col,
cat_mag_col=cat_mag_col,
sex_ra_col=sex_ra_col,
sex_dec_col=sex_dec_col,
sex_x_col=sex_x_col,
sex_y_col=sex_y_col,
dist_tol=pix_tol,
flux_column=sex_flux_col,
mag_range_sex_upper=mag_range_sex_upper,
mag_range_sex_lower=mag_range_sex_lower,
stars_only=stars_only,
star_class_tol=star_class_tol,
star_class_col=star_class_col,
exp_time=exp_time,
y_lower=0,
cat_type=cat_type,
)
output_dict = {f + '_zeropoints': zeropoints}
p.add_output_values(obj=epoch, instrument='FORS2', params=output_dict)
outputs = p.object_output_params(obj=epoch, instrument='fors2')
output_path_final = output + "collated/"
mkdir_check(output_path_final)
print("Collating zeropoints...")
for fil in filters:
print(f"For {fil}:")
f = fil[0]
output_path_final_f = output_path_final + fil + "/"
mkdir_check(output_path_final_f)
zeropoints = outputs[f"{f}_zeropoints"]
airmass_sci = outputs[f"{f}_airmass_mean"]
airmass_sci_err = outputs[f"{f}_airmass_err"]
extinction = outputs[f"{f}_extinction"]
extinction_err = outputs[f"{f}_extinction_err"]
zeropoint_tbl = table.Table(
dtype=[("type", 'S15'), ("field", 'S25'), (
"cat", 'S10'), ("zeropoint",
float), ("zeropoint_err",
float), ("airmass",
float), ("airmass_err",
float),
("extinction",
float), ("extinction_err",
float), ("zeropoint_ext_corr",
float), ("zeropoint_ext_corr_err",
float), ("n_matches", float)])
if f"provided" in zeropoints:
print(f"\tProvided:")
zeropoint_prov = zeropoints["provided"]
zeropoint_tbl.add_row([
"provided", "N/A", "N/A", zeropoint_prov["zeropoint"],
zeropoint_prov["zeropoint_err"], 0.0, 0.0, extinction,
extinction_err, zeropoint_prov["zeropoint"],
zeropoint_prov["zeropoint_err"], 0
])
print(f"\tScience field:")
for cat_name in zeropoints["science_field"]:
print(f"\t\t{cat_name}")
zeropoint_sci = zeropoints["science_field"][cat_name]
if zeropoint_sci is not None:
zeropoint_corrected = zeropoint_sci[
"zeropoint"] + extinction * airmass_sci
print("\t\t\t", zeropoint_sci["zeropoint"], extinction,
airmass_sci)
zeropoint_corrected_err = zeropoint_sci[
"zeropoint_err"] + error_product(
value=extinction * airmass_sci,
measurements=[extinction, airmass_sci],
errors=[extinction_err, airmass_sci_err])
zp_cat = table.Table.read(zeropoint_sci["matches_cat_path"])
zeropoint_tbl.add_row([
"science_field", epoch[:-2], cat_name,
zeropoint_sci["zeropoint"], zeropoint_sci["zeropoint_err"],
airmass_sci, airmass_sci_err, extinction, extinction_err,
zeropoint_corrected, zeropoint_corrected_err,
len(zp_cat)
])
plt.scatter(zp_cat["mag_cat"], zp_cat["mag"], c='green')
plt.errorbar(zp_cat["mag_cat"],
zp_cat["mag"],
yerr=zp_cat["mag_err"],
linestyle="None",
c='black')
plt.plot(zp_cat["mag_cat"],
zp_cat["mag_cat"] - zeropoint_sci["zeropoint"],
c="violet")
plt.title(f"{fil}: science-field, {cat_name}")
plt.figtext(
0, 0.01,
f'zeropoint - kX: {zeropoint_sci["zeropoint"]} ± {zeropoint_sci["zeropoint_err"]}\n'
f'zeropoint: {zeropoint_corrected} ± {zeropoint_corrected_err}\n'
f'num stars: {len(zp_cat)}')
plt.gca().set_aspect('equal', adjustable='box')
plt.savefig(output_path_final_f +
f"zeropoint_science_{cat_name}.pdf")
plt.show()
plt.close()
print("\tStandard fields:")
for field in zeropoints["standard_field"]:
print(f"\t\t{field}")
for cat_name in zeropoints["standard_field"][field]:
print(f"\t\t\t{cat_name}")
zeropoint_std = zeropoints["standard_field"][field][cat_name]
if zeropoint_std is not None:
zeropoint_corrected = zeropoint_std[
"zeropoint"] + extinction * zeropoint_std["airmass"]
print("\t\t\t\t", zeropoint_std["zeropoint"], extinction,
zeropoint_std["airmass"])
zeropoint_corrected_err = zeropoint_std[
"zeropoint_err"] + error_product(
value=extinction * zeropoint_std["airmass"],
measurements=[
extinction, zeropoint_std["airmass"]
],
errors=[extinction_err, 0.0])
zp_cat = table.Table.read(
zeropoint_std["matches_cat_path"])
zeropoint_tbl.add_row([
"standard_field", field, cat_name,
zeropoint_std["zeropoint"],
zeropoint_std["zeropoint_err"],
zeropoint_std["airmass"], 0.0, extinction,
extinction_err, zeropoint_corrected,
zeropoint_corrected_err,
len(zp_cat)
])
plt.scatter(zp_cat["mag_cat"], zp_cat["mag"], c='green')
plt.errorbar(zp_cat["mag_cat"],
zp_cat["mag"],
yerr=zp_cat["mag_err"],
linestyle="None",
c='black')
plt.plot(zp_cat["mag_cat"],
zp_cat["mag_cat"] - zeropoint_std["zeropoint"],
c="violet")
plt.title(f"{fil}: standard-field {field}, {cat_name}")
plt.gca().set_aspect('equal', adjustable='box')
plt.figtext(
0, 0.01,
f'zeropoint - kX: {zeropoint_std["zeropoint"]} ± {zeropoint_std["zeropoint_err"]}\n'
f'zeropoint: {zeropoint_corrected} ± {zeropoint_corrected_err}\n'
f'num stars: {len(zp_cat)}')
plt.savefig(output_path_final_f +
f"zeropoint_standard_{field}_{cat_name}.pdf")
plt.show()
plt.close()
zeropoint_tbl["selection_index"] = zeropoint_tbl[
"n_matches"] / zeropoint_tbl["zeropoint_ext_corr_err"]
best_arg = np.argmax(zeropoint_tbl["selection_index"])
print("Best zeropoint:")
best_zeropoint = zeropoint_tbl[best_arg]
print(best_zeropoint)
zeropoints = outputs[f + '_zeropoints']
zeropoints['best'] = {
"zeropoint": float(best_zeropoint['zeropoint']),
"zeropoint_err": float(best_zeropoint["zeropoint_err"]),
"airmass": float(best_zeropoint["airmass"]),
"airmass_err": float(best_zeropoint["airmass_err"]),
"type": str(best_zeropoint["type"]),
"catalogue": str(best_zeropoint["cat"])
}
output_dict = {f + '_zeropoints': zeropoints}
p.add_output_values(obj=epoch, instrument='FORS2', params=output_dict)
zeropoint_tbl.write(output_path_final_f + "zeropoints.csv",
format="ascii.csv",
overwrite=True)
def main(data_title, sextractor_path, origin, destination):
print(
"\nExecuting Python script pipeline_fors2/4-divide_by_exp_time.py, with:"
)
print(f"\tepoch {data_title}")
print(f"\tsextractor path {sextractor_path}")
print(f"\torigin directory {origin}")
print(f"\tdestination directory {destination}")
print()
properties = p.object_params_fors2(data_title)
outputs = p.object_output_params(obj=data_title, instrument='FORS2')
data_dir = properties['data_dir']
if sextractor_path is not None:
if not os.path.isdir(sextractor_path):
os.mkdir(sextractor_path)
do_sextractor = True
else:
do_sextractor = False
origin_path = data_dir + origin
destination_path = data_dir + destination
u.mkdir_check(destination_path)
u.mkdir_check(destination_path + "science/")
u.mkdir_check(destination_path + "backgrounds/")
filters = outputs['filters']
for fil in filters:
u.mkdir_check(destination_path + "science/" + fil)
u.mkdir_check(destination_path + "backgrounds/" + fil)
if do_sextractor:
if not os.path.isdir(sextractor_path + fil):
os.mkdir(sextractor_path + fil)
files = os.listdir(origin_path + "science/" + fil + "/")
for file_name in files:
if file_name[-5:] == '.fits':
science_origin = origin_path + "science/" + fil + "/" + file_name
science_destination = destination_path + "science/" + fil + "/" + file_name.replace(
"trim", "norm")
background_origin = origin_path + "backgrounds/" + fil + "/" + file_name.replace(
"SCIENCE_REDUCED", "PHOT_BACKGROUND_SCI")
background_destination = destination_path + "backgrounds/" + fil + "/" + \
file_name.replace("SCIENCE_REDUCED", "PHOT_BACKGROUND_SCI").replace("trim",
"norm")
print(science_origin)
# Divide by exposure time to get an image in counts/second.
ff.divide_by_exp_time(file=science_origin,
output=science_destination)
ff.divide_by_exp_time(file=background_origin,
output=background_destination)
if do_sextractor:
copyfile(
science_destination, sextractor_path + fil + "/" +
file_name.replace("trim", "norm"))
if os.path.isfile(origin_path + data_title + '.log'):
copyfile(origin_path + data_title + '.log',
destination_path + data_title + ".log")
u.write_log(path=destination_path + data_title + ".log",
action=f'Divided by exposure time.')
def main(data_title: 'str', delete_output: bool = True):
print(
"\nExecuting Python script pipeline_fors2/2-sort_after_esoreflex.py, with:"
)
print(f"\tepoch {data_title}")
print()
eso_dir = p.config['esoreflex_output_dir']
if os.path.isdir(eso_dir):
obj_params = p.object_params_fors2(data_title)
data_dir = obj_params['data_dir']
destination = data_dir + "2-sorted/"
date = None
output_values = p.object_output_params(obj=data_title,
instrument='FORS2')
mjd = int(output_values['mjd_obs'])
obj = output_values['object']
print(
f"Looking for data with object '{obj}' and MJD of observation {mjd} inside {eso_dir}"
)
# Look for files with the appropriate object and MJD, as recorded in output_values
# List directories in eso_output_dir; these are dates on which data was reduced using ESOReflex.
eso_dirs = filter(lambda d: os.path.isdir(eso_dir + "/" + d),
os.listdir(eso_dir))
for directory in eso_dirs:
directory += "/"
# List directories within 'reduction date' directories.
# These should represent individual images reduced.
print(f"Searching {eso_dir + directory}")
eso_subdirs = filter(
lambda d: os.path.isdir(eso_dir + directory + d) and "FORS2" in
d, os.listdir(eso_dir + directory))
for subdirectory in eso_subdirs:
subdirectory += "/"
subpath = eso_dir + "/" + directory + subdirectory
print(f"\tSearching {subpath}")
# Get the files within the image directory.
files = filter(lambda d: os.path.isfile(subpath + d),
os.listdir(subpath))
for file_name in files:
# Retrieve the target object name from the fits file.
file_path = subpath + file_name
file = fits.open(file_path)
file_obj = ff.get_object(file)
file_mjd = int(ff.get_header_attribute(file, 'MJD-OBS'))
file_filter = ff.get_filter(file)
# Check the object name and observation date against those of the epoch we're concerned with.
if file_obj == obj and file_mjd == mjd:
# Check which type of file we have.
date = directory[:-1]
if file_name[-28:] == "PHOT_BACKGROUND_SCI_IMG.fits":
file_destination = f"{destination}/backgrounds/"
suffix = "PHOT_BACKGROUND_SCI_IMG.fits"
elif file_name[-25:] == "OBJECT_TABLE_SCI_IMG.fits":
file_destination = f"{destination}/obj_tbls/"
suffix = "OBJECT_TABLE_SCI_IMG.fits"
elif file_name[-24:] == "SCIENCE_REDUCED_IMG.fits":
file_destination = f"{destination}/science/"
suffix = "SCIENCE_REDUCED_IMG.fits"
else:
file_destination = f"{destination}/sources/"
suffix = "SOURCES_SCI_IMG.fits"
# Make this directory, if it doesn't already exist.
u.mkdir_check(file_destination)
# Make a subdirectory by filter.
file_destination += file_filter + "/"
u.mkdir_check(file_destination)
# Title new file.
file_destination += f"{data_title}_{subdirectory[:-1]}_{suffix}"
# Copy file to new location.
print(
f"Copying: {file_path} to \n\t {file_destination}")
file.writeto(file_destination, overwrite=True)
if delete_output and os.path.isfile(file_destination):
os.remove(file_path)
if not os.path.isfile(f"{destination}/{data_title}.log"):
sh.copy(f"{data_dir}0-data_with_raw_calibs/{data_title}.log",
f"{destination}/{data_title}.log")
u.write_log(path=destination + data_title + ".log",
action=f'Data reduced with ESOReflex.',
date=date)
u.write_log(path=destination + data_title + ".log",
action='Files sorted using 2-sort_after_esoreflex.sh')
else:
print(f"ESO output directory '{eso_dir}' not found.")
exit(1)
