def _build_param_dir(cls, instrument_name: str):
path = os.path.join(p.param_dir, "instruments")
u.mkdir_check(path)
u.debug_print(2, "Instrument._build_param_dir(): instrument_name ==", instrument_name)
path = os.path.join(path, instrument_name)
u.mkdir_check(path)
return path
def main(data_title, sextractor_path, origin, destination):
properties = p.object_params_imacs(data_title)
data_dir = properties['data_dir']
if sextractor_path is not None:
if not os.path.isdir(sextractor_path):
os.mkdir(sextractor_path)
do_sextractor = True
ap_diams_sex = p.load_params(f'param/aperture_diameters_fors2')
else:
do_sextractor = False
origin_path = data_dir + origin
destination_path = data_dir + destination
u.mkdir_check(destination_path)
filters = next(os.walk(origin_path))[1]
for fil in filters:
u.mkdir_check(destination_path + fil)
if do_sextractor:
if not os.path.isdir(sextractor_path + fil):
os.mkdir(sextractor_path + fil)
files = os.listdir(origin_path + fil + "/")
for file_name in files:
if file_name[-5:] == '.fits':
science_origin = origin_path + fil + "/" + file_name
science_destination = destination_path + fil + "/" + file_name
print(science_origin)
# Divide by exposure time to get an image in counts/second.
f.divide_by_exp_time(file=science_origin,
output=science_destination)
if do_sextractor:
copyfile(science_origin,
sextractor_path + fil + "/" + file_name)
# Write a sextractor file for photometric testing of the data from the upper chip.
if do_sextractor:
# Write a csv table of file properties to each filter directory.
tbl = f.fits_table(input_path=sextractor_path + fil,
output_path=sextractor_path + fil + "/" + fil +
"_fits_tbl.csv",
science_only=False)
# TODO: Rewrite to use psf-fitting (in FORS2 pipeline as well)
for i, d in enumerate(ap_diams_sex):
f.write_sextractor_script(table=tbl,
output_path=sextractor_path + fil +
"/sextract_aperture_" + str(d) +
".sh",
sex_params=['c', 'PHOT_APERTURES'],
sex_param_values=['im.sex',
str(d)],
cat_name='sextracted_' + str(d),
cats_dir='aperture_' + str(d),
criterion='chip',
value='CHIP1')
if os.path.isfile(origin_path + data_title + '.log'):
copyfile(origin_path + data_title + '.log',
destination_path + data_title + ".log")
u.write_log(path=destination_path + data_title + ".log",
action=f'Astrometry solved using 3-astrometry.py')
def check_for_config():
u.mkdir_check(config_dir)
p = load_params(config_file)
if p is None:
config_text = pkg_resources.resource_string(
__name__, os.path.join("..", f"param",
"config_template.yaml")).decode()
print(type(config_text))
config_text = config_text.replace(
"proj_dir: <some_directory>/craft-optical-followup/",
f"proj_dir: {os.getcwd()}/")
with open(config_file, "w") as cfg:
cfg.write(config_text)
print(f"No config file was detected at {config_file}.")
print(f"A fresh config file has been created at '{config_file}'")
print(
"In this file, please set 'top_data_dir' to a valid path in which to store all "
"data products of this package (This may require a large amount of space.)."
)
print("You may also like to specify an alternate param_dir")
input("\nOnce you have edited this file, press any key to proceed.")
p = load_params(config_file)
else:
for param in p:
p[param] = u.check_trailing_slash(p[param])
save_params(config_file, p)
yaml_to_json(config_file)
return p
def main(comb_path, output_dir, obj, sextractor_path, path_suffix):
print("\nExecuting Python script pipeline_fors2/7-trim_combined.py, with:")
print(f"\tepoch {obj}")
print(f"\toutput directory {output_dir}")
print(f"\tsextractor directory {sextractor_path}")
print()
if sextractor_path is not None:
u.mkdir_check_nested(sextractor_path)
do_sextractor = True
else:
do_sextractor = False
# Build a list of the filter prefixes used.
fils = []
files = list(filter(lambda x: x[-4:] == '.tbl', os.listdir(comb_path)))
for file in files:
if file[0] not in fils:
fils.append(str(file[0]))
for fil in fils:
if do_sextractor:
u.mkdir_check(sextractor_path)
area_file = fil + "_coadded_area.fits"
comb_file = fil + "_coadded.fits"
left, right, bottom, top = f.detect_edges_area(comb_path + area_file)
# Trim a little extra to be safe.
left = left + 5
right = right - 5
top = top - 5
bottom = bottom + 5
f.trim_file(comb_path + comb_file,
left=left,
right=right,
top=top,
bottom=bottom,
new_path=output_dir + "/" + comb_file)
# Keep a trimmed version of the area file, it comes in handy later.
f.trim_file(comb_path + area_file,
left=left,
right=right,
top=top,
bottom=bottom,
new_path=output_dir + "/" + area_file)
if do_sextractor:
copyfile(output_dir + "/" + comb_file, sextractor_path + comb_file)
if path_suffix is None:
path_suffix = output_dir.split("/")[-2]
path_suffix = u.remove_trailing_slash(path_suffix)
p.add_output_path(obj=obj,
instrument='fors2',
key=fil[0] + '_trimmed_image' + path_suffix,
path=output_dir + "/" + comb_file)
def check_data_path(self):
if self.field is not None:
u.debug_print(2, "", self.name)
# print(self.field.data_path, self.name_filesys)
self.data_path = os.path.join(self.field.data_path, "objects", self.name_filesys)
u.mkdir_check(self.data_path)
self.output_file = os.path.join(self.data_path, f"{self.name_filesys}_outputs.yaml")
return True
else:
return False
def main(data_title: str, origin: str, destination: str, redo: bool = False):
properties = p.object_params_imacs(data_title)
path = properties['data_dir']
origin_path = path + origin
astrometry_path = path + destination
u.mkdir_check(astrometry_path)
keys = params.load_params('param/keys')
key = keys['astrometry']
reduced_list = os.listdir(origin_path)
astrometry_list = os.listdir(astrometry_path)
if redo:
to_send = list(filter(lambda f: f[-5:] == '.fits', reduced_list))
else:
to_send = list(
filter(lambda f: f[-5:] == '.fits' and f not in astrometry_list,
reduced_list))
filters = list(filter(lambda f: os.path.isdir(f), os.listdir(origin)))
for f in filters:
reduced_path_filter = origin_path + f + '/'
astrometry_path_filter = astrometry_path + f + '/'
print(f'To send to Astrometry.net from {f}:')
for file in to_send:
print('\t' + file)
for file in to_send:
hdu = fits.open(origin_path + file)
header = hdu[0].header
ra = header['RA-D']
dec = header['DEC-D']
scale_upper = header['SCALE'] + 0.1
scale_lower = header['SCALE'] - 0.1
hdu.close()
print('Sending to Astrometry.net:', file)
os.system(f'python /astrometry-client.py '
f'--apikey {key} '
f'-u {reduced_path_filter}{file} '
f'-w '
f'--newfits {astrometry_path_filter}{file} '
f'--ra {ra} --dec {dec} --radius {1.} '
f'--scale-upper {scale_upper} '
f'--scale-lower {scale_lower} '
f'--private --no_commercial')
if os.path.isfile(origin_path + data_title + '.log'):
shutil.copy(origin_path + data_title + '.log',
astrometry_path + data_title + ".log")
u.write_log(path=astrometry_path + data_title + ".log",
action=f'Astrometry solved using 3-astrometry.py')
def main(ob, path):
print("\nExecuting Python script pipeline_fors2/9-esorex_zeropoint_prep.py, with:")
print(f"\tepoch {ob}")
print(f"\tcalibration data path {path}")
print()
output = object_output_params(obj=ob, instrument='FORS2')
filters = output['filters']
for f in filters:
mkdir_check(path + '/' + f)
def main(field, destination, epoch, force_subtract_better_seeing, subtraction_type, template_instrument):
if destination[-1] != '/':
destination = destination + '/'
params = p.object_params_frb(field)
destination_path = f'{params["data_dir"]}subtraction/{destination}/'
u.mkdir_check(destination_path)
params = p.object_params_frb(field)
template_epoch = params['template_epoch_' + template_instrument.lower()]
comparison_title = f'{field}_{epoch}'
template_title = f'{field}_{template_epoch}'
filters = params['filters']
for f in filters:
f_0 = f[0]
destination_path_filter = f'{destination_path}{f}/'
u.mkdir_check(destination_path_filter)
template_params = p.load_params(destination_path_filter + f'{template_title}_template_output_values.yaml')
comparison_params = p.load_params(destination_path_filter + f'{comparison_title}_comparison_output_values.yaml')
if f_0 + '_fwhm_arcsec' in template_params:
fwhm_template = template_params[f_0 + '_fwhm_arcsec']
elif f_0.lower() + '_fwhm_arcsec' in template_params:
fwhm_template = template_params[f_0.lower() + '_fwhm_arcsec']
else:
raise ValueError(f_0 + '_fwhm_arcsec or ' + f_0.lower() + '_fwhm_arcsec not found for template image.')
if f_0 + '_fwhm_arcsec' in comparison_params:
fwhm_comparison = comparison_params[f_0 + '_fwhm_arcsec']
elif f_0.lower() + '_fwhm_arcsec' in comparison_params:
fwhm_comparison = comparison_params[f_0.lower() + '_fwhm_arcsec']
else:
raise ValueError(f_0 + '_fwhm_arcsec or ' + f_0.lower() + '_fwhm_arcsec not found for comparison image.')
template = destination_path_filter + f'{template_title}_template_aligned.fits'
comparison = destination_path_filter + f'{comparison_title}_comparison_aligned.fits'
_, difference = ph.subtract(template_origin=template,
comparison_origin=comparison,
output=destination_path_filter,
template_fwhm=fwhm_template,
comparison_fwhm=fwhm_comparison,
force_subtract_better_seeing=force_subtract_better_seeing,
comparison_title=comparison_title,
template_title=template_title,
field=field, comparison_epoch=epoch, template_epoch=template_epoch)
def main(data_title: str):
properties = p.object_params_sdss(data_title)
path = properties['data_dir']
data_path = path + '/0-data/'
u.mkdir_check(data_path)
os.listdir(path)
print(path)
for file in filter(lambda fil: fil[-5:] == '.fits', os.listdir(path)):
f = file[-6]
shutil.copy(path + file, data_path + str(f) + '_cutout.fits')
def main(data_title: str):
properties = p.object_params_des(data_title)
path = properties['data_dir']
data_path = path + '/0-data/'
u.mkdir_check(data_path)
os.listdir(path)
print(path)
for file in filter(lambda fil: fil[-5:] == '.fits', os.listdir(path)):
f = file[-6]
shutil.copy(path + file, data_path + str(f) + '_cutout.fits')
p.add_output_path(obj=data_title, instrument='DES', key=f + '_subtraction_image', path=data_path + str(f) + '_cutout.fits')
def test_mkdir_check():
path = os.path.join(test_file_path, "path_test")
u.rm_check(path)
u.mkdir_check(path)
assert os.path.isdir(path)
u.rmtree_check(path)
paths = [
os.path.join(test_file_path, "test_path_1"),
os.path.join(test_file_path, "test_path_2"),
os.path.join(test_file_path, "test_path_3")
]
u.mkdir_check_args(*paths)
for path in paths:
assert os.path.isdir(path)
u.rmtree_check(path)
def source_extractor(
image_path: str,
output_dir: str = None,
configuration_file: str = None,
parameters_file: str = None,
catalog_name: str = None,
copy_params: bool = True,
template_image_path: str = None,
**configs):
"""
:param configs: Any source-extractor (sextractor) parameter, normally read via the config file but that can be
overridden by passing to the shell command, can be given here.
"""
if "gain" in configs:
configs["gain"] = u.check_quantity(number=configs["gain"], unit=gain_unit).to(gain_unit).value
old_dir = os.getcwd()
if output_dir is None:
output_dir = os.getcwd()
else:
u.mkdir_check(output_dir)
os.chdir(output_dir)
if copy_params:
os.system(f"cp {os.path.join(p.path_to_config_psfex(), '*')} .")
sys_str = "source-extractor "
if template_image_path is not None:
sys_str += f"{template_image_path},"
sys_str += image_path + " "
if configuration_file is not None:
sys_str += f" -c {configuration_file}"
if catalog_name is None:
image_name = os.path.split(image_path)[-1]
catalog_name = f"{image_name}.cat"
sys_str += f" -CATALOG_NAME {catalog_name}"
if parameters_file is not None:
sys_str += f" -PARAMETERS_NAME {parameters_file}"
for param in configs:
sys_str += f" -{param.upper()} {configs[param]}"
u.system_command_verbose(sys_str, error_on_exit_code=True)
catalog_path = os.path.join(output_dir, catalog_name)
os.chdir(old_dir)
return catalog_path
def main(obj, test, magnitude):
print('Got here.')
properties = p.object_params_fors2(obj)
synth_path = properties['data_dir'] + 'synthetic/'
u.mkdir_check(synth_path)
synth_path = synth_path + 'frb_position/'
u.mkdir_check(synth_path)
now = time.Time.now()
now.format = 'isot'
test = str(now) + '_' + test
test_path = synth_path + test + '/'
print(properties['filters'])
ph.insert_synthetic_at_frb(obj=obj,
test_path=test_path,
filters=properties['filters'],
magnitudes=[magnitude, magnitude],
add_path=True)
def generate_astrometry_indices(cat_name: str,
cat: Union[str, table.Table],
output_file_prefix: str,
unique_id_prefix: int,
index_output_dir: str,
fits_cat_output: str = None,
p_lower: int = 0,
p_upper: int = 2):
u.mkdir_check(index_output_dir)
cat_name = cat_name.lower()
if fits_cat_output is None and isinstance(cat, str):
if cat.endswith(".csv"):
fits_cat_output = cat.replace(".csv", ".fits")
else:
fits_cat_output = cat + ".fits"
elif fits_cat_output is None:
raise ValueError(
"fits_cat_output must be provided if cat is given as a Table instead of a path."
)
cat = u.path_or_table(cat, fmt="ascii.csv")
cols = cat_columns(cat=cat_name, f="rank")
cat.write(fits_cat_output, format='fits', overwrite=True)
unique_id_prefix = str(unique_id_prefix)
for scale in range(p_lower, p_upper + 1):
unique_id = unique_id_prefix + str(scale).replace("-", "0")
unique_id = int(unique_id)
output_file_name_scale = f"{output_file_prefix}_{scale}"
try:
astrometry_net.build_astrometry_index(
input_fits_catalog=fits_cat_output,
unique_id=unique_id,
output_index=os.path.join(index_output_dir,
output_file_name_scale),
scale_number=scale,
sort_column=cols["mag_auto"],
scan_through_catalog=True)
except SystemError:
print(f"Building index for scale {scale} failed.")
def main(data_title, sextractor_path, origin, destination):
properties = p.object_params_xshooter(data_title)
data_dir = properties['data_dir']
if sextractor_path is not None:
if not os.path.isdir(sextractor_path):
os.mkdir(sextractor_path)
do_sextractor = True
else:
do_sextractor = False
origin_path = data_dir + origin
destination_path = data_dir + destination
u.mkdir_check(destination_path)
filters = next(os.walk(origin_path))[1]
for fil in filters:
u.mkdir_check(destination_path + fil)
if do_sextractor:
if not os.path.isdir(sextractor_path + fil):
os.mkdir(sextractor_path + fil)
files = os.listdir(origin_path + fil + "/")
for file_name in files:
if file_name[-5:] == '.fits':
science_origin = origin_path + fil + "/" + file_name
science_destination = destination_path + fil + "/" + file_name
print(science_origin)
# Divide by exposure time to get an image in counts/second.
f.divide_by_exp_time(file=science_origin,
output=science_destination)
if do_sextractor:
copyfile(science_origin,
sextractor_path + fil + "/" + file_name)
if os.path.isfile(origin_path + data_title + '.log'):
copyfile(origin_path + data_title + '.log',
destination_path + data_title + ".log")
u.write_log(path=destination_path + data_title + ".log",
action=f'Divided by exposure time.')
def sort_by_filter(path: 'str'):
if path[-1] != "/":
path = path + "/"
files = os.listdir(path)
filters = {}
# Gather the information we need.
for file in files:
if file[-5:] == ".fits":
filter = get_filter(path + file)
if type(filter) is list:
filter = filter[0]
if filter not in filters:
filters[filter] = []
filters[filter].append(file)
# Do sort
for filter in filters:
# Create a directory for each filter
filter_path = path + filter + "/"
u.mkdir_check(filter_path)
# Move all the files with that filter
for file in filters[filter]:
if os.path.isfile(filter_path + file):
os.remove(filter_path + file)
sh.move(path + file, filter_path)
def main(obj, test, curves_path):
properties = p.object_params_fors2(obj)
if curves_path[-1] != '/':
curves_path += '/'
mag_table_file = filter(lambda file: file[-4:] == '.csv',
os.listdir(curves_path)).__next__()
mag_table = table.Table.read(curves_path + mag_table_file)
filters = mag_table.colnames.copy()
filters.remove('model')
synth_path = properties['data_dir'] + 'synthetic/'
u.mkdir_check(synth_path)
synth_path = synth_path + 'frb_position/'
u.mkdir_check(synth_path)
now = time.Time.now()
now.format = 'isot'
synth_path += f'sn_models_{now}/'
u.mkdir_check(synth_path)
for row in mag_table:
model = row['model']
magnitudes = []
for f in filters:
magnitudes.append(row[f])
test_spec = test + '_' + model
test_path = synth_path + test_spec + '/'
ph.insert_synthetic_at_frb(obj=obj,
test_path=test_path,
filters=filters,
magnitudes=magnitudes,
add_path=False)
for f in filters:
p.add_output_path(obj=obj,
key=f[0] + '_subtraction_image_synth_frb_sn_models',
path=synth_path)
def main(data_title, origin, destination):
properties = p.object_params_xshooter(data_title)
path = properties['data_dir']
destination = path + destination
u.mkdir_check(destination)
origin = path + origin
u.mkdir_check(origin)
dirs = next(os.walk(origin))[1]
left = 27
right = 537
top = 526
bottom = 15
for fil in dirs:
u.mkdir_check(destination + fil)
print('HERE:')
print(origin + fil)
files = os.listdir(origin + fil)
files.sort()
for i, file in enumerate(files):
# Split the files into upper CCD and lower CCD, with even-numbered being upper and odd-numbered being lower
new_path = destination + fil + "/" + file.replace(
".fits", "_trim.fits")
# Add GAIN and SATURATE keywords to headers.
path = origin + fil + "/" + file
f.trim_file(path,
left=left,
right=right,
top=top,
bottom=bottom,
new_path=new_path)
copyfile(origin + data_title + ".log", destination + data_title + ".log")
u.write_log(path=destination + data_title + ".log",
action='Edges trimmed using 4-trim.py\n')
def main(obj, test, mag_min, mag_max, increment, instrument):
properties = p.object_params_instrument(obj, instrument=instrument)
output = p.object_output_params(obj=obj, instrument=instrument)
paths = p.object_output_paths(obj=obj, instrument=instrument)
synth_path = properties['data_dir'] + 'synthetic/'
u.mkdir_check(synth_path)
synth_path = synth_path + 'frb_position/'
u.mkdir_check(synth_path)
now = time.Time.now()
now.format = 'isot'
synth_path += f'range_{now}/'
u.mkdir_check(synth_path)
filters = output['filters']
for magnitude in np.arange(mag_min, mag_max, increment):
magnitudes = []
for i in range(len(filters)):
magnitudes.append(magnitude)
test_spec = test + '_' + str(u.round_to_sig_fig(magnitude, 4))
test_path = synth_path + test_spec + '/'
ph.insert_synthetic_at_frb(obj=properties,
test_path=test_path,
filters=filters,
magnitudes=magnitudes,
add_path=False,
psf=True,
output_properties=output,
instrument=instrument,
paths=paths)
p.add_output_path(obj=obj,
key='subtraction_image_synth_frb_range',
path=synth_path,
instrument=instrument)
def standard_script(
input_directory: str,
output_directory: str,
output_file_name: str = None,
ignore_differences: bool = False,
coadd_types: Union[List[str], str] = 'median',
# unit="electron / second",
do_inject_header: bool = True,
**kwargs):
"""
Does a standard median coaddition of fits files in input_directory.
Adapted from an example bash script found at http://montage.ipac.caltech.edu/docs/first_mosaic_tutorial.html
:param input_directory: path to directory containing input images.
:param output_directory: path to directory to write data products to; will be created if it doesn't exist.
:param output_file_name: Name of final coadded image file.
:param ignore_differences: If False, checks if input images have compatible exposure times, filter, etc. and raises ValueError if not.
:return:
"""
u.mkdir_check(output_directory)
old_dir = os.getcwd()
proj_dir = os.path.join(output_directory, "projdir")
diff_dir = os.path.join(output_directory, "diffdir")
corr_dir = os.path.join(output_directory, "corrdir")
u.mkdir_check(proj_dir, diff_dir, corr_dir)
os.chdir(output_directory)
print("Creating directories to hold processed images.")
proj_dir = "projdir"
diff_dir = "diffdir"
corr_dir = "corrdir"
if not ignore_differences:
print("Checking input images...")
check_input_images(input_directory=input_directory, **kwargs)
print("Creating metadata tables of the input images.")
table_path = "images.tbl"
image_table(input_directory=input_directory, output_path=table_path)
print("Creating FITS headers describing the footprint of the mosaic.")
header_path = "template.hdr"
make_header(table_path=table_path, output_path=header_path)
print("Reprojecting input images.")
stats_table_path = "stats.tbl"
project_execute(input_directory=input_directory,
table_path=table_path,
header_path=header_path,
proj_dir=proj_dir,
stats_table_path=stats_table_path)
print("Creating metadata table of the reprojected images.")
reprojected_table_path = "proj.tbl"
image_table(input_directory=proj_dir, output_path=reprojected_table_path)
print("Analyzing the overlaps between images.")
difference_table_path = "diffs.tbl"
fit_table_path = "fits.tbl"
overlaps(table_path=reprojected_table_path,
difference_table_path=difference_table_path)
difference_execute(input_directory=proj_dir,
difference_table_path=difference_table_path,
header_path=header_path,
diff_dir=diff_dir)
fit_execute(difference_table_path=difference_table_path,
fit_table_path=fit_table_path,
diff_dir=diff_dir)
print(
"Performing background modeling and compute corrections for each image."
)
corrections_table_path = "corrections.tbl"
background_model(table_path=reprojected_table_path,
fit_table_path=fit_table_path,
correction_table_path=corrections_table_path)
print("Applying corrections to each image")
background_execute(input_directory=proj_dir,
table_path=reprojected_table_path,
correction_table_path=corrections_table_path,
corr_dir=corr_dir)
if isinstance(coadd_types, str):
coadd_types = [coadd_types]
file_paths = []
for i, coadd_type in enumerate(coadd_types):
print(f"Coadding the images with {coadd_type}.")
if output_file_name is None:
output_file_name = "coadded.fits"
output_file_name_coadd = output_file_name.replace(
".fits", f"_{coadd_type}.fits")
add(input_directory=corr_dir,
coadd_type=coadd_type,
table_path=reprojected_table_path,
header_path=header_path,
output_path=output_file_name_coadd)
if do_inject_header:
inject_header(file_path=output_file_name_coadd,
input_directory=input_directory,
coadd_type=coadd_type)
file_paths.append(
os.path.join(output_directory, output_file_name_coadd))
os.chdir(old_dir)
u.debug_print(1, "montage.standard_script():", file_paths)
return file_paths
def main(obj, test, n, filter_dist, instrument, limit):
properties = p.object_params_instrument(obj, instrument=instrument)
burst_properties = p.object_params_frb(obj=obj[:-2])
output = p.object_output_params(obj=obj, instrument=instrument)
paths = p.object_output_paths(obj=obj, instrument=instrument)
z = burst_properties['z']
mjd_burst = burst_properties['mjd_burst']
ebv_mw = burst_properties['dust_ebv']
mjd_obs = properties['mjd']
synth_path = properties['data_dir'] + 'synthetic/'
u.mkdir_check(synth_path)
synth_path = synth_path + 'sn_random/'
u.mkdir_check(synth_path)
epoch = mjd_obs - mjd_burst
f_0 = filter_dist[0]
hg_ra = burst_properties['hg_ra']
hg_dec = burst_properties['hg_dec']
burst_ra = burst_properties['burst_ra']
burst_dec = burst_properties['burst_dec']
image_path = paths[f_0 + '_' + properties['subtraction_image']]
image = fits.open(image_path)
wcs_info = wcs.WCS(image[0].header)
burst_x, burst_y = wcs_info.all_world2pix(burst_ra, burst_dec, 0)
now = time.Time.now()
now.format = 'isot'
synth_path += test + '_' + str(now) + '/'
u.mkdir_check(synth_path)
filters = output['filters']
psf_models = []
for i, f in enumerate(filters):
sn.register_filter(f=f, instrument=instrument)
psf_model = fits.open(paths[f[0] + '_psf_model'])
psf_models.append(psf_model)
for i in range(n):
test_spec = test + '_' + str(i)
test_path = synth_path + test_spec + '/'
magnitudes = []
mags_filters, model, x, y, tbl = sn.random_light_curves_type_ia(
filters=filters,
image=image,
hg_ra=hg_ra,
hg_dec=hg_dec,
z=z,
ebv_mw=ebv_mw,
output_path=test_path,
output_title=test,
limit=limit,
x=burst_x,
y=burst_y,
ra=burst_ra,
dec=burst_dec)
days = mags_filters['days']
for f in filters:
magnitude = sn.magnitude_at_epoch(epoch=epoch,
days=days,
mags=mags_filters[f])
print(f, 'mag:', magnitude)
magnitudes.append(magnitude)
ph.insert_synthetic(x=float(x),
y=float(y),
obj=properties,
test_path=test_path,
filters=filters,
magnitudes=magnitudes,
suffix='sn_random_random_ia',
extra_values=tbl,
paths=paths,
output_properties=output,
psf_models=psf_models,
instrument=instrument)
p.add_output_path(obj=obj,
key='subtraction_image_synth_sn_random_ia',
path=synth_path,
instrument=instrument)
def main(epoch, origin, destination):
print("\nExecuting Python script pipeline_fors2/4.1-insert_test_synth.py, with:")
print(f"\tepoch {epoch}")
print(f"\torigin directory {origin}")
print(f"\tdestination directory {destination}")
print()
epoch_params = p.object_params_fors2(obj=epoch)
outputs = p.object_output_params(obj=epoch, instrument='FORS2')
data_dir = epoch_params['data_dir']
insert = epoch_params['test_synths']
origin_path = data_dir + "analysis/sextractor/" + origin
destination_path = data_dir + destination
u.mkdir_check(destination_path)
u.mkdir_check(destination_path + "science/")
u.mkdir_check(destination_path + "backgrounds/")
filters = outputs['filters']
for fil in filters:
f = fil[0]
path_fil_output = destination_path + "science/" + fil + "/"
path_fil_input = origin_path + fil + "/"
u.mkdir_check(path_fil_output)
u.mkdir_check(destination_path + "backgrounds/" + fil)
zeropoint, _, airmass, _, extinction, _ = ph.select_zeropoint(obj=epoch,
filt=fil,
instrument='fors2',
outputs=outputs)
print(path_fil_input)
# print(os.listdir(path_fil_input))
for fits_file in filter(lambda f: f.endswith("_norm.fits"), os.listdir(path_fil_input)):
print(fits_file)
path_fits_file_input = path_fil_input + fits_file
path_fits_file_output = path_fil_output + fits_file
path_psf_model = path_fits_file_input.replace(".fits", "_psfex.psf")
try:
ph.insert_point_sources_to_file(file=path_fits_file_input,
x=array(insert["ra"]),
y=array(insert["dec"]),
mag=insert[f"{f}_mag"],
output=path_fits_file_output,
zeropoint=zeropoint,
extinction=extinction,
airmass=airmass,
world_coordinates=True,
psf_model=path_psf_model
)
except ValueError:
ph.insert_point_sources_to_file(file=path_fits_file_input,
x=array(insert["ra"]),
y=array(insert["dec"]),
mag=insert[f"{f}_mag"],
output=path_fits_file_output,
zeropoint=zeropoint,
extinction=extinction,
airmass=airmass,
world_coordinates=True,
fwhm=fits.open(path_psf_model)[1].header['PSF_FWHM']
)
if os.path.isfile(origin_path + epoch + '.log'):
copyfile(origin_path + epoch + '.log', destination_path + epoch + ".log")
u.write_log(path=destination_path + epoch + ".log", action=f'Divided by exposure time.')
def main(field, destination, epoch, instrument, template_instrument,
comparison_type):
if destination[-1] != '/':
destination = destination + '/'
p.refresh_params_frbs()
types = ['normal', 'synth_random', 'synth_frb']
if comparison_type not in types:
raise ValueError(comparison_type +
' is not a valid synthetic argument; choose from ' +
str(types))
if comparison_type == 'normal':
comparison_type = ''
else:
comparison_type = '_' + comparison_type
params = p.object_params_frb(field)
u.mkdir_check(f'{params["data_dir"]}subtraction/')
destination_path = f'{params["data_dir"]}subtraction/{destination}/'
u.mkdir_check(destination_path)
comparison_title = f'{field}_{epoch}'
comparison_paths = p.object_output_paths(obj=comparison_title,
instrument=instrument)
comparison_params = p.object_params_instrument(obj=comparison_title,
instrument=instrument)
params = p.object_params_frb(field)
template_epoch = params['template_epoch_' + template_instrument.lower()]
template_title = f'{field}_{template_epoch}'
template_paths = p.object_output_paths(obj=template_title,
instrument=template_instrument)
template_outputs = p.object_output_params(obj=template_title,
instrument=template_instrument)
template_params = p.object_params_instrument(
obj=template_title, instrument=template_instrument)
filters = params['filters']
for f in filters:
values = {}
f_0 = f[0]
destination_path_filter = f'{destination_path}{f}/'
u.mkdir_check(destination_path_filter)
# COMPARISON IMAGE:
comparison_image_name = comparison_params[
'subtraction_image'] + comparison_type
# Get path to comparison image from parameter .yaml file
if f'{f_0}_{comparison_image_name}' in comparison_paths:
comparison_origin = comparison_paths[
f'{f_0}_{comparison_image_name}']
elif f'{f_0.lower()}_{comparison_image_name}' in comparison_paths:
comparison_origin = comparison_paths[
f'{f_0.lower()}_{comparison_image_name}']
else:
raise ValueError(
f'{f_0.lower()}_{comparison_image_name} not found in {comparison_title} paths'
)
comparison_destination = f'{comparison_title}_comparison.fits'
if comparison_type != '':
shutil.copyfile(
comparison_origin.replace('.fits',
'.csv'), destination_path_filter +
comparison_destination.replace('.fits', '.csv'))
print('Copying comparison image')
print('From:')
print('\t', comparison_origin)
print('To:')
print(f'\t {destination_path}{f}/{comparison_destination}')
shutil.copy(
comparison_params['data_dir'] + 'output_values.yaml',
f'{destination_path}{f}/{comparison_title}_comparison_output_values.yaml'
)
shutil.copy(
comparison_params['data_dir'] + 'output_values.json',
f'{destination_path}{f}/{comparison_title}_comparison_output_values.json'
)
shutil.copy(comparison_origin,
f'{destination_path}{f}/{comparison_destination}')
values['comparison_file'] = comparison_origin
# TEMPLATE IMAGE
if template_instrument != 'FORS2' and template_instrument != 'XSHOOTER':
f_0 = f_0.lower()
template_image_name = template_params[
'subtraction_image'] + comparison_type
if f'{f_0}_{template_image_name}' in template_paths:
template_origin = template_paths[f'{f_0}_{template_image_name}']
elif f'{f_0.lower()}_{template_image_name}' in template_paths:
template_origin = template_paths[
f'{f_0.lower()}_{template_image_name}']
else:
raise ValueError(
f'{f_0.lower()}_{template_image_name} not found in {template_title} paths'
)
fwhm_template = template_outputs[f_0 + '_fwhm_pix']
template_destination = f'{template_title}_template.fits'
print('Copying template')
print('From:')
print('\t', template_origin)
print('To:')
print(f'\t {destination_path}{f}/{template_destination}')
shutil.copy(
template_params['data_dir'] + 'output_values.yaml',
f'{destination_path}{f}/{template_title}_template_output_values.yaml'
)
shutil.copy(
template_params['data_dir'] + 'output_values.json',
f'{destination_path}{f}/{template_title}_template_output_values.json'
)
shutil.copy(template_origin,
f'{destination_path}{f}/{template_destination}')
values['template_file'] = comparison_origin
p.add_params(f'{destination_path}{f}/output_values.yaml', values)
def light_curves_salt2(z: float, filters: list, peak: float = None, days: Union[tuple, list, np.ndarray] = (0, 85),
show: bool = True, rise_time: float = None, ebv_mw: float = 0, ebv_host: float = 0.,
x1: float = None, output_path: str = None, output_title: str = None, day_markers: list = None,
fil_peak='bessellb', r_v: float = 2.3):
""" Produces light curves, in the provided filters, for a Type Ia supernova using the SALT2 models as implemented
in sncosmo.
:param z: Redshift of source.
:param filters: Filters to obtain light curves in. Must be in the sncosmo Registry.
:param peak: Peak (ie lowest) absolute magnitude, in the filter given by fil_peak, to calibrate curves to.
:param days: Either an array of times (in days) to calculate the light curves over, or a tuple describing the range
of days, ie (first_day, last_day)
:param show: Show plot onscreen?
:param rise_time: Rest-frame time, from beginning of SN to peak magnitude, in days.
:param ebv_mw: Reddening parameter E(B-V), using S&F11 law, for the Milky Way along the SN's line-of-sight.
:param ebv_host: Reddening parameter E(B-V), using S&F11 law, for the host galaxy.
:param x1: SALT2 light curve stretch parameter. See SALT2 documentation for further information.
:param output_path: Path to which to save output. If None, does not save.
:param output_title: Title to give output plot and table.
:param day_markers: List of times (in days) to mark on plot, eg observation dates.
:param fil_peak: Filter in which to set the peak absolute magnitude; usually reported in B or V.
:return: mag_table, model
mag_table: an astropy.table.Table with the times, in days, and the magnitudes in each filter.
model: the entire sncosmo model instance.
"""
if output_path is not None and output_path[-1] != '/':
output_path += '/'
u.mkdir_check(output_path)
# Find time at which model begins and time of peak.
t_first, t_peak = find_model_times(source='salt2-extended', z=z, fil=fil_peak, show=False)
# If both are None, the model is invalid over these wavelengths
if t_first is t_peak is None:
return None, None
# Set up model in sncosmo.
dust_mw = sncosmo.F99Dust()
dust_host = sncosmo.CCM89Dust()
model = sncosmo.Model(source='salt2-extended', effects=[dust_mw, dust_host],
effect_names=['mw', 'host'],
effect_frames=['obs', 'rest'])
model.set(x1=x1)
# If a rise time is not given, allow the model to determine this itself.
if rise_time is None:
if t_peak <= 0:
model.set(z=z, t0=-t_first, mwebv=ebv_mw, hostebv=ebv_host, hostr_v=r_v)
else:
model.set(z=z, t0=0, mwebv=ebv_mw, hostebv=ebv_host, hostr_v=r_v)
elif rise_time == -1:
model.set(z=z, t0=t_first, mwebv=ebv_mw, hostebv=ebv_host, hostr_v=r_v)
else:
# Correct rise time to observer frame
rise_time_obs = rise_time * (1 + z)
model.set(z=z, t0=rise_time_obs - t_peak, mwebv=ebv_mw, hostebv=ebv_host, hostr_v=r_v)
print(model)
# Set peak absolute magnitude of model.
if peak is not None:
model.set_source_peakabsmag(peak, fil_peak, 'ab')
if type(days) is tuple:
# Set up array of times.
days = np.arange(days[0], days[1] + 1, 0.1)
mags_filters = table.Table()
mags_filters['days'] = days
maxes = []
t_peaks = []
peaks = []
for f in filters:
# Get light curve.
mags = model.bandmag(f, 'ab', days)
# If the light curve is mostly flat, the model has probably broken down.
if np.sum(mags == mags[0]) < 0.9 * len(mags):
# If this is False, the entire light curve must be nan, and we'll get nothing useful out of it.
if not np.isnan(np.nanmax(mags)):
# Collect peak (lowest) magnitudes, peak times, and maximum (faintest) magnitudes for each filter.
maxes.append(np.nanmax(mags[mags != np.inf]))
peaks.append(np.nanmin(mags[mags != np.inf]))
t_peaks.append(days[np.nanargmin(mags)])
# Write light curve to table.
mags_filters[f] = mags
if output_path is not None or show:
# Plot curve.
plt.plot(days, mags, label=f)
# If we have no maxima, the model has broken down.
if len(maxes) > 0:
# Collect this for plotting purposes.
max_mag = np.nanmax(maxes)
min_mag = np.nanmin(peaks)
else:
return None, None
# If an output_path directory is not given and 'show' is not True, there's no point doing the plot.
if output_path is not None or show:
for i, t_peak in enumerate(t_peaks):
# Plot blue lines marking the peak of each filter light curve.
plt.plot([t_peak, t_peak], [max_mag + 1, min_mag - 1], c='blue')
if day_markers is not None:
for other_day in day_markers:
# Plot red lines marking the observation dates.
plt.plot([other_day, other_day], [max_mag + 1, min_mag - 1], c='red')
plt.xlabel('Time (days)')
plt.ylabel('Magnitude')
plt.ylim(max_mag + 1, min_mag - 1)
plt.legend()
if output_path is not None:
# Save figure.
plt.savefig(output_path + output_title + '.png')
# Save table to csv.
mags_filters.write(output_path + output_title + '.csv', format='ascii.csv')
if show:
# Show figure onscreen.
plt.show()
plt.close()
return mags_filters, model
def main(data_title: 'str'):
print("\nExecuting Python script pipeline_fors2/1-initial.py, with:")
print(f"\tepoch {data_title}")
print()
epoch_params = p.object_params_fors2(obj=data_title)
data_dir = epoch_params['data_dir']
output_dir = data_dir + "/0-data_with_raw_calibs/"
# Write tables of fits files to main directory; firstly, science images only:
table = fits_table(input_path=output_dir,
output_path=data_dir + data_title + "_fits_table_science.csv",
science_only=True)
# Then including all calibration files
table_full = fits_table(input_path=output_dir,
output_path=data_dir + data_title + "_fits_table_all.csv",
science_only=False)
fits_table_all(input_path=output_dir,
output_path=data_dir + data_title + "_fits_table_detailled.csv",
science_only=False)
# Clear output files for fresh start.
u.rm_check(data_dir + '/output_values.yaml')
u.rm_check(data_dir + '/output_values.json')
# Collect list of filters used:
filters = []
columns = []
for j in [1, 2, 3, 4, 5]:
column = 'filter' + str(j)
for name in table[column]:
if name != 'free':
if name not in filters:
filters.append(name)
columns.append(column)
# Collect pointings of standard-star observations.
std_ras = []
std_decs = []
std_pointings = []
# TODO: This is a horrible way to do this. Take the time to find a better one.
for ra in table_full[table_full['object'] == 'STD']['ref_ra']:
if ra not in std_ras:
std_ras.append(ra)
for dec in table_full[table_full['object'] == 'STD']['ref_dec']:
if dec not in std_decs:
std_decs.append(dec)
for i, ra in enumerate(std_ras):
std_pointings.append(f'RA{ra}_DEC{std_decs[i]}')
print(std_ras)
print(std_decs)
print(std_pointings)
# Collect and save some stats on those filters:
param_dict = {}
exp_times = []
ns_exposures = []
param_dict['filters'] = filters
param_dict['object'] = table['object'][0]
param_dict['obs_name'] = table['obs_name'][0]
mjd = param_dict['mjd_obs'] = float(table['mjd_obs'][0])
for i, f in enumerate(filters):
f_0 = f[0]
exp_time = table['exp_time'][table[columns[i]] == f]
exp_times.append(exp_time)
airmass_col = table['airmass'][table[columns[i]] == f]
n_frames = sum(table[columns[i]] == f)
n_exposures = n_frames / 2
ns_exposures.append(n_exposures)
airmass = float(np.nanmean(airmass_col))
param_dict[f_0 + '_exp_time_mean'] = float(np.nanmean(exp_time))
param_dict[f_0 + '_exp_time_err'] = float(2 * np.nanstd(exp_time))
param_dict[f_0 + '_airmass_mean'] = airmass
param_dict[f_0 + '_airmass_err'] = float(
max(np.nanmax(airmass_col) - airmass, airmass - np.nanmin(airmass_col)))
param_dict[f_0 + '_n_frames'] = float(n_frames)
param_dict[f_0 + '_n_exposures'] = float(n_exposures)
param_dict[f_0 + '_mjd_obs'] = float(np.nanmean(table['mjd_obs'][table[columns[i]] == f]))
std_filter_dir = f'{data_dir}calibration/std_star/{f}/'
u.mkdir_check(std_filter_dir)
print(f'Copying {f} calibration data to std_star folder...')
# Sort the STD files by filter, and within that by pointing.
for j, ra in enumerate(std_ras):
at_pointing = False
pointing = std_pointings[j]
pointing_dir = std_filter_dir + pointing + '/'
for file in \
table_full[
(table_full['object'] == 'STD') &
(table_full['ref_ra'] == ra) &
(table_full[columns[i]] == f)]['identifier']:
at_pointing = True
u.mkdir_check(pointing_dir)
shutil.copyfile(output_dir + file, pointing_dir + file)
if at_pointing:
for file in table_full[table_full['object'] == 'BIAS']['identifier']:
shutil.copyfile(output_dir + file, pointing_dir + file)
for file in table_full[(table_full['object'] == 'FLAT,SKY') & (table_full[columns[i]] == f)][
'identifier']:
shutil.copyfile(output_dir + file, pointing_dir + file)
p.add_output_values(obj=data_title, params=param_dict)
if "new_epoch" in data_dir:
mjd = f"MJD{int(float(mjd))}"
new_data_dir = data_dir.replace("new_epoch", mjd)
p.add_epoch_param(obj=data_title, params={"data_dir": new_data_dir})
def main(data_dir, data_title, origin, destination, all_synths):
print("\nExecuting Python script pipeline_fors2/5-background_subtract.py, with:")
print(f"\tepoch {data_title}")
print(f"\torigin directory {origin}")
print(f"\tdestination directory {destination}")
print()
methods = ["ESO backgrounds only", "SExtractor backgrounds only", "polynomial fit", "Gaussian fit", "median value"]
if all_synths:
frame = 56
method = "polynomial fit"
degree = 5
do_mask = True
local = True
global_sub = False
trim_image = False
recorrect_subbed = True
eso_back = False
else:
frame = 200
# frame_arcsec = 30 * units.arcsec
# frame_deg = frame_arcsec.to(units.deg)
eso_back = False
_, method = u.select_option(message="Please select the background subtraction method.", options=methods,
default="polynomial fit")
degree = None
if method == "polynomial fit":
degree = u.user_input(message=f"Please enter the degree of {method} to use:", typ=int, default=3)
elif method == "ESO backgrounds only":
eso_back = True
do_mask = False
if method not in ["ESO backgrounds only", "SExtractor backgrounds only", "median value"]:
do_mask = u.select_yn(message="Mask sources using SExtractor catalogue?", default=True)
if method in ["polynomial fit", "Gaussian fit"]:
local = u.select_yn(message="Use a local fit?", default=True)
else:
local = False
global_sub = False
trim_image = False
recorrect_subbed = False
if local:
global_sub = u.select_yn(message="Subtract local fit from entire image?", default="n")
if not global_sub:
trim_image = u.select_yn(message="Trim images to subtracted region?", default="y")
recorrect_subbed = u.select_yn(message="Re-normalise background of subtracted region?", default="y")
# if not eso_back and method != "SExtractor backgrounds only":
# eso_back = u.select_yn(message="Subtract ESO Reflex fitted backgrounds first?", default=False)
outputs = p.object_output_params(data_title, instrument='FORS2')
data_dir = u.check_trailing_slash(data_dir)
destination = u.check_trailing_slash(destination)
destination = data_dir + destination
u.mkdir_check_nested(destination)
origin = u.check_trailing_slash(origin)
science_origin = data_dir + origin + "science/"
print(science_origin)
filters = outputs['filters']
frb_params = p.object_params_frb(obj=data_title[:-2])
epoch_params = p.object_params_fors2(obj=data_title)
background_origin_eso = ""
if eso_back:
background_origin_eso = data_dir + "/" + origin + "/backgrounds/"
if method == "SExtractor backgrounds only":
background_origin = f"{data_dir}{origin}backgrounds_sextractor/"
elif method == "polynomial fit":
background_origin = f"{destination}backgrounds/" # f"{destination}backgrounds_{method.replace(' ', '')}_degree_{degree}_local_{local}_globalsub_{global_sub}/"
else:
background_origin = f"{destination}backgrounds/" # f"{destination}backgrounds_{method.replace(' ', '')}_local_{local}_globalsub_{global_sub}/"
trimmed_path = ""
if trim_image:
trimmed_path = f"{data_dir}{origin}trimmed_to_background/"
u.mkdir_check_nested(trimmed_path)
ra = frb_params["burst_ra"]
dec = frb_params["burst_dec"]
if all_synths:
ras = epoch_params["test_synths"]["ra"]
decs = epoch_params["test_synths"]["dec"]
else:
ras = [ra]
decs = [dec]
for fil in filters:
trimmed_path_fil = ""
if trim_image:
trimmed_path_fil = f"{trimmed_path}{fil}/"
u.mkdir_check(trimmed_path_fil)
background_fil_dir = f"{background_origin}{fil}/"
u.mkdir_check_nested(background_fil_dir)
science_destination_fil = f"{destination}science/{fil}/"
u.mkdir_check_nested(science_destination_fil)
files = os.listdir(science_origin + fil + "/")
for file_name in files:
if file_name.endswith('.fits'):
new_file = file_name.replace("norm", "bg_sub")
new_path = f"{science_destination_fil}/{new_file}"
print("NEW_PATH:", new_path)
science = science_origin + fil + "/" + file_name
# First subtract ESO Reflex background images
# frame = (frame_deg / f.get_pixel_scale(file=science, astropy_units=True)[1]).to(f.pix).value
if eso_back:
background_eso = background_origin_eso + fil + "/" + file_name.replace("SCIENCE_REDUCED",
"PHOT_BACKGROUND_SCI")
ff.subtract_file(file=science, sub_file=background_eso, output=new_path)
science_image = new_path
if method != "ESO backgrounds only":
print(ra, dec)
print("Science image:", science)
science_image = fits.open(science)
print("Science file:", science_image)
wcs_this = WCS(header=science_image[0].header)
if method == "SExtractor backgrounds only":
background = background_origin + fil + "/" + file_name + "_back.fits"
print("Background image:", background)
else:
if method == "median value":
print(science_image[0].data.shape)
_, background_value, _ = sigma_clipped_stats(science_image[0].data)
background = deepcopy(science_image)
background[0].data = np.full(shape=science_image[0].data.shape, fill_value=background_value)
background_path = background_origin + fil + "/" + file_name.replace("SCIENCE_REDUCED",
"PHOT_BACKGROUND_MEDIAN")
# Next do background fitting.
else:
background = deepcopy(science_image)
background[0].data = np.zeros(background[0].data.shape)
background_path = background_origin + fil + "/" + file_name.replace("SCIENCE_REDUCED",
"PHOT_BACKGROUND_FITTED")
for i, ra in enumerate(ras):
dec = decs[i]
x, y = wcs_this.all_world2pix(ra, dec, 0)
print(x, y)
bottom, top, left, right = ff.subimage_edges(data=science_image[0].data, x=x, y=y,
frame=frame)
if do_mask:
# Produce a pixel mask that roughly masks out the true sources in the image so that
# they don't get fitted.
mask_max = 10
_, pixel_scale = ff.get_pixel_scale(science_image)
sextractor = Table.read(
f"{data_dir}analysis/sextractor/4-divided_by_exp_time/{fil}/{file_name.replace('.fits', '_psf-fit.cat')}",
format='ascii.sextractor')
weights = np.ones(shape=science_image[0].data.shape)
for obj in filter(
lambda o: left < o["X_IMAGE"] < right and bottom < o["Y_IMAGE"] < top,
sextractor):
mask_rad = min(int(obj["A_WORLD"] * obj["KRON_RADIUS"] / pixel_scale), mask_max)
x_prime = int(np.round(obj["X_IMAGE"]))
y_prime = int(np.round(obj["Y_IMAGE"]))
weights[y_prime - mask_rad:y_prime + mask_rad,
x_prime - mask_rad:x_prime + mask_rad] = 0.0
plt.imshow(weights, origin="lower")
plt.savefig(
background_origin + fil + "/" + file_name.replace("norm.fits", "mask.png"))
else:
weights = None
background_this = fit_background_fits(image=science_image,
model_type=method[:method.find(" ")],
deg=degree, local=local,
global_sub=global_sub,
centre_x=x, centre_y=y, frame=frame,
weights=weights)
background[0].data += background_this[0].data
if recorrect_subbed:
offset = get_median_background(image=science,
ra=epoch_params["renormalise_centre_ra"],
dec=epoch_params["renormalise_centre_dec"], frame=50,
show=False,
output=new_path[
:new_path.find("bg_sub")] + "renorm_patch_")
print("RECORRECT_SUBBED:", recorrect_subbed)
print("SUBTRACTING FROM BACKGROUND:", offset)
print(bottom, top, left, right)
print(background[0].data[bottom:top, left:right].shape)
print(np.median(background[0].data[bottom:top, left:right]))
background[0].data[bottom:top, left:right] -= offset
print(np.median(background[0].data[bottom:top, left:right]))
if trim_image:
print("TRIMMED_PATH_FIL:", trimmed_path_fil)
science_image = ff.trim_file(path=science_image, left=left, right=right, top=top,
bottom=bottom,
new_path=trimmed_path_fil + file_name.replace(
"norm.fits",
"trimmed_to_back.fits"))
print("Science after trim:", science_image)
background = ff.trim_file(path=background, left=left, right=right, top=top,
bottom=bottom,
new_path=background_path)
print("Writing background to:")
print(background_path)
background.writeto(background_path, overwrite=True)
print("SCIENCE:", science_image)
print("BACKGROUND:", background)
subbed = ff.subtract_file(file=science_image, sub_file=background, output=new_path)
# # TODO: check if regions overlap
#
# plt.hist(subbed[0].data[int(y - frame + 1):int(y + frame - 1),
# int(x - frame + 1):int(x + frame - 1)].flatten(),
# bins=10)
# plt.savefig(new_path[:new_path.find("bg_sub")] + "histplot.png")
# plt.close()
copyfile(data_dir + "/" + origin + "/" + data_title + ".log", destination + data_title + ".log")
u.write_log(path=destination + data_title + ".log",
action=f'Backgrounds subtracted using 4-background_subtract.py with method {method}\n')
def main(data_title: str, show: bool = False):
properties = p.object_params_xshooter(data_title)
path = properties['data_dir']
master_path = path + '/1-master_calibs/'
reduced_path = path + '/2-reduced/'
defringed_path = path + '/3-defringed/'
u.mkdir_check(defringed_path)
# Define fringe measurement points.
# high_xs = [267, 267, 267, 267, 267, 267, 267, 267]
# high_ys = [279, 279, 279, 279, 279, 279, 279, 279]
# low_xs = [266, 266, 266, 267, 267, 270, 274, 273]
# low_ys = [293, 295, 298, 301, 303, 305, 303, 292]
high_xs = [219, 380, 426, 515, 156, 495, 310]
high_ys = [166, 369, 185, 33, 59, 195, 70]
low_xs = [219, 380, 424, 474, 160, 500, 315]
low_ys = [120, 342, 213, 39, 34, 160, 35]
# n_random = 1000
#
# high_xs = np.random.random(n_random)
# high_xs *= 507
# high_xs += 29
# high_xs = np.round(high_xs)
# high_xs = high_xs.astype(int)
#
# high_ys = np.random.random(n_random)
# high_ys *= 200
# high_ys += 20
# high_ys = np.round(high_ys)
# high_ys = high_ys.astype(int)
#
# low_xs = np.random.random(n_random)
# low_xs *= 507
# low_xs += 29
# low_xs = np.round(low_xs)
# low_xs = low_xs.astype(int)
#
# low_ys = np.random.random(n_random)
# low_ys *= 200
# low_ys += 20
# low_ys = np.round(low_ys)
# low_ys = low_ys.astype(int)
filters = filter(lambda f: os.path.isdir(reduced_path + f),
os.listdir(reduced_path))
for f in filters:
print('Constructing fringe map for', f)
filter_path = reduced_path + f + '/'
defringed_filter_path = defringed_path + f + '/'
master_filter_path = master_path + f + '/'
u.mkdir_check(defringed_filter_path)
files = list(
filter(lambda file: file[-5:] == '.fits', os.listdir(filter_path)))
# Construct fringe map by median-combining science images.
fringe_map = ff.stack(files,
directory=filter_path,
output=master_filter_path + 'fringe_map.fits',
stack_type='median',
inherit=False,
show=show,
normalise=True)
fringe_map = fringe_map[0].data
map_differences = []
for i in range(len(high_xs)):
# Take
high_y = high_ys[i]
high_x = high_xs[i]
high_cut = fringe_map[high_y - 1:high_y + 1, high_x - 1:high_x + 1]
high = np.nanmedian(high_cut)
low_y = low_ys[i]
low_x = low_xs[i]
low_cut = fringe_map[low_y - 1:low_y + 1, low_x - 1:low_x + 1]
low = np.nanmedian(low_cut)
map_differences.append(high - low)
for file in os.listdir(filter_path):
print(file)
hdu = fits.open(filter_path + file)
data = hdu[0].data
image_differences = []
factors = []
for i in range(len(high_xs)):
high_y = high_ys[i]
high_x = high_xs[i]
high_cut = data[high_y - 2:high_y + 2, high_x - 2:high_x + 2]
high = np.nanmedian(high_cut)
low_y = low_ys[i]
low_x = low_xs[i]
low_cut = data[low_y - 2:low_y + 2, low_x - 2:low_x + 2]
low = np.nanmedian(low_cut)
difference = high - low
image_differences.append(difference)
factor = difference / map_differences[i]
factors.append(factor)
used_factor = np.nanmedian(factors)
adjusted_map = fringe_map * used_factor
data = data - adjusted_map
hdu[0].data = data
norm = pl.nice_norm(data)
if show:
plt.imshow(data, norm=norm, origin='lower')
plt.show()
hdu.writeto(defringed_filter_path + file, overwrite=True)
if show:
norm = pl.nice_norm(fringe_map)
plt.imshow(fringe_map, norm=norm, origin='lower')
plt.scatter(high_xs, high_ys)
plt.scatter(low_xs, low_ys)
plt.show()
copyfile(reduced_path + data_title + ".log",
defringed_path + data_title + ".log")
u.write_log(path=defringed_path + data_title + ".log",
action='Edges trimmed using 4-trim.py\n')
def main(
field_name: str,
epoch_name: str,
imaging: bool,
spectroscopy: bool,
instrument: str,
furby_path: str,
do: str,
do_not_reuse_masters: bool,
overwrite_download: bool,
distance_tolerance: float,
snr_min: float,
class_star_tolerance: float,
debug_level: int,
):
u.debug_level = debug_level
new_field = False
directory = fld.load_epoch_directory()
if epoch_name is not None:
print(f"Looking for {epoch_name} in directory...")
if epoch_name in directory:
epoch_dict = directory[epoch_name]
field_name = epoch_dict["field_name"]
instrument = epoch_dict["instrument"]
mode = epoch_dict["mode"]
if mode == "imaging":
imaging = True
elif mode == "spectroscopy":
spectroscopy = True
else:
print(f"{epoch_name} not found.")
# Do automated FURBY process.
if furby_path is not None:
new_field = True
furby = True
imaging = True
healpix_path = furby_path.replace(".json", "_hp.fits")
if not os.path.isfile(healpix_path):
healpix_path = None
params = fld.FRBField.param_from_furby_json(
json_path=furby_path,
healpix_path=healpix_path,
)
field_name = params["name"]
field = fld.Field.from_params(name=field_name)
instrument = "vlt-fors2"
epoch_name = f"{field_name}_FORS2_1"
fld.FORS2ImagingEpoch.new_yaml(
name=epoch_name,
path=fld.FORS2ImagingEpoch.build_param_path(
instrument_name=instrument,
field_name=field_name,
epoch_name=epoch_name),
field=field.name,
instrument=instrument,
data_path=os.path.join(field_name, "imaging", instrument,
epoch_name, ""))
# epoch = fld.FORS2ImagingEpoch.from_params(
# name=epoch_name,
# instrument=instrument,
# field=field,
# old_format=False,
# )
else:
if field_name is None:
fields = ["New field"]
fields += fld.list_fields()
old_fields = fld.list_fields_old()
for old_field in old_fields:
if old_field not in fields and f"FRB20{old_field[3:]}" not in fields:
fields.append(old_field)
opt, field_name = u.select_option(
"No field specified. Please select one:",
options=fields,
sort=False)
if opt == 0:
new_field = True
field_name = input("Please enter the name of the new field:\n")
# Check for field param file
if not new_field:
field = fld.Field.from_params(name=field_name)
else:
field = None
# If this field has no parameter file, ask to create one.
if field is None:
param_path = os.path.join(p.param_dir, "fields", "")
# Check for old format param file, and ask to convert if found.
old_field_name = f"FRB{field_name[-8:]}"
old_params = p.object_params_frb(obj=old_field_name)
print()
field_param_path = os.path.join(param_path, field_name)
u.mkdir_check(field_param_path)
field_param_path_yaml = os.path.join(field_param_path,
f"{field_name}.yaml")
if old_params is None:
if not new_field:
print(f"{field_name} not found in the param directory.")
if u.select_yn(
f"Create a new param file at '{field_param_path_yaml}'?"
):
fld.Field.new_params_from_input(
field_name=field_name,
field_param_path=field_param_path)
else:
print("Exiting.")
exit(0)
else:
print("Old format param file detected.")
if u.select_yn("Convert to new format?"):
fld.FRBField.convert_old_param(frb=old_field_name)
else:
print("Exiting...")
exit(0)
field = fld.Field.from_params(name=field_name)
if spectroscopy:
mode = "Spectroscopy"
elif imaging:
mode = "Imaging"
else:
_, mode = u.select_option(message="Please select a mode.",
options=["Imaging", "Spectroscopy"])
if mode == "Spectroscopy":
if epoch_name is None:
# Build a list of imaging epochs from that field.
field.gather_epochs_spectroscopy()
# Let the user select an epoch.
epoch = field.select_epoch_spectroscopy()
else:
if instrument is None:
instrument = fld.select_instrument(mode="spectroscopy")
epoch = fld.SpectroscopyEpoch.from_params(epoch_name,
instrument=instrument,
field=field)
else: # if mode == "Imaging"
if epoch_name is None:
# Build a list of imaging epochs from that field.
if type(field) is fld.FRBField:
field.gather_epochs_old()
field.gather_epochs_imaging()
# Let the user select an epoch.
epoch = field.select_epoch_imaging()
else:
if instrument is None:
instrument = fld.select_instrument(mode="imaging")
epoch = fld.ImagingEpoch.from_params(epoch_name,
instrument=instrument,
field=field)
epoch.field = field
u.debug_print(2, "pipeline.py: type(epoch) ==", type(epoch))
epoch.do = do
epoch.pipeline(do_not_reuse_masters=do_not_reuse_masters,
overwrite_download=overwrite_download,
distance_tolerance=distance_tolerance,
snr_min=snr_min,
class_star_tolerance=class_star_tolerance)
def main(data_title: str, show: bool = False):
properties = p.object_params_xshooter(data_title)
path = properties['data_dir']
raw_path = path + '/0-data_with_raw_calibs/'
master_path = path + '/1-master_calibs/'
reduced_path = path + '/2-reduced/'
u.mkdir_check(raw_path)
u.mkdir_check(master_path)
u.mkdir_check(reduced_path)
files = os.listdir(raw_path)
biases = []
flats = {}
science = {}
airmasses = {}
print('Creating lists of files.')
for file in files:
if file[-5:] == '.fits':
hdu = fits.open(raw_path + file)
header = hdu[0].header
obj = header['OBJECT']
f = header['ESO INS FILT1 NAME']
if 'BIAS' in obj:
biases.append(file)
elif 'FLAT' in obj:
if f not in flats:
flats[f] = []
flats[f].append(file)
else:
if f not in science:
science[f] = []
airmasses[f] = []
science[f].append(file)
bias_hdus = []
for bias in biases:
bias_hdus.append(bias)
# Stack biases.
print(f'Processing biases.')
ff.stack(bias_hdus, output=master_path + f'master_bias.fits', stack_type='median', directory=raw_path, inherit=False)
master_bias = CCDData.read(master_path + f'master_bias.fits', unit='du')
# Loop through filters.
for f in science:
flats_filter = flats[f]
master_path_filter = master_path + f + '/'
u.mkdir_check(master_path_filter)
print(f'Processing flats for filter {f}.')
flats_ccds = []
for flat in flats_filter:
flat_ccd = CCDData.read(raw_path + flat, unit='du')
# Subtract master bias from each flat.
flat_ccd = ccdproc.subtract_bias(ccd=flat_ccd, master=master_bias)
flats_ccds.append(flat_ccd)
# Stack debiased flats.
master_flat = ff.stack(flats_ccds, output=None, stack_type='median', inherit=False)
master_flat.writeto(master_path_filter + f'master_flat.fits', overwrite=True)
master_flat = CCDData.read(master_path_filter + f'master_flat.fits', unit='du')
science_filter = science[f]
reduced_path_filter = reduced_path + f + '/'
u.mkdir_check(reduced_path_filter)
# Loop through the science images.
for image in science_filter:
print(f'Reducing {image}.')
image_ccd = CCDData.read(raw_path + image, unit='du')
if show:
norm = ImageNormalize(image_ccd.data, interval=ZScaleInterval(), stretch=SqrtStretch())
plt.imshow(image_ccd.data, origin='lower', norm=norm)
plt.title('Unreduced image')
plt.show()
# Subtract master bias from science image.
image_ccd = ccdproc.subtract_bias(image_ccd, master_bias)
if show:
norm = ImageNormalize(image_ccd.data, interval=ZScaleInterval(), stretch=SqrtStretch())
plt.imshow(image_ccd.data, origin='lower', norm=norm)
plt.title('After debiasing')
plt.show()
# Divide by master flat.
image_ccd = ccdproc.flat_correct(image_ccd, master_flat)
if show:
norm = ImageNormalize(image_ccd.data, interval=ZScaleInterval(), stretch=SqrtStretch())
plt.imshow(image_ccd.data, origin='lower', norm=norm)
plt.title('After flatfielding')
plt.show()
# Convert back to HDU object for saving.
image_ccd = image_ccd.to_hdu()
image_ccd.writeto(reduced_path_filter + image, overwrite=True)
u.write_log(path=reduced_path + data_title + ".log", action=f'Reduced using 1-reduce.py')
def main(obj, test, n, mag_lower, mag_upper, colour_upper, colour_lower):
properties = p.object_params_fors2(obj)
output = p.object_output_params(obj=obj, instrument='FORS2')
paths = p.object_output_paths(obj)
burst_properties = p.object_params_frb(obj[:-2])
synth_path = properties['data_dir'] + 'synthetic/'
u.mkdir_check(synth_path)
synth_path = synth_path + 'random/'
u.mkdir_check(synth_path)
now = time.Time.now()
now.format = 'isot'
test = str(now) + '_' + test
test_path = synth_path + test + '/'
u.mkdir_check(test_path)
filters = {}
bluest = None
bluest_lambda = np.inf
for f in output['filters']:
filter_properties = p.filter_params(f=f, instrument='FORS2')
filters[f] = filter_properties
lambda_eff = filter_properties['lambda_eff']
if lambda_eff < bluest_lambda:
bluest_lambda = lambda_eff
bluest = f
# Insert random sources in the bluest filter.
f_0 = bluest[0]
output_properties = p.object_output_params(obj)
fwhm = output_properties[f_0 + '_fwhm_pix']
zeropoint, _, airmass, _ = ph.select_zeropoint(obj,
bluest,
instrument='fors2')
base_path = paths[f_0 + '_subtraction_image']
output_path = test_path + f_0 + '_random_sources.fits'
_, sources = ph.insert_random_point_sources_to_file(file=base_path,
fwhm=fwhm,
output=output_path,
n=n,
airmass=airmass,
zeropoint=zeropoint)
p.add_output_path(obj=obj,
key=f_0 + '_subtraction_image_synth_random',
path=output_path)
# Now insert sources at the same positions in other filters, but with magnitudes randomised.
for f in filters:
if f != bluest:
f_0 = f[0]
output_properties = p.object_output_params(obj)
fwhm = output_properties[f_0 + '_fwhm_pix']
zeropoint, _, airmass, _ = ph.select_zeropoint(obj,
f,
instrument='fors2')
base_path = paths[f_0 + '_subtraction_image']
mag = np.random.uniform(mag_lower, mag_upper, size=n)
output_path = test_path + f_0 + '_random_sources.fits'
ph.insert_point_sources_to_file(file=base_path,
fwhm=fwhm,
output=output_path,
x=sources['x_0'],
y=sources['y_0'],
mag=mag,
airmass=airmass,
zeropoint=zeropoint)
p.add_output_path(obj=obj,
key=f_0 + '_subtraction_image_synth_random',
path=output_path)
