def main(data_dir, data_title, destination, fil, object):
table = (destination + '/' + fil, science_only=False)
fil = fil.replace('/', '')
header_file = destination + '/' + fil + '_template.hdr'
header_stream = open(header_file, 'r')
header = header_stream.readlines()
header_stream.close()
# Transfer and/or modify header information
params = p.object_output_params(data_title, instrument='IMACS')
airmass = table['airmass'].mean()
saturate = table['saturate'].mean()
old_gain = table['gain'].mean()
n_frames = params[fil + '_n_exposures']
gain = gain_median_combine(old_gain=old_gain, n_frames=n_frames)
header.insert(-1, f'AIRMASS = {airmass}\n')
header.insert(-1, f'FILTER = {fil}\n')
header.insert(-1, f'OBJECT = {object}\n')
header.insert(-1, f'EXPTIME = 1.\n')
header.insert(-1, f'GAIN = {gain}\n')
header.insert(-1, f'SATURATE= {saturate}\n')
header.insert(-1, f'MJD-OBS = {float(np.nanmean(table["mjd_obs"]))}\n')
os.remove(header_file)
with open(header_file, 'w') as file:
file.writelines(header)
p.add_output_path(obj=data_title, key=fil + '_subtraction_image',
path=destination + '/' + fil + '_coadded.fits', instrument='IMACS')
def main(obj, input_path, output, path, prefix):
print(
"\nExecuting Python script pipeline_fors2/9-esorex_zeropoint_trim.py, with:"
)
print(f"\tepoch {obj}")
print(f"\tinput path {input_path}")
print(f"\toutput path {output}")
print(f"\tstandard star data path {path}")
print(f"\tfilter prefix {prefix}")
print()
ff.trim_file(path=input_path,
bottom=0,
top=950,
left=195,
right=1825,
new_path=output)
airmass = ff.get_airmass(input_path)
exptime = ff.get_exp_time(input_path)
p.add_params(
path + '/output_values', {
prefix + '_airmass': airmass,
prefix + '_airmass_err': 0,
prefix + '_exptime': exptime
})
p.add_output_path(obj=obj,
instrument='FORS2',
key=prefix + '_std_image',
path=output)
def main(comb_path, output_dir, obj, sextractor_path, path_suffix):
print("\nExecuting Python script pipeline_fors2/7-trim_combined.py, with:")
print(f"\tepoch {obj}")
print(f"\toutput directory {output_dir}")
print(f"\tsextractor directory {sextractor_path}")
print()
if sextractor_path is not None:
u.mkdir_check_nested(sextractor_path)
do_sextractor = True
else:
do_sextractor = False
# Build a list of the filter prefixes used.
fils = []
files = list(filter(lambda x: x[-4:] == '.tbl', os.listdir(comb_path)))
for file in files:
if file[0] not in fils:
fils.append(str(file[0]))
for fil in fils:
if do_sextractor:
u.mkdir_check(sextractor_path)
area_file = fil + "_coadded_area.fits"
comb_file = fil + "_coadded.fits"
left, right, bottom, top = f.detect_edges_area(comb_path + area_file)
# Trim a little extra to be safe.
left = left + 5
right = right - 5
top = top - 5
bottom = bottom + 5
f.trim_file(comb_path + comb_file,
left=left,
right=right,
top=top,
bottom=bottom,
new_path=output_dir + "/" + comb_file)
# Keep a trimmed version of the area file, it comes in handy later.
f.trim_file(comb_path + area_file,
left=left,
right=right,
top=top,
bottom=bottom,
new_path=output_dir + "/" + area_file)
if do_sextractor:
copyfile(output_dir + "/" + comb_file, sextractor_path + comb_file)
if path_suffix is None:
path_suffix = output_dir.split("/")[-2]
path_suffix = u.remove_trailing_slash(path_suffix)
p.add_output_path(obj=obj,
instrument='fors2',
key=fil[0] + '_trimmed_image' + path_suffix,
path=output_dir + "/" + comb_file)
def main(data_title: str):
properties = p.object_params_des(data_title)
path = properties['data_dir']
data_path = path + '/0-data/'
u.mkdir_check(data_path)
os.listdir(path)
print(path)
for file in filter(lambda fil: fil[-5:] == '.fits', os.listdir(path)):
f = file[-6]
shutil.copy(path + file, data_path + str(f) + '_cutout.fits')
p.add_output_path(obj=data_title, instrument='DES', key=f + '_subtraction_image', path=data_path + str(f) + '_cutout.fits')
def main(obj, test, curves_path):
properties = p.object_params_fors2(obj)
if curves_path[-1] != '/':
curves_path += '/'
mag_table_file = filter(lambda file: file[-4:] == '.csv',
os.listdir(curves_path)).__next__()
mag_table = table.Table.read(curves_path + mag_table_file)
filters = mag_table.colnames.copy()
filters.remove('model')
synth_path = properties['data_dir'] + 'synthetic/'
u.mkdir_check(synth_path)
synth_path = synth_path + 'frb_position/'
u.mkdir_check(synth_path)
now = time.Time.now()
now.format = 'isot'
synth_path += f'sn_models_{now}/'
u.mkdir_check(synth_path)
for row in mag_table:
model = row['model']
magnitudes = []
for f in filters:
magnitudes.append(row[f])
test_spec = test + '_' + model
test_path = synth_path + test_spec + '/'
ph.insert_synthetic_at_frb(obj=obj,
test_path=test_path,
filters=filters,
magnitudes=magnitudes,
add_path=False)
for f in filters:
p.add_output_path(obj=obj,
key=f[0] + '_subtraction_image_synth_frb_sn_models',
path=synth_path)
def main(obj, test, mag_min, mag_max, increment, instrument):
properties = p.object_params_instrument(obj, instrument=instrument)
output = p.object_output_params(obj=obj, instrument=instrument)
paths = p.object_output_paths(obj=obj, instrument=instrument)
synth_path = properties['data_dir'] + 'synthetic/'
u.mkdir_check(synth_path)
synth_path = synth_path + 'frb_position/'
u.mkdir_check(synth_path)
now = time.Time.now()
now.format = 'isot'
synth_path += f'range_{now}/'
u.mkdir_check(synth_path)
filters = output['filters']
for magnitude in np.arange(mag_min, mag_max, increment):
magnitudes = []
for i in range(len(filters)):
magnitudes.append(magnitude)
test_spec = test + '_' + str(u.round_to_sig_fig(magnitude, 4))
test_path = synth_path + test_spec + '/'
ph.insert_synthetic_at_frb(obj=properties,
test_path=test_path,
filters=filters,
magnitudes=magnitudes,
add_path=False,
psf=True,
output_properties=output,
instrument=instrument,
paths=paths)
p.add_output_path(obj=obj,
key='subtraction_image_synth_frb_range',
path=synth_path,
instrument=instrument)
def main(obj, test, n, filter_dist, instrument, limit):
properties = p.object_params_instrument(obj, instrument=instrument)
burst_properties = p.object_params_frb(obj=obj[:-2])
output = p.object_output_params(obj=obj, instrument=instrument)
paths = p.object_output_paths(obj=obj, instrument=instrument)
z = burst_properties['z']
mjd_burst = burst_properties['mjd_burst']
ebv_mw = burst_properties['dust_ebv']
mjd_obs = properties['mjd']
synth_path = properties['data_dir'] + 'synthetic/'
u.mkdir_check(synth_path)
synth_path = synth_path + 'sn_random/'
u.mkdir_check(synth_path)
epoch = mjd_obs - mjd_burst
f_0 = filter_dist[0]
hg_ra = burst_properties['hg_ra']
hg_dec = burst_properties['hg_dec']
burst_ra = burst_properties['burst_ra']
burst_dec = burst_properties['burst_dec']
image_path = paths[f_0 + '_' + properties['subtraction_image']]
image = fits.open(image_path)
wcs_info = wcs.WCS(image[0].header)
burst_x, burst_y = wcs_info.all_world2pix(burst_ra, burst_dec, 0)
now = time.Time.now()
now.format = 'isot'
synth_path += test + '_' + str(now) + '/'
u.mkdir_check(synth_path)
filters = output['filters']
psf_models = []
for i, f in enumerate(filters):
sn.register_filter(f=f, instrument=instrument)
psf_model = fits.open(paths[f[0] + '_psf_model'])
psf_models.append(psf_model)
for i in range(n):
test_spec = test + '_' + str(i)
test_path = synth_path + test_spec + '/'
magnitudes = []
mags_filters, model, x, y, tbl = sn.random_light_curves_type_ia(
filters=filters,
image=image,
hg_ra=hg_ra,
hg_dec=hg_dec,
z=z,
ebv_mw=ebv_mw,
output_path=test_path,
output_title=test,
limit=limit,
x=burst_x,
y=burst_y,
ra=burst_ra,
dec=burst_dec)
days = mags_filters['days']
for f in filters:
magnitude = sn.magnitude_at_epoch(epoch=epoch,
days=days,
mags=mags_filters[f])
print(f, 'mag:', magnitude)
magnitudes.append(magnitude)
ph.insert_synthetic(x=float(x),
y=float(y),
obj=properties,
test_path=test_path,
filters=filters,
magnitudes=magnitudes,
suffix='sn_random_random_ia',
extra_values=tbl,
paths=paths,
output_properties=output,
psf_models=psf_models,
instrument=instrument)
p.add_output_path(obj=obj,
key='subtraction_image_synth_sn_random_ia',
path=synth_path,
instrument=instrument)
def main(obj, test, n, mag_lower, mag_upper, colour_upper, colour_lower):
properties = p.object_params_fors2(obj)
output = p.object_output_params(obj=obj, instrument='FORS2')
paths = p.object_output_paths(obj)
burst_properties = p.object_params_frb(obj[:-2])
synth_path = properties['data_dir'] + 'synthetic/'
u.mkdir_check(synth_path)
synth_path = synth_path + 'random/'
u.mkdir_check(synth_path)
now = time.Time.now()
now.format = 'isot'
test = str(now) + '_' + test
test_path = synth_path + test + '/'
u.mkdir_check(test_path)
filters = {}
bluest = None
bluest_lambda = np.inf
for f in output['filters']:
filter_properties = p.filter_params(f=f, instrument='FORS2')
filters[f] = filter_properties
lambda_eff = filter_properties['lambda_eff']
if lambda_eff < bluest_lambda:
bluest_lambda = lambda_eff
bluest = f
# Insert random sources in the bluest filter.
f_0 = bluest[0]
output_properties = p.object_output_params(obj)
fwhm = output_properties[f_0 + '_fwhm_pix']
zeropoint, _, airmass, _ = ph.select_zeropoint(obj,
bluest,
instrument='fors2')
base_path = paths[f_0 + '_subtraction_image']
output_path = test_path + f_0 + '_random_sources.fits'
_, sources = ph.insert_random_point_sources_to_file(file=base_path,
fwhm=fwhm,
output=output_path,
n=n,
airmass=airmass,
zeropoint=zeropoint)
p.add_output_path(obj=obj,
key=f_0 + '_subtraction_image_synth_random',
path=output_path)
# Now insert sources at the same positions in other filters, but with magnitudes randomised.
for f in filters:
if f != bluest:
f_0 = f[0]
output_properties = p.object_output_params(obj)
fwhm = output_properties[f_0 + '_fwhm_pix']
zeropoint, _, airmass, _ = ph.select_zeropoint(obj,
f,
instrument='fors2')
base_path = paths[f_0 + '_subtraction_image']
mag = np.random.uniform(mag_lower, mag_upper, size=n)
output_path = test_path + f_0 + '_random_sources.fits'
ph.insert_point_sources_to_file(file=base_path,
fwhm=fwhm,
output=output_path,
x=sources['x_0'],
y=sources['y_0'],
mag=mag,
airmass=airmass,
zeropoint=zeropoint)
p.add_output_path(obj=obj,
key=f_0 + '_subtraction_image_synth_random',
path=output_path)
def main(obj, test, n, filter_dist, sn_type, instrument, limit):
properties = p.object_params_instrument(obj, instrument=instrument)
burst_properties = p.object_params_frb(obj=obj[:-2])
output = p.object_output_params(obj=obj, instrument=instrument)
paths = p.object_output_paths(obj=obj, instrument=instrument)
z = burst_properties['z']
mjd_burst = burst_properties['mjd_burst']
ebv_mw = burst_properties['dust_ebv']
mjd_obs = properties['mjd']
synth_path = properties['data_dir'] + 'synthetic/'
u.mkdir_check(synth_path)
synth_path = synth_path + 'sn_random/'
u.mkdir_check(synth_path)
epoch = mjd_obs - mjd_burst
f_0 = filter_dist[0]
hg_ra = burst_properties['hg_ra']
hg_dec = burst_properties['hg_dec']
image_path = paths[f_0 + '_' + properties['subtraction_image']]
image = fits.open(image_path)
now = time.Time.now()
now.format = 'isot'
test += sn_type + '_'
synth_path += test + str(now) + '/'
u.mkdir_check(synth_path)
filters = output['filters']
if 'FRB190608' in obj:
limit = 45
psf_models = []
for f in filters:
sn.register_filter(f=f)
psf_model = fits.open(paths[f[0] + '_psf_model'])
psf_models.append(psf_model)
for i in range(n):
test_spec = test + str(i)
test_path = synth_path + test_spec + '/'
magnitudes = []
mags_filters, model, x, y, tbl = sn.random_light_curves(
sn_type=sn_type,
filters=filters,
image=image,
hg_ra=hg_ra,
hg_dec=hg_dec,
z=z,
ebv_mw=ebv_mw,
output_path=test_path,
output_title=test,
limit=limit)
days = mags_filters['days']
for f in filters:
magnitude = sn.magnitude_at_epoch(epoch=epoch,
days=days,
mags=mags_filters[f])
print(f, 'mag:', magnitude)
magnitudes.append(magnitude)
ph.insert_synthetic(x=x,
y=y,
obj=properties,
test_path=test_path,
filters=filters,
magnitudes=magnitudes,
suffix='sn_random_random_' + sn_type,
extra_values=tbl,
paths=paths,
output_properties=output,
psf_models=psf_models,
instrument=instrument)
p.add_output_path(obj=obj,
key='subtraction_image_synth_sn_random_' + sn_type,
path=synth_path,
instrument=instrument)
def tweak_final(sextractor_path: str,
destination: str,
epoch: str,
instrument: str,
show: bool,
tolerance: float = 10.,
output_suffix: str = 'astrometry',
input_suffix='coadded',
stars_only: bool = False,
path_add: str = 'subtraction_image',
manual: bool = False,
specific_star: bool = False):
"""
A wrapper for tweak, to interface with the .yaml param files and provide offsets to all filters used in an
observation.
:param sextractor_path: Path to SExtractor-generated catalogue.
:param destination: Directory to write tweaked image to.
:param epoch: The epoch number of the observation to be tweaked.
:param instrument: The instrument on which the observation was taken.
:param show: Plot matches onscreen?
:param tolerance: Tolerance, in pixels, within which matches will be accepted.
:param output_suffix: String to append to filename of output file.
:param input_suffix: Suffix appended to filenmae of input file.
:param stars_only: Only match using stars, determined using SExtractor's 'class_star' output.
:param path_add: Key under which to add the output path in the 'output_paths.yaml' file.
:param manual: Use manual offset?
:param specific_star: Use a specific star to tweak? This means, instead of finding the closest matches for many
stars, alignment is attempted with a single star nearest the given position.
:return: None
"""
u.mkdir_check(destination)
properties = p.object_params_instrument(obj=epoch, instrument=instrument)
frb_properties = p.object_params_frb(obj=epoch[:-2])
cat_name = properties['cat_field_name']
manual = properties['manual_astrometry'] and manual
cat_path = frb_properties['data_dir'] + cat_name.upper(
) + "/" + cat_name.upper() + ".csv"
if cat_path is not None:
outputs = p.object_output_params(obj=epoch, instrument=instrument)
filters = outputs['filters']
param_dict = {}
for filt in filters:
print(filt)
f = filt[0]
if manual:
offset_x = properties[f + '_offset_x']
offset_y = properties[f + '_offset_y']
else:
offset_x = None
offset_y = None
if specific_star:
burst_properties = p.object_params_frb(epoch[:-2])
star_ra = burst_properties['alignment_ra']
star_dec = burst_properties['alignment_dec']
else:
star_ra = star_dec = None
param = tweak(
sextractor_path=sextractor_path + f + '_psf-fit.cat',
destination=destination + f + '_' + output_suffix + '.fits',
image_path=destination + f + '_' + input_suffix + '.fits',
cat_path=cat_path,
cat_name=cat_name,
tolerance=tolerance,
show=show,
stars_only=stars_only,
manual=manual,
offset_x=offset_x,
offset_y=offset_y,
specific_star=specific_star,
star_ra=star_ra,
star_dec=star_dec)
for par in param:
param_dict[f + '_' + par] = param[par]
p.add_output_path(obj=epoch,
key=f + '_' + path_add,
path=destination + f + '_' + output_suffix +
'.fits',
instrument=instrument)
p.add_params(properties['data_dir'] + 'output_values.yaml',
params=param_dict)
else:
print('No catalogue found for this alignment.')
def main(obj, astrometry_path, sextractor_path, template, show):
print("\nExecuting Python script pipeline_fors2/8-astrometry.py, with:")
print(f"\tepoch {obj}")
print(f"\tastrometry path {astrometry_path}")
print(f"\tsextractor path {sextractor_path}")
print(f"\ttemplate {template}")
print(f"\tshow {show}")
print()
files = os.listdir(astrometry_path)
template_file = template[0] + '_astrometry.fits'
params = p.object_params_fors2(obj)
for file in files:
if 'coadded.fits' in file:
astrometry_file = file.replace('_coadded', '_astrometry')
coadd = fits.open(astrometry_path + file)
header = coadd[0].header
if os.path.isfile(astrometry_path + astrometry_file):
astrometry = fits.open(astrometry_path + astrometry_file, mode='update')
astrometry[0].header['AIRMASS'] = header['AIRMASS']
astrometry[0].header['FILTER'] = header['FILTER']
astrometry[0].header['OBJECT'] = header['OBJECT']
gain = header['GAIN']
astrometry[0].header['GAIN'] = gain
astrometry[0].header['EXPTIME'] = header['EXPTIME']
astrometry[0].header['RDNOISE'] = 0
astrometry[0].header['MJD-OBS'] = header['MJD-OBS']
ff.add_log(astrometry, 'Refined astrometric solution using Astrometry.net')
coadd.close()
astrometry.close()
print()
p.add_output_path(obj=obj, key=file[0] + '_astrometry_image',
path=astrometry_path + astrometry_file)
if file != template_file:
shutil.copy(astrometry_path + astrometry_file,
astrometry_path + astrometry_file.replace('.fits', '_orig.fits'))
shutil.copy(astrometry_path + template_file,
astrometry_path + template_file.replace('.fits', '_orig.fits'))
ff.align(comparison=astrometry_path + astrometry_file,
template=astrometry_path + template_file,
comparison_output=astrometry_path + astrometry_file,
template_output=astrometry_path + template_file)
else:
print(f'No astrometry file corresponding to {file} found.')
# TODO: This will break if more than two filters have been used on this observation.
# Rewrite align to work with one template and multiple comparisons.
if params['astrometry_tweak']:
tweak_final(epoch=obj, sextractor_path=sextractor_path,
destination=astrometry_path, instrument='FORS2',
show=show, output_suffix='astrometry_tweaked', input_suffix='astrometry', stars_only=True,
specific_star=False,
path_add='tweaked_image', manual=False)
tweak_final(epoch=obj, sextractor_path=sextractor_path,
destination=astrometry_path, instrument='FORS2',
show=show, output_suffix='astrometry_tweaked_hg', input_suffix='astrometry', stars_only=True,
specific_star=True,
path_add='tweaked_image_hg', manual=False)
if params['manual_astrometry']:
tweak_final(epoch=obj, sextractor_path=sextractor_path,
destination=astrometry_path, instrument='FORS2',
show=show, output_suffix='astrometry_tweaked_manual', input_suffix='astrometry', stars_only=True,
path_add='tweaked_image_manual', manual=True)
def main(obj, key, path, instrument):
p.add_output_path(obj=obj, key=key, path=path, instrument=instrument)