def check_input_images(input_directory: str, **kwargs):
table = image.fits_table_all(input_directory, science_only=True)
table.sort("FILENAME")
template = table[0]
template_path = os.path.join(input_directory, template["FILENAME"])
template_img = image.ImagingImage.select_child_class(
instrument=image.detect_instrument(path=template_path))(template_path)
keys = template_img.header_keys()
exptime_key = keys["exposure_time"]
instrument_key = keys["instrument"]
if "gain_key" in kwargs:
gain_key = kwargs["gain_key"]
else:
gain_key = keys["gain"]
if gain_key.startswith("HIERARCH"):
gain_key = gain_key[9:]
exptime = np.round(float(template[exptime_key]))
instrument = template[instrument_key]
gain = np.round(template[gain_key])
for file in table[1:]:
if np.round(float(file[exptime_key])) != exptime:
raise ValueError(
f"Input files have different EXPTIME ({file[exptime_key]} != {exptime}), file {file['FILENAME']}"
)
if file[instrument_key] != instrument:
raise ValueError(
f"Input files were taken with different instruments ({file[instrument_key]} != {instrument}), file {file['filename']}."
)
if np.round(file[gain_key]) != gain:
raise ValueError(
f"Files specify different gains ({gain} != {file[gain_key]}, file {file['FILENAME']}"
)
def inject_header(file_path: str,
input_directory: str,
extra_items: dict = None,
keys: dict = None,
coadd_type: str = 'median',
ext: int = 0):
table = image.fits_table_all(input_directory, science_only=False)
table.sort("FILENAME")
template = image.ImagingImage.from_fits(table[0]["PATH"])
cls = type(template)
important_keys = template.header_keys()
if keys is not None:
important_keys.update(keys)
airmasses = np.float64(table[important_keys['airmass']])
airmass_mean = np.nanmean(airmasses)
insert_dict = {
f"AIRMASS":
airmass_mean,
f"AIRMASS_ERR":
max(
np.nanmax(airmasses) - airmass_mean,
airmass_mean - np.nanmin(airmasses)),
f"SATURATE":
np.nanmean(np.float64(table[important_keys['saturate']])),
f"OBJECT":
template.extract_object(),
f"MJD-OBS":
float(np.nanmin(np.float64(table[important_keys["mjd-obs"]]))),
f"DATE-OBS":
template.extract_date_obs(),
f"EXPTIME":
np.nanmean(table[important_keys["exposure_time"]]),
f"FILTER":
template.extract_filter(),
f"INSTRUME":
template.instrument_name,
f"RON":
template.extract_noise_read().value,
f"BUNIT":
template.extract_unit(),
f"GAIA":
template.extract_header_item("GAIA")
}
frame_paths = list(
map(lambda f: os.path.join(input_directory, f), table["PATH"]))
n_frames = cls.count_exposures(frame_paths)
insert_dict["NCOMBINE"] = n_frames
if "OLD_EXPTIME" in table.colnames:
insert_dict["OLD_EXPTIME"] = np.nanmean(table["OLD_EXPTIME"])
insert_dict["INTTIME"] = insert_dict["OLD_EXPTIME"] * n_frames
if "OLD_GAIN" in table.colnames:
insert_dict["OLD_GAIN"] = np.nanmean(table["OLD_GAIN"])
if important_keys["filter"] in table.colnames:
insert_dict["FILTER"] = table[important_keys["filter"]][0]
gain_key = important_keys["gain"]
if gain_key.startswith("HIERARCH"):
gain_key = gain_key[9:]
if gain_key in table.colnames:
old_gain = np.nanmean(np.float64(table[gain_key]))
if coadd_type == "median":
new_gain = gain_median_combine(old_gain=old_gain,
n_frames=n_frames)
elif coadd_type == "mean":
new_gain = gain_mean_combine(old_gain=old_gain, n_frames=n_frames)
elif coadd_type == "sum":
new_gain = old_gain
else:
raise ValueError(f"Unrecognised coadd_type {coadd_type}")
insert_dict["GAIN"] = new_gain
colnames = table.colnames
for i, frame in enumerate(table):
for colname in colnames:
header_key = f"HIERARCH FRAME{i} {colname.upper()}"
val = frame[colname]
insert = True
if isinstance(val, str) and (
not val.isascii() or "\n" in val) or np.ma.is_masked(
val) or len(f"{header_key}={val}") > 70:
insert = False
if insert:
insert_dict[header_key] = val
if extra_items is not None:
insert_dict.update(extra_items)
with fits.open(file_path, mode="update", output_verify="fix") as file:
file[ext].header.update(insert_dict)
def main(output_dir: 'str', data_title: 'str'):
data_dir = output_dir + "/0-data_with_raw_calibs/"
# Write tables of fits files to main directory; firstly, science images only:
table = fits_table_all(input_path=data_dir,
output_path=output_dir + data_title +
"_fits_table_science.csv",
science_only=True)
# Then including all calibration files
table_full = fits_table_all(input_path=data_dir,
output_path=output_dir + data_title +
"_fits_table_all.csv",
science_only=False)
# Clear output files for fresh start.
u.rm_check(output_dir + '/output_values.yaml')
u.rm_check(output_dir + '/output_values.json')
# Collect list of filters used:
filters = []
for name in table['ESO INS FILT1 NAME']:
if name != 'free':
if name not in filters:
filters.append(name)
# Collect pointings of standard-star observations.
std_ras = []
std_decs = []
std_pointings = []
# TODO: This is a horrible way to do this. Take the time to find a better one.
for ra in table_full[table_full['OBJECT'] == 'STD']['CRVAL1']:
if ra not in std_ras:
std_ras.append(ra)
for dec in table_full[table_full['OBJECT'] == 'STD']['CRVAL1']:
if dec not in std_decs:
std_decs.append(dec)
for i, ra in enumerate(std_ras):
std_pointings.append(f'RA{ra}_DEC{std_decs[i]}')
# Collect and save some stats on those filters:
param_dict = {}
exp_times = []
ns_exposures = []
param_dict['filters'] = filters
for i, f in enumerate(filters):
f_0 = f[0]
exp_time = table['EXPTIME'][table['ESO INS FILT1 NAME'] == f]
exp_times.append(exp_time)
airmass = table['AIRMASS'][table['ESO INS FILT1 NAME'] == f]
n_frames = sum(table['ESO INS FILT1 NAME'] == f)
n_exposures = n_frames / 2
ns_exposures.append(n_exposures)
param_dict[f_0 + '_exp_time_mean'] = float(np.nanmean(exp_time))
param_dict[f_0 + '_exp_time_err'] = float(2 * np.nanstd(exp_time))
param_dict[f_0 + '_airmass_mean'] = float(np.nanmean(airmass))
param_dict[f_0 + '_airmass_err'] = float(
max(
np.nanmax(airmass) - np.nanmean(airmass),
(np.nanmean(airmass) - np.nanmin(airmass))))
param_dict[f_0 + '_n_frames'] = float(n_frames)
param_dict[f_0 + '_n_exposures'] = float(n_exposures)
std_filter_dir = f'{output_dir}calibration/std_star/{f}/'
u.mkdir_check(std_filter_dir)
print(f'Copying {f} calibration data to std_star folder...')
# Sort the STD files by filter, and within that by pointing.
for j, ra in enumerate(std_ras):
at_pointing = False
pointing = std_pointings[j]
pointing_dir = std_filter_dir + pointing + '/'
for file in \
table_full[
(table_full['OBJECT'] == 'STD') &
(table_full['CRVAL1'] == ra) &
(table_full['ESO INS FILT1 NAME'] == f)]['ARCFILE']:
at_pointing = True
u.mkdir_check(pointing_dir)
shutil.copyfile(data_dir + file, pointing_dir + file)
if at_pointing:
for file in table_full[table_full['object'] ==
'BIAS']['ARCFILE']:
shutil.copyfile(data_dir + file, pointing_dir + file)
for file in table_full[(table_full['object'] == 'FLAT,SKY') & (
table_full['ESO INS FILT1 NAME'] == f)]['ARCFILE']:
shutil.copyfile(data_dir + file, pointing_dir + file)
p.add_params(output_dir + '/output_values', param_dict)
def main(data_title: 'str'):
print("\nExecuting Python script pipeline_fors2/1-initial.py, with:")
print(f"\tepoch {data_title}")
print()
epoch_params = p.object_params_fors2(obj=data_title)
data_dir = epoch_params['data_dir']
output_dir = data_dir + "/0-data_with_raw_calibs/"
# Write tables of fits files to main directory; firstly, science images only:
table = fits_table(input_path=output_dir,
output_path=data_dir + data_title + "_fits_table_science.csv",
science_only=True)
# Then including all calibration files
table_full = fits_table(input_path=output_dir,
output_path=data_dir + data_title + "_fits_table_all.csv",
science_only=False)
fits_table_all(input_path=output_dir,
output_path=data_dir + data_title + "_fits_table_detailled.csv",
science_only=False)
# Clear output files for fresh start.
u.rm_check(data_dir + '/output_values.yaml')
u.rm_check(data_dir + '/output_values.json')
# Collect list of filters used:
filters = []
columns = []
for j in [1, 2, 3, 4, 5]:
column = 'filter' + str(j)
for name in table[column]:
if name != 'free':
if name not in filters:
filters.append(name)
columns.append(column)
# Collect pointings of standard-star observations.
std_ras = []
std_decs = []
std_pointings = []
# TODO: This is a horrible way to do this. Take the time to find a better one.
for ra in table_full[table_full['object'] == 'STD']['ref_ra']:
if ra not in std_ras:
std_ras.append(ra)
for dec in table_full[table_full['object'] == 'STD']['ref_dec']:
if dec not in std_decs:
std_decs.append(dec)
for i, ra in enumerate(std_ras):
std_pointings.append(f'RA{ra}_DEC{std_decs[i]}')
print(std_ras)
print(std_decs)
print(std_pointings)
# Collect and save some stats on those filters:
param_dict = {}
exp_times = []
ns_exposures = []
param_dict['filters'] = filters
param_dict['object'] = table['object'][0]
param_dict['obs_name'] = table['obs_name'][0]
mjd = param_dict['mjd_obs'] = float(table['mjd_obs'][0])
for i, f in enumerate(filters):
f_0 = f[0]
exp_time = table['exp_time'][table[columns[i]] == f]
exp_times.append(exp_time)
airmass_col = table['airmass'][table[columns[i]] == f]
n_frames = sum(table[columns[i]] == f)
n_exposures = n_frames / 2
ns_exposures.append(n_exposures)
airmass = float(np.nanmean(airmass_col))
param_dict[f_0 + '_exp_time_mean'] = float(np.nanmean(exp_time))
param_dict[f_0 + '_exp_time_err'] = float(2 * np.nanstd(exp_time))
param_dict[f_0 + '_airmass_mean'] = airmass
param_dict[f_0 + '_airmass_err'] = float(
max(np.nanmax(airmass_col) - airmass, airmass - np.nanmin(airmass_col)))
param_dict[f_0 + '_n_frames'] = float(n_frames)
param_dict[f_0 + '_n_exposures'] = float(n_exposures)
param_dict[f_0 + '_mjd_obs'] = float(np.nanmean(table['mjd_obs'][table[columns[i]] == f]))
std_filter_dir = f'{data_dir}calibration/std_star/{f}/'
u.mkdir_check(std_filter_dir)
print(f'Copying {f} calibration data to std_star folder...')
# Sort the STD files by filter, and within that by pointing.
for j, ra in enumerate(std_ras):
at_pointing = False
pointing = std_pointings[j]
pointing_dir = std_filter_dir + pointing + '/'
for file in \
table_full[
(table_full['object'] == 'STD') &
(table_full['ref_ra'] == ra) &
(table_full[columns[i]] == f)]['identifier']:
at_pointing = True
u.mkdir_check(pointing_dir)
shutil.copyfile(output_dir + file, pointing_dir + file)
if at_pointing:
for file in table_full[table_full['object'] == 'BIAS']['identifier']:
shutil.copyfile(output_dir + file, pointing_dir + file)
for file in table_full[(table_full['object'] == 'FLAT,SKY') & (table_full[columns[i]] == f)][
'identifier']:
shutil.copyfile(output_dir + file, pointing_dir + file)
p.add_output_values(obj=data_title, params=param_dict)
if "new_epoch" in data_dir:
mjd = f"MJD{int(float(mjd))}"
new_data_dir = data_dir.replace("new_epoch", mjd)
p.add_epoch_param(obj=data_title, params={"data_dir": new_data_dir})
def test_fits_table_all():
tbl = img.fits_table_all(good_input)
print(tbl.colnames)
assert isinstance(tbl, table.Table)
assert len(tbl) == 2
assert isinstance(tbl["EXPTIME"][0], float)