def image_table(input_directory: str, output_path: str = "images.tbl"):
"""
Executes the Montage task mImgtbl <input_directory> <output_path>
:param input_directory:
:param output_path:
:return:
"""
u.sanitise_file_ext(filename=output_path, ext=".tbl")
return u.system_command("mImgtbl", [input_directory, output_path])
def make_header(table_path: str, output_path: str):
"""
Executes Montage task mMakeHdr <table_path> <output_path>
:param table_path:
:param output_path:
:return:
"""
u.sanitise_file_ext(output_path, ".hdr")
return u.system_command("mMakeHdr", [table_path, output_path])
def overlaps(table_path: str, difference_table_path: str):
"""
Executes mOverlaps <table_path> <difference_table_path>
:param table_path:
:param difference_table_path:
:return:
"""
table_path = u.sanitise_file_ext(filename=table_path, ext=".tbl")
difference_table_path = u.sanitise_file_ext(filename=difference_table_path,
ext=".tbl")
return u.system_command(command="mOverlaps",
arguments=[table_path, difference_table_path])
def difference_execute(input_directory: str, difference_table_path: str,
header_path: str, diff_dir: str):
"""
Executes mDiffExec <difference_table_path> <header_file> <diff_dir> -p <input_directory>
:param input_directory:
:return:
"""
difference_table_path = u.sanitise_file_ext(filename=difference_table_path,
ext=".tbl")
header_path = u.sanitise_file_ext(filename=header_path, ext=".hdr")
return u.system_command(
command="mDiffExec",
arguments=[difference_table_path, header_path, diff_dir],
p=input_directory,
)
def fit_execute(difference_table_path: str, fit_table_path: str,
diff_dir: str):
"""
Executes mFitExec <difference_table_path> <fit_table_path> <diff_dir>
:param difference_table_path:
:param fit_table_path:
:param diff_dir:
:return:
"""
difference_table_path = u.sanitise_file_ext(filename=difference_table_path,
ext=".tbl")
fit_table_path = u.sanitise_file_ext(filename=fit_table_path, ext=".tbl")
return u.system_command(
command="mFitExec",
arguments=[difference_table_path, fit_table_path, diff_dir])
def background_model(table_path: str, fit_table_path: str,
correction_table_path: str):
"""
Executes mBGModel <table_path> <fit_table_path> <correction_table_path>
:param table_path:
:param fit_table_path:
:param correction_table_path:
:return:
"""
table_path = u.sanitise_file_ext(filename=table_path, ext=".tbl")
fit_table_path = u.sanitise_file_ext(filename=fit_table_path, ext=".tbl")
correction_table_path = u.sanitise_file_ext(filename=correction_table_path,
ext=".tbl")
return u.system_command(
command="mBgModel",
arguments=[table_path, fit_table_path, correction_table_path])
def fors_photometry(aligned_phot: List[str],
master_sky_flat_img: str,
output_dir: str,
output_filename: str = None,
chip_num: int = 1,
sof_name: str = "photometry.sof"):
sof_name = u.sanitise_file_ext(sof_name, ".sof")
old_dir = os.getcwd()
os.chdir(output_dir)
sof_path = os.path.join(output_dir, sof_name)
phot_table = select_phot_table(chip_num)
sof(
{
"MASTER_SKY_FLAT_IMG": [master_sky_flat_img],
"ALIGNED_PHOT": aligned_phot,
"PHOT_TABLE": [os.path.join(fors2_calib_path, phot_table)],
}, sof_path)
u.system_command_verbose(
f"{eso_bin_path} fors_photometry --fite=one {sof_path}")
master_path = os.path.join(output_dir, "phot_coeff_table.fits")
if output_filename is not None:
final_path = os.path.join(output_dir, output_filename)
shutil.move(master_path, final_path)
else:
final_path = master_path
os.chdir(old_dir)
return final_path
def save_params(file: str, dictionary: dict, quiet: bool = False):
file = u.sanitise_file_ext(filename=file, ext=".yaml")
if not quiet:
print('Saving parameter file to ' + str(file))
with open(file, 'w') as f:
yaml.dump(dictionary, f)
def save_params(file: str, dictionary: dict):
file = u.sanitise_file_ext(filename=file, ext=".yaml")
u.debug_print(1, 'Saving parameter file to ' + str(file))
u.debug_print(2, "params.save_params: dictionary ==", dictionary)
with open(file, 'w') as f:
yaml.dump(dictionary, f)
def background_execute(input_directory: str, table_path: str,
correction_table_path: str, corr_dir: str):
"""
Executes mBgExec <table_path> <corrections_table_path> <corr_dir> -p <input_directory>
:param input_directory:
:param table_path:
:param correction_table_path:
:param corr_dir:
:return:
"""
table_path = u.sanitise_file_ext(filename=table_path, ext=".tbl")
correction_table_path = u.sanitise_file_ext(filename=correction_table_path,
ext=".tbl")
return u.system_command(
command="mBgExec",
arguments=[table_path, correction_table_path, corr_dir],
p=input_directory)
def add_params(file: str, params: dict, quiet: bool = False):
file = u.sanitise_file_ext(file, '.yaml')
if os.path.isfile(file):
param_dict = load_params(file)
else:
param_dict = {}
param_dict.update(params)
save_params(file, param_dict, quiet=quiet)
yaml_to_json(file, quiet=quiet)
def project_execute(input_directory: str, table_path: str, header_path: str,
proj_dir: str, stats_table_path: str):
"""
Executes mProjExec <table_path> <header_path> <proj_dir> <stats_table_path> -p <input_directory>
:param input_directory:
:param table_path:
:param header_path:
:param proj_dir:
:param stats_table_path:
:return:
"""
table_path = u.sanitise_file_ext(filename=table_path, ext=".tbl")
header_path = u.sanitise_file_ext(filename=header_path, ext=".hdr")
stats_table_path = u.sanitise_file_ext(filename=stats_table_path,
ext=".tbl")
return u.system_command(
command="mProjExec",
arguments=[table_path, header_path, proj_dir, stats_table_path],
p=input_directory)
def yaml_to_json(yaml_file: str, output: str = None):
yaml_file = u.sanitise_file_ext(yaml_file, '.yaml')
if output is not None:
output = u.sanitise_file_ext(output, '.json')
elif output is None:
output = yaml_file.replace('.yaml', '.json')
p = load_params(file=yaml_file)
u.debug_print(1, 'Saving parameter file to ' + output)
for param in p:
if type(p[param]) is date:
p[param] = str(p[param])
with open(output, 'w') as fj:
json.dump(p, fj)
return p
def path_or_hdu(hdu: Union[fits.HDUList, str], update=False):
# TODO: Propagate this method to where it's needed.
path = None
if type(hdu) is str:
path = u.sanitise_file_ext(filename=hdu, ext=".fits")
if update:
hdu = fits.open(hdu, mode='update')
else:
hdu = fits.open(hdu)
return hdu, path
def add(input_directory: str,
table_path: str,
header_path: str,
output_path: str = "coadded.fits",
coadd_type: str = 'median'):
"""
Executes mAdd <table_path> <template_path> <output_path> -p <input_directory> -a <coadd_type>
:param input_directory:
:param table_path:
:param header_path:
:param output_path:
:param coadd_type:
:return:
"""
table_path = u.sanitise_file_ext(filename=table_path, ext=".tbl")
header_path = u.sanitise_file_ext(filename=header_path, ext=".hdr")
output_path = u.sanitise_file_ext(filename=output_path, ext=".fits")
return u.system_command(command="mAdd",
arguments=[table_path, header_path, output_path],
p=input_directory,
a=coadd_type)
def load_params(file: str):
file = u.sanitise_file_ext(file, '.yaml')
u.debug_print(2, 'Loading parameter file from ' + str(file))
if os.path.isfile(file):
with open(file) as f:
p = yaml.load(f)
else:
p = None
u.debug_print(1, 'No parameter file found at',
str(file) + ', returning None.')
return p
def params_init(param_file: Union[str, dict]):
if type(param_file) is str:
# Load params from .yaml at path.
param_file = u.sanitise_file_ext(filename=param_file, ext="yaml")
param_dict = load_params(file=param_file)
if param_dict is None:
return None, param_file, None # raise FileNotFoundError(f"No parameter file found at {param_file}.")
name = u.get_filename(path=param_file, include_ext=False)
param_dict["param_path"] = param_file
else:
param_dict = param_file
name = param_dict["name"]
param_file = param_dict["param_path"]
return name, param_file, param_dict
def load_params(file: str, quiet: bool = False):
file = u.sanitise_file_ext(file, '.yaml')
if not quiet:
print('Loading parameter file from ' + str(file))
if os.path.isfile(file):
with open(file) as f:
p = yaml.safe_load(f)
else:
p = None
if not quiet:
print('No parameter file found at',
str(file) + ', returning None.')
return p
def tabulate_output_values(path: str, output: str = None):
path = u.check_trailing_slash(path=path)
outputs = []
for file in filter(lambda filename: 'output_values.yaml' in filename,
os.listdir(path)):
output_values = load_params(file=path + file)
output_values["filename"] = file
outputs.append(output_values)
outputs = tbl.Table(outputs)
if output is not None:
output = u.sanitise_file_ext(filename=output, ext='.csv')
outputs.write(output)
outputs.sort(keys="filename")
return outputs
def fors_bias(bias_frames: List[str],
output_dir: str,
output_filename: str = None,
sof_name: str = "bias.sof"):
sof_name = u.sanitise_file_ext(sof_name, ".sof")
old_dir = os.getcwd()
os.chdir(output_dir)
sof_path = os.path.join(output_dir, sof_name)
sof({"BIAS": bias_frames}, sof_path)
u.system_command_verbose(f"{eso_bin_path} fors_bias {sof_path}")
master_path = os.path.join(output_dir, "master_bias.fits")
if output_filename is not None:
final_path = os.path.join(output_dir, output_filename)
shutil.move(master_path, final_path)
else:
final_path = master_path
os.chdir(old_dir)
return final_path
def trim_file(path: Union[str, fits.HDUList],
left: int = None,
right: int = None,
bottom: int = None,
top: int = None,
new_path: str = None):
"""
Trims the edges of a .fits file while retaining its WCS information.
:param path:
:param left:
:param right:
:param top:
:param bottom:
:return:
"""
file, path = path_or_hdu(path)
if new_path is not None:
new_path = u.sanitise_file_ext(filename=new_path, ext='.fits')
file = trim(hdu=file, left=left, right=right, bottom=bottom, top=top)
if new_path is None:
new_path = path.replace(".fits", "_trim.fits")
print('Trimming: \n' + str(path))
print('left', left, 'right', right, 'bottom', bottom, 'top', top)
print('Moving to: \n' + str(new_path))
add_log(
file=file,
action='Trimmed using PyCRAFT.fits_files.trim() with borders at x = ' +
str(left) + ', ' + str(right) + '; y=' + str(bottom) + ', ' +
str(top) + '; moved from ' + str(path) + ' to ' + str(new_path))
print()
print(new_path)
file.writeto(new_path, overwrite=True)
if path is not None:
file.close()
return new_path
else:
return file
def fors_img_sky_flat(flat_frames: List[str],
master_bias: str,
output_dir: str,
output_filename: str = None,
sof_name: str = "flat.sof"):
sof_name = u.sanitise_file_ext(sof_name, ".sof")
old_dir = os.getcwd()
os.chdir(output_dir)
sof_path = os.path.join(output_dir, sof_name)
sof({
"SKY_FLAT_IMG": flat_frames,
"MASTER_BIAS": [master_bias],
}, sof_path)
u.system_command_verbose(f"{eso_bin_path} fors_img_sky_flat {sof_path}")
master_path = os.path.join(output_dir, "master_sky_flat_img.fits")
if output_filename is not None:
final_path = os.path.join(output_dir, output_filename)
shutil.move(master_path, final_path)
else:
final_path = master_path
os.chdir(old_dir)
return final_path
def fors_zeropoint(
standard_img: str,
master_bias: str,
master_sky_flat_img: str,
output_dir: str,
output_filename: str = None,
chip_num: int = 1,
sof_name: str = "zeropoint.sof",
flux_std_imgs: List[str] = fors_flux_std_imgs,
):
sof_name = u.sanitise_file_ext(sof_name, ".sof")
old_dir = os.getcwd()
os.chdir(output_dir)
sof_path = os.path.join(output_dir, sof_name)
phot_table = select_phot_table(chip_num)
sof(
{
"STANDARD_IMG": [standard_img],
"MASTER_BIAS": [master_bias],
"MASTER_SKY_FLAT_IMG": [master_sky_flat_img],
"FLX_STD_IMG":
list(
map(lambda f: os.path.join(fors2_calib_path, f),
flux_std_imgs)),
"PHOT_TABLE": [os.path.join(fors2_calib_path, phot_table)],
}, sof_path)
u.system_command_verbose(f"{eso_bin_path} fors_zeropoint {sof_path}")
master_path = os.path.join(output_dir, "aligned_phot.fits")
std_path = os.path.join(output_dir, "standard_reduced_img.fits")
if output_filename is not None:
final_path = os.path.join(output_dir, output_filename)
shutil.move(master_path, final_path)
else:
final_path = master_path
os.chdir(old_dir)
return final_path, std_path
def refresh_params_folder(folder: str, template: str, quiet: bool = False):
template = u.sanitise_file_ext(template, '.yaml')
user_dir = f"{param_path}/{folder}/"
proj_dir = f"param/{folder}/"
# Get template file from within this project; use to update param files in param directory as specified in
# config.yaml
if not quiet:
print(f'Loading template from {proj_dir}/{template}')
template_params = load_params(proj_dir + template, quiet=quiet)
per_filter = False
if 'imacs' in folder:
imacs = True
else:
imacs = False
paths = [user_dir]
if user_dir != proj_dir:
paths.append(proj_dir)
for path in paths:
files = filter(lambda x: x[-5:] == '.yaml' and x != template,
os.listdir(path))
for file in files:
file_params = load_params(path + file, quiet=quiet)
if not quiet:
print(path + file)
if 'filters' in file_params:
per_filter = True
filter_trunc = []
filters = file_params['filters']
for f in filters:
if imacs:
filter_trunc.append(f[-1:] + '_')
else:
if len(f) > 1:
filter_trunc.append(f[:2])
else:
filter_trunc.append(f + '_')
# Use template to insert missing parameters.
for param in template_params:
# Apply filter-specific parameters to all filters listed in 'filters'.
if per_filter and param[:2] == 'f_':
for f in filter_trunc:
param_true = param.replace('f_', f, 1)
if param_true not in file_params:
file_params[param_true] = template_params[param]
elif param not in file_params:
file_params[param] = template_params[param]
# Use template to remove extraneous parameters.
new_file_params = {}
for param in file_params:
# Apply filter-specific parameters to all filters listed in 'filters'.
if per_filter and param[:2] in filter_trunc and param.replace(
param[:2], 'f_', 1) in template_params:
new_file_params[param] = file_params[param]
elif param in template_params and param[:2] != 'f_':
new_file_params[param] = file_params[param]
# Write to .yaml file
save_params(path + '/' + file, new_file_params, quiet=quiet)
# Convert to json
yaml_to_json(path + '/' + file, quiet=quiet)
def stack(files: list,
output: str = None,
directory: str = '',
stack_type: str = 'median',
inherit: bool = True,
show: bool = False,
normalise: bool = False):
accepted_stack_types = ['mean', 'median', 'add']
if stack_type not in accepted_stack_types:
raise ValueError('stack_type must be in ' + str(accepted_stack_types))
if directory != '' and directory[-1] != '/':
directory = directory + '/'
data = []
template = None
print('Stacking:')
for f in files:
# Extract image data and append to list.
if type(f) is str:
f = u.sanitise_file_ext(f, 'fits')
print(' ' + f)
f = fits.open(directory + f)
if type(f) is fits.hdu.hdulist.HDUList:
data_append = f[0].data
if template is None and inherit:
# Get a template file to use for output.
# TODO: Refine this - keep a record of which files went in in the header.
template = f.copy()
elif type(f) is CCDData:
data_append = f.data
else:
raise TypeError(
'files must contain only strings, HDUList or CCDData objects.')
if normalise:
data_append = data_append / np.nanmedian(
data_append[np.isfinite(data_append)])
if show:
norm = ImageNormalize(data_append,
interval=ZScaleInterval(),
stretch=SqrtStretch())
plt.imshow(data_append, origin='lower', norm=norm)
plt.show()
data.append(data_append)
data = np.array(data)
if stack_type == 'mean':
stacked = np.mean(data, axis=0)
elif stack_type == 'median':
stacked = np.median(data, axis=0)
else:
stacked = np.sum(data, axis=0)
if show:
norm = ImageNormalize(stacked,
interval=ZScaleInterval(),
stretch=SqrtStretch())
plt.imshow(stacked, origin='lower', norm=norm)
plt.show()
if inherit:
template[0].data = stacked
else:
template = fits.PrimaryHDU(stacked)
template = fits.HDUList([template])
add_log(template, f'Stacked.')
if output is not None:
print('Writing stacked image to', output)
template[0].header['BZERO'] = 0
template.writeto(output, overwrite=True, output_verify='warn')
return template
