def save_des_cutout(ra: float, dec: float, output: str):
"""
:param ra:
:param dec:
:return:
"""
print(f"Requesting cutout images from DES DR2 archive for field centring on RA={ra}, DEC={dec}")
login_des()
u.mkdir_check_nested(output)
response = submit_cutout_job_des(ra=ra, dec=dec)
if response['status'] == 'ok':
# Store the unique job ID for the new job
job_id = response['jobid']
print(f'New job submitted: {job_id}')
response = job_status_poll_des(job_id)
job_id = response['jobs'][0]['job_id']
url = f'{des_files_url}/{keys["des_user"]}/cutout/{job_id}'
r = requests.get(f'{url}/json')
for item in r.json():
if item['type'] == "directory":
sub_url = f'{url}/{item["name"]}/'
r_2 = requests.get(f'{sub_url}/json').json()
sub_sub_url = f'{sub_url}/{r_2[0]["name"]}'
r_3 = requests.get(f'{sub_sub_url}/json').json()
for f in r_3:
if f["name"][-5:] == ".fits":
data = requests.get(f'{sub_sub_url}/{f["name"]}')
new_title = f["name"][-6] + "_cutout.fits"
with open(f'{output}/{new_title}', "wb") as file:
file.write(data.content)
return True
else:
print(f"No cutouts could be retrieved for field {ra}, {dec}")
return False
def main(comb_path, output_dir, obj, sextractor_path, path_suffix):
print("\nExecuting Python script pipeline_fors2/7-trim_combined.py, with:")
print(f"\tepoch {obj}")
print(f"\toutput directory {output_dir}")
print(f"\tsextractor directory {sextractor_path}")
print()
if sextractor_path is not None:
u.mkdir_check_nested(sextractor_path)
do_sextractor = True
else:
do_sextractor = False
# Build a list of the filter prefixes used.
fils = []
files = list(filter(lambda x: x[-4:] == '.tbl', os.listdir(comb_path)))
for file in files:
if file[0] not in fils:
fils.append(str(file[0]))
for fil in fils:
if do_sextractor:
u.mkdir_check(sextractor_path)
area_file = fil + "_coadded_area.fits"
comb_file = fil + "_coadded.fits"
left, right, bottom, top = f.detect_edges_area(comb_path + area_file)
# Trim a little extra to be safe.
left = left + 5
right = right - 5
top = top - 5
bottom = bottom + 5
f.trim_file(comb_path + comb_file,
left=left,
right=right,
top=top,
bottom=bottom,
new_path=output_dir + "/" + comb_file)
# Keep a trimmed version of the area file, it comes in handy later.
f.trim_file(comb_path + area_file,
left=left,
right=right,
top=top,
bottom=bottom,
new_path=output_dir + "/" + area_file)
if do_sextractor:
copyfile(output_dir + "/" + comb_file, sextractor_path + comb_file)
if path_suffix is None:
path_suffix = output_dir.split("/")[-2]
path_suffix = u.remove_trailing_slash(path_suffix)
p.add_output_path(obj=obj,
instrument='fors2',
key=fil[0] + '_trimmed_image' + path_suffix,
path=output_dir + "/" + comb_file)
def save_skymapper_photometry(ra: float, dec: float, output: str):
response = retrieve_skymapper_photometry(ra=ra, dec=dec)
if response is not None:
u.mkdir_check_nested(path=output)
print("Saving SkyMapper photometry to" + output)
with open(output, "wb") as file:
file.write(response)
else:
print('No data retrieved from SkyMapper.')
return response
def sof(frames: Dict[str, list], output_path: str):
output_lines = []
for frame_type in frames:
u.debug_print(0, output_lines)
for frame in frames[frame_type]:
output_lines.append(f"{frame} {frame_type}\n")
u.mkdir_check_nested(output_path)
print(f"Writing SOF file to {output_path}")
with open(output_path, "w") as file:
file.writelines(output_lines)
return output_lines
def __init__(self, **kwargs):
self.name = None
if "name" in kwargs:
self.name = kwargs["name"]
self.formatted_name = None
if "formatted_name" in kwargs:
self.formatted_name = kwargs["formatted_name"]
self.band_name = None
if "band_name" in kwargs:
self.band_name = kwargs["band_name"]
elif self.name is not None:
self.band_name = self.name[0]
self.svo_id = None
self.svo_instrument = None
if "svo_service" in kwargs:
svo = kwargs["svo_service"]
if "id" in svo:
self.svo_id = svo["id"]
if "instrument" in svo:
self.svo_instrument = svo["instrument"]
self.data_path = None
if "data_path" in kwargs:
self.data_path = kwargs["data_path"]
u.mkdir_check_nested(self.data_path)
self.votable = None
self.votable_path = None
self.instrument = None
if "instrument" in kwargs:
self.instrument = kwargs["instrument"]
self.lambda_eff = None
self.lambda_fwhm = None
self.transmission_table_filter = None
self.transmission_table_filter_path = None
self.transmission_table_filter_instrument = None
self.transmission_table_filter_instrument_path = None
self.transmission_table_filter_instrument_atmosphere = None
self.transmission_table_filter_instrument_atmosphere_path = None
self.transmission_table_filter_atmosphere = None
self.transmission_table_filter_atmosphere_path = None
self.photometry_table = None
self.load_output_file()
active_filters[f"{self.instrument}_{self.name}"] = self
def save_sdss_photometry(ra: float, dec: float, output: str):
"""
Retrieves and writes to disk the SDSS photometry for a given field, in a 0.2 x 0.2 degree box
centred on the field coordinates. (Note - the width of the box is in RA degrees, not corrected for spherical
distortion)
:param ra: Right Ascension of the centre of the desired field, in degrees.
:param dec: Declination of the centre of the desired field, in degrees.
:param output: The location on disk to which to write the file.
:return: Retrieved photometry table, as a pandas dataframe, if successful; if not, None.
"""
df = retrieve_sdss_photometry(ra=ra, dec=dec)
if df is not None:
u.mkdir_check_nested(path=output)
print("Saving SDSS photometry to" + output)
df.to_csv(output)
else:
print("No data retrieved from SDSS.")
return df
def save_des_photometry(ra: float, dec: float, output: str):
"""
Retrieves and writes to disk the DES photometry for a given field, in a 0.2 x 0.2 degree box
centred on the field coordinates. (Note - the width of the box is in RA degrees, not corrected for spherical
distortion)
:param ra: Right Ascension of the centre of the desired field, in degrees.
:param dec: Declination of the centre of the desired field, in degrees.
:param output: The location on disk to which to write the file.
:return: Retrieved photometry table, as a Bytes object, if successful; None if not.
"""
data = retrieve_des_photometry(ra=ra, dec=dec)
if data is not None:
u.mkdir_check_nested(path=output)
print("Saving DES photometry to" + output)
with open(output, "wb") as file:
file.write(data)
else:
print('No data retrieved from DES.')
return data
def test_mkdir_check_nested():
u.debug_level = 2
path_test = os.path.join(test_file_path, "path_test", "nested", "and_then")
u.rm_check(os.path.join(test_file_path, "path_test"))
u.mkdir_check_nested(path_test, remove_last=False)
assert os.path.isdir(path_test)
u.rmtree_check(os.path.join(test_file_path, "path_test"))
path_test = os.path.join(test_file_path, "path_test", "nested", "and_then", "/")
u.mkdir_check_nested(path_test)
assert os.path.isdir(path_test)
u.rmtree_check(os.path.join(test_file_path, "path_test"))
path_test = os.path.join(test_file_path, "path_test", "nested", "and_then", "gaia.csv")
u.mkdir_check_nested(path_test)
assert os.path.isdir(os.path.split(path_test)[0])
assert not os.path.isfile(path_test)
u.rmtree_check(os.path.join(test_file_path, "path_test"))
def main(data_dir, data_title, origin, destination, all_synths):
print("\nExecuting Python script pipeline_fors2/5-background_subtract.py, with:")
print(f"\tepoch {data_title}")
print(f"\torigin directory {origin}")
print(f"\tdestination directory {destination}")
print()
methods = ["ESO backgrounds only", "SExtractor backgrounds only", "polynomial fit", "Gaussian fit", "median value"]
if all_synths:
frame = 56
method = "polynomial fit"
degree = 5
do_mask = True
local = True
global_sub = False
trim_image = False
recorrect_subbed = True
eso_back = False
else:
frame = 200
# frame_arcsec = 30 * units.arcsec
# frame_deg = frame_arcsec.to(units.deg)
eso_back = False
_, method = u.select_option(message="Please select the background subtraction method.", options=methods,
default="polynomial fit")
degree = None
if method == "polynomial fit":
degree = u.user_input(message=f"Please enter the degree of {method} to use:", typ=int, default=3)
elif method == "ESO backgrounds only":
eso_back = True
do_mask = False
if method not in ["ESO backgrounds only", "SExtractor backgrounds only", "median value"]:
do_mask = u.select_yn(message="Mask sources using SExtractor catalogue?", default=True)
if method in ["polynomial fit", "Gaussian fit"]:
local = u.select_yn(message="Use a local fit?", default=True)
else:
local = False
global_sub = False
trim_image = False
recorrect_subbed = False
if local:
global_sub = u.select_yn(message="Subtract local fit from entire image?", default="n")
if not global_sub:
trim_image = u.select_yn(message="Trim images to subtracted region?", default="y")
recorrect_subbed = u.select_yn(message="Re-normalise background of subtracted region?", default="y")
# if not eso_back and method != "SExtractor backgrounds only":
# eso_back = u.select_yn(message="Subtract ESO Reflex fitted backgrounds first?", default=False)
outputs = p.object_output_params(data_title, instrument='FORS2')
data_dir = u.check_trailing_slash(data_dir)
destination = u.check_trailing_slash(destination)
destination = data_dir + destination
u.mkdir_check_nested(destination)
origin = u.check_trailing_slash(origin)
science_origin = data_dir + origin + "science/"
print(science_origin)
filters = outputs['filters']
frb_params = p.object_params_frb(obj=data_title[:-2])
epoch_params = p.object_params_fors2(obj=data_title)
background_origin_eso = ""
if eso_back:
background_origin_eso = data_dir + "/" + origin + "/backgrounds/"
if method == "SExtractor backgrounds only":
background_origin = f"{data_dir}{origin}backgrounds_sextractor/"
elif method == "polynomial fit":
background_origin = f"{destination}backgrounds/" # f"{destination}backgrounds_{method.replace(' ', '')}_degree_{degree}_local_{local}_globalsub_{global_sub}/"
else:
background_origin = f"{destination}backgrounds/" # f"{destination}backgrounds_{method.replace(' ', '')}_local_{local}_globalsub_{global_sub}/"
trimmed_path = ""
if trim_image:
trimmed_path = f"{data_dir}{origin}trimmed_to_background/"
u.mkdir_check_nested(trimmed_path)
ra = frb_params["burst_ra"]
dec = frb_params["burst_dec"]
if all_synths:
ras = epoch_params["test_synths"]["ra"]
decs = epoch_params["test_synths"]["dec"]
else:
ras = [ra]
decs = [dec]
for fil in filters:
trimmed_path_fil = ""
if trim_image:
trimmed_path_fil = f"{trimmed_path}{fil}/"
u.mkdir_check(trimmed_path_fil)
background_fil_dir = f"{background_origin}{fil}/"
u.mkdir_check_nested(background_fil_dir)
science_destination_fil = f"{destination}science/{fil}/"
u.mkdir_check_nested(science_destination_fil)
files = os.listdir(science_origin + fil + "/")
for file_name in files:
if file_name.endswith('.fits'):
new_file = file_name.replace("norm", "bg_sub")
new_path = f"{science_destination_fil}/{new_file}"
print("NEW_PATH:", new_path)
science = science_origin + fil + "/" + file_name
# First subtract ESO Reflex background images
# frame = (frame_deg / f.get_pixel_scale(file=science, astropy_units=True)[1]).to(f.pix).value
if eso_back:
background_eso = background_origin_eso + fil + "/" + file_name.replace("SCIENCE_REDUCED",
"PHOT_BACKGROUND_SCI")
ff.subtract_file(file=science, sub_file=background_eso, output=new_path)
science_image = new_path
if method != "ESO backgrounds only":
print(ra, dec)
print("Science image:", science)
science_image = fits.open(science)
print("Science file:", science_image)
wcs_this = WCS(header=science_image[0].header)
if method == "SExtractor backgrounds only":
background = background_origin + fil + "/" + file_name + "_back.fits"
print("Background image:", background)
else:
if method == "median value":
print(science_image[0].data.shape)
_, background_value, _ = sigma_clipped_stats(science_image[0].data)
background = deepcopy(science_image)
background[0].data = np.full(shape=science_image[0].data.shape, fill_value=background_value)
background_path = background_origin + fil + "/" + file_name.replace("SCIENCE_REDUCED",
"PHOT_BACKGROUND_MEDIAN")
# Next do background fitting.
else:
background = deepcopy(science_image)
background[0].data = np.zeros(background[0].data.shape)
background_path = background_origin + fil + "/" + file_name.replace("SCIENCE_REDUCED",
"PHOT_BACKGROUND_FITTED")
for i, ra in enumerate(ras):
dec = decs[i]
x, y = wcs_this.all_world2pix(ra, dec, 0)
print(x, y)
bottom, top, left, right = ff.subimage_edges(data=science_image[0].data, x=x, y=y,
frame=frame)
if do_mask:
# Produce a pixel mask that roughly masks out the true sources in the image so that
# they don't get fitted.
mask_max = 10
_, pixel_scale = ff.get_pixel_scale(science_image)
sextractor = Table.read(
f"{data_dir}analysis/sextractor/4-divided_by_exp_time/{fil}/{file_name.replace('.fits', '_psf-fit.cat')}",
format='ascii.sextractor')
weights = np.ones(shape=science_image[0].data.shape)
for obj in filter(
lambda o: left < o["X_IMAGE"] < right and bottom < o["Y_IMAGE"] < top,
sextractor):
mask_rad = min(int(obj["A_WORLD"] * obj["KRON_RADIUS"] / pixel_scale), mask_max)
x_prime = int(np.round(obj["X_IMAGE"]))
y_prime = int(np.round(obj["Y_IMAGE"]))
weights[y_prime - mask_rad:y_prime + mask_rad,
x_prime - mask_rad:x_prime + mask_rad] = 0.0
plt.imshow(weights, origin="lower")
plt.savefig(
background_origin + fil + "/" + file_name.replace("norm.fits", "mask.png"))
else:
weights = None
background_this = fit_background_fits(image=science_image,
model_type=method[:method.find(" ")],
deg=degree, local=local,
global_sub=global_sub,
centre_x=x, centre_y=y, frame=frame,
weights=weights)
background[0].data += background_this[0].data
if recorrect_subbed:
offset = get_median_background(image=science,
ra=epoch_params["renormalise_centre_ra"],
dec=epoch_params["renormalise_centre_dec"], frame=50,
show=False,
output=new_path[
:new_path.find("bg_sub")] + "renorm_patch_")
print("RECORRECT_SUBBED:", recorrect_subbed)
print("SUBTRACTING FROM BACKGROUND:", offset)
print(bottom, top, left, right)
print(background[0].data[bottom:top, left:right].shape)
print(np.median(background[0].data[bottom:top, left:right]))
background[0].data[bottom:top, left:right] -= offset
print(np.median(background[0].data[bottom:top, left:right]))
if trim_image:
print("TRIMMED_PATH_FIL:", trimmed_path_fil)
science_image = ff.trim_file(path=science_image, left=left, right=right, top=top,
bottom=bottom,
new_path=trimmed_path_fil + file_name.replace(
"norm.fits",
"trimmed_to_back.fits"))
print("Science after trim:", science_image)
background = ff.trim_file(path=background, left=left, right=right, top=top,
bottom=bottom,
new_path=background_path)
print("Writing background to:")
print(background_path)
background.writeto(background_path, overwrite=True)
print("SCIENCE:", science_image)
print("BACKGROUND:", background)
subbed = ff.subtract_file(file=science_image, sub_file=background, output=new_path)
# # TODO: check if regions overlap
#
# plt.hist(subbed[0].data[int(y - frame + 1):int(y + frame - 1),
# int(x - frame + 1):int(x + frame - 1)].flatten(),
# bins=10)
# plt.savefig(new_path[:new_path.find("bg_sub")] + "histplot.png")
# plt.close()
copyfile(data_dir + "/" + origin + "/" + data_title + ".log", destination + data_title + ".log")
u.write_log(path=destination + data_title + ".log",
action=f'Backgrounds subtracted using 4-background_subtract.py with method {method}\n')
import os
from typing import Union
import astropy.table as table
import astropy.units as units
import craftutils.params as p
import craftutils.utils as u
config = p.config
u.mkdir_check_nested(config["table_dir"])
def _construct_column_lists(columns: dict):
dtypes = []
un = []
colnames = []
default_data = []
columns_revised = {}
for colname in columns:
# if "{:s}" in colname:
# for fil in filters:
# colname_fil = colname.format(fil)
# colnames.append(colname_fil)
# columns_revised[colname_fil] = columns[colname]
if "{:s}" not in colname:
colnames.append(colname)
columns_revised[colname] = columns[colname]
packages = setuptools.find_packages()
with open("README.md", "r", encoding="utf-8") as fh:
long_description = fh.read()
setuptools.setup(
name="craftutils",
version="0.9.0",
author="Lachlan Marnoch",
# short_description=long_description,
long_description=long_description,
url="https://github.com/Lachimax/craft-optical-followup",
packages=setuptools.find_packages(),
python_requires='>=3.5',
install_requires=["astropy", "matplotlib", "requests", "numpy", "PyYAML"],
license='Attribution-NonCommercial-ShareAlike 4.0 International')
param_path = os.path.join(os.getcwd(), "param")
import craftutils.params as p
import craftutils.utils as u
data_dir = p.config["top_data_dir"]
u.mkdir_check_nested(data_dir)
u.mkdir_check(os.path.join(param_path, "fields"))
u.mkdir_check(os.path.join(param_path), "instruments")
key_path = os.path.join(p.config["param_dir"], 'keys.json')
if not os.path.isfile(key_path):
copy(os.path.join(param_path, "keys.json"), key_path)
def _build_param_dir(cls, survey_name: str):
path = cls._build_survey_dir()
path = os.path.join(path, survey_name)
u.mkdir_check_nested(path, remove_last=False)
return path