def main(data_title, sextractor_path, origin, destination): properties = p.object_params_imacs(data_title) data_dir = properties['data_dir'] if sextractor_path is not None: if not os.path.isdir(sextractor_path): os.mkdir(sextractor_path) do_sextractor = True ap_diams_sex = p.load_params(f'param/aperture_diameters_fors2') else: do_sextractor = False origin_path = data_dir + origin destination_path = data_dir + destination u.mkdir_check(destination_path) filters = next(os.walk(origin_path))[1] for fil in filters: u.mkdir_check(destination_path + fil) if do_sextractor: if not os.path.isdir(sextractor_path + fil): os.mkdir(sextractor_path + fil) files = os.listdir(origin_path + fil + "/") for file_name in files: if file_name[-5:] == '.fits': science_origin = origin_path + fil + "/" + file_name science_destination = destination_path + fil + "/" + file_name print(science_origin) # Divide by exposure time to get an image in counts/second. f.divide_by_exp_time(file=science_origin, output=science_destination) if do_sextractor: copyfile(science_origin, sextractor_path + fil + "/" + file_name) # Write a sextractor file for photometric testing of the data from the upper chip. if do_sextractor: # Write a csv table of file properties to each filter directory. tbl = f.fits_table(input_path=sextractor_path + fil, output_path=sextractor_path + fil + "/" + fil + "_fits_tbl.csv", science_only=False) # TODO: Rewrite to use psf-fitting (in FORS2 pipeline as well) for i, d in enumerate(ap_diams_sex): f.write_sextractor_script(table=tbl, output_path=sextractor_path + fil + "/sextract_aperture_" + str(d) + ".sh", sex_params=['c', 'PHOT_APERTURES'], sex_param_values=['im.sex', str(d)], cat_name='sextracted_' + str(d), cats_dir='aperture_' + str(d), criterion='chip', value='CHIP1') if os.path.isfile(origin_path + data_title + '.log'): copyfile(origin_path + data_title + '.log', destination_path + data_title + ".log") u.write_log(path=destination_path + data_title + ".log", action=f'Astrometry solved using 3-astrometry.py')
def main(data_title: str, origin: str, destination: str, redo: bool = False): properties = p.object_params_imacs(data_title) path = properties['data_dir'] origin_path = path + origin astrometry_path = path + destination u.mkdir_check(astrometry_path) keys = params.load_params('param/keys') key = keys['astrometry'] reduced_list = os.listdir(origin_path) astrometry_list = os.listdir(astrometry_path) if redo: to_send = list(filter(lambda f: f[-5:] == '.fits', reduced_list)) else: to_send = list( filter(lambda f: f[-5:] == '.fits' and f not in astrometry_list, reduced_list)) filters = list(filter(lambda f: os.path.isdir(f), os.listdir(origin))) for f in filters: reduced_path_filter = origin_path + f + '/' astrometry_path_filter = astrometry_path + f + '/' print(f'To send to Astrometry.net from {f}:') for file in to_send: print('\t' + file) for file in to_send: hdu = fits.open(origin_path + file) header = hdu[0].header ra = header['RA-D'] dec = header['DEC-D'] scale_upper = header['SCALE'] + 0.1 scale_lower = header['SCALE'] - 0.1 hdu.close() print('Sending to Astrometry.net:', file) os.system(f'python /astrometry-client.py ' f'--apikey {key} ' f'-u {reduced_path_filter}{file} ' f'-w ' f'--newfits {astrometry_path_filter}{file} ' f'--ra {ra} --dec {dec} --radius {1.} ' f'--scale-upper {scale_upper} ' f'--scale-lower {scale_lower} ' f'--private --no_commercial') if os.path.isfile(origin_path + data_title + '.log'): shutil.copy(origin_path + data_title + '.log', astrometry_path + data_title + ".log") u.write_log(path=astrometry_path + data_title + ".log", action=f'Astrometry solved using 3-astrometry.py')
def main(data_title, origin, destination): properties = p.object_params_xshooter(data_title) path = properties['data_dir'] destination = path + destination u.mkdir_check(destination) origin = path + origin u.mkdir_check(origin) dirs = next(os.walk(origin))[1] left = 27 right = 537 top = 526 bottom = 15 for fil in dirs: u.mkdir_check(destination + fil) print('HERE:') print(origin + fil) files = os.listdir(origin + fil) files.sort() for i, file in enumerate(files): # Split the files into upper CCD and lower CCD, with even-numbered being upper and odd-numbered being lower new_path = destination + fil + "/" + file.replace( ".fits", "_trim.fits") # Add GAIN and SATURATE keywords to headers. path = origin + fil + "/" + file f.trim_file(path, left=left, right=right, top=top, bottom=bottom, new_path=new_path) copyfile(origin + data_title + ".log", destination + data_title + ".log") u.write_log(path=destination + data_title + ".log", action='Edges trimmed using 4-trim.py\n')
def main(data_title, sextractor_path, origin, destination): properties = p.object_params_xshooter(data_title) data_dir = properties['data_dir'] if sextractor_path is not None: if not os.path.isdir(sextractor_path): os.mkdir(sextractor_path) do_sextractor = True else: do_sextractor = False origin_path = data_dir + origin destination_path = data_dir + destination u.mkdir_check(destination_path) filters = next(os.walk(origin_path))[1] for fil in filters: u.mkdir_check(destination_path + fil) if do_sextractor: if not os.path.isdir(sextractor_path + fil): os.mkdir(sextractor_path + fil) files = os.listdir(origin_path + fil + "/") for file_name in files: if file_name[-5:] == '.fits': science_origin = origin_path + fil + "/" + file_name science_destination = destination_path + fil + "/" + file_name print(science_origin) # Divide by exposure time to get an image in counts/second. f.divide_by_exp_time(file=science_origin, output=science_destination) if do_sextractor: copyfile(science_origin, sextractor_path + fil + "/" + file_name) if os.path.isfile(origin_path + data_title + '.log'): copyfile(origin_path + data_title + '.log', destination_path + data_title + ".log") u.write_log(path=destination_path + data_title + ".log", action=f'Divided by exposure time.')
def main(epoch, origin, destination): print("\nExecuting Python script pipeline_fors2/4.1-insert_test_synth.py, with:") print(f"\tepoch {epoch}") print(f"\torigin directory {origin}") print(f"\tdestination directory {destination}") print() epoch_params = p.object_params_fors2(obj=epoch) outputs = p.object_output_params(obj=epoch, instrument='FORS2') data_dir = epoch_params['data_dir'] insert = epoch_params['test_synths'] origin_path = data_dir + "analysis/sextractor/" + origin destination_path = data_dir + destination u.mkdir_check(destination_path) u.mkdir_check(destination_path + "science/") u.mkdir_check(destination_path + "backgrounds/") filters = outputs['filters'] for fil in filters: f = fil[0] path_fil_output = destination_path + "science/" + fil + "/" path_fil_input = origin_path + fil + "/" u.mkdir_check(path_fil_output) u.mkdir_check(destination_path + "backgrounds/" + fil) zeropoint, _, airmass, _, extinction, _ = ph.select_zeropoint(obj=epoch, filt=fil, instrument='fors2', outputs=outputs) print(path_fil_input) # print(os.listdir(path_fil_input)) for fits_file in filter(lambda f: f.endswith("_norm.fits"), os.listdir(path_fil_input)): print(fits_file) path_fits_file_input = path_fil_input + fits_file path_fits_file_output = path_fil_output + fits_file path_psf_model = path_fits_file_input.replace(".fits", "_psfex.psf") try: ph.insert_point_sources_to_file(file=path_fits_file_input, x=array(insert["ra"]), y=array(insert["dec"]), mag=insert[f"{f}_mag"], output=path_fits_file_output, zeropoint=zeropoint, extinction=extinction, airmass=airmass, world_coordinates=True, psf_model=path_psf_model ) except ValueError: ph.insert_point_sources_to_file(file=path_fits_file_input, x=array(insert["ra"]), y=array(insert["dec"]), mag=insert[f"{f}_mag"], output=path_fits_file_output, zeropoint=zeropoint, extinction=extinction, airmass=airmass, world_coordinates=True, fwhm=fits.open(path_psf_model)[1].header['PSF_FWHM'] ) if os.path.isfile(origin_path + epoch + '.log'): copyfile(origin_path + epoch + '.log', destination_path + epoch + ".log") u.write_log(path=destination_path + epoch + ".log", action=f'Divided by exposure time.')
def main(data_dir, data_title, origin, destination, all_synths): print("\nExecuting Python script pipeline_fors2/5-background_subtract.py, with:") print(f"\tepoch {data_title}") print(f"\torigin directory {origin}") print(f"\tdestination directory {destination}") print() methods = ["ESO backgrounds only", "SExtractor backgrounds only", "polynomial fit", "Gaussian fit", "median value"] if all_synths: frame = 56 method = "polynomial fit" degree = 5 do_mask = True local = True global_sub = False trim_image = False recorrect_subbed = True eso_back = False else: frame = 200 # frame_arcsec = 30 * units.arcsec # frame_deg = frame_arcsec.to(units.deg) eso_back = False _, method = u.select_option(message="Please select the background subtraction method.", options=methods, default="polynomial fit") degree = None if method == "polynomial fit": degree = u.user_input(message=f"Please enter the degree of {method} to use:", typ=int, default=3) elif method == "ESO backgrounds only": eso_back = True do_mask = False if method not in ["ESO backgrounds only", "SExtractor backgrounds only", "median value"]: do_mask = u.select_yn(message="Mask sources using SExtractor catalogue?", default=True) if method in ["polynomial fit", "Gaussian fit"]: local = u.select_yn(message="Use a local fit?", default=True) else: local = False global_sub = False trim_image = False recorrect_subbed = False if local: global_sub = u.select_yn(message="Subtract local fit from entire image?", default="n") if not global_sub: trim_image = u.select_yn(message="Trim images to subtracted region?", default="y") recorrect_subbed = u.select_yn(message="Re-normalise background of subtracted region?", default="y") # if not eso_back and method != "SExtractor backgrounds only": # eso_back = u.select_yn(message="Subtract ESO Reflex fitted backgrounds first?", default=False) outputs = p.object_output_params(data_title, instrument='FORS2') data_dir = u.check_trailing_slash(data_dir) destination = u.check_trailing_slash(destination) destination = data_dir + destination u.mkdir_check_nested(destination) origin = u.check_trailing_slash(origin) science_origin = data_dir + origin + "science/" print(science_origin) filters = outputs['filters'] frb_params = p.object_params_frb(obj=data_title[:-2]) epoch_params = p.object_params_fors2(obj=data_title) background_origin_eso = "" if eso_back: background_origin_eso = data_dir + "/" + origin + "/backgrounds/" if method == "SExtractor backgrounds only": background_origin = f"{data_dir}{origin}backgrounds_sextractor/" elif method == "polynomial fit": background_origin = f"{destination}backgrounds/" # f"{destination}backgrounds_{method.replace(' ', '')}_degree_{degree}_local_{local}_globalsub_{global_sub}/" else: background_origin = f"{destination}backgrounds/" # f"{destination}backgrounds_{method.replace(' ', '')}_local_{local}_globalsub_{global_sub}/" trimmed_path = "" if trim_image: trimmed_path = f"{data_dir}{origin}trimmed_to_background/" u.mkdir_check_nested(trimmed_path) ra = frb_params["burst_ra"] dec = frb_params["burst_dec"] if all_synths: ras = epoch_params["test_synths"]["ra"] decs = epoch_params["test_synths"]["dec"] else: ras = [ra] decs = [dec] for fil in filters: trimmed_path_fil = "" if trim_image: trimmed_path_fil = f"{trimmed_path}{fil}/" u.mkdir_check(trimmed_path_fil) background_fil_dir = f"{background_origin}{fil}/" u.mkdir_check_nested(background_fil_dir) science_destination_fil = f"{destination}science/{fil}/" u.mkdir_check_nested(science_destination_fil) files = os.listdir(science_origin + fil + "/") for file_name in files: if file_name.endswith('.fits'): new_file = file_name.replace("norm", "bg_sub") new_path = f"{science_destination_fil}/{new_file}" print("NEW_PATH:", new_path) science = science_origin + fil + "/" + file_name # First subtract ESO Reflex background images # frame = (frame_deg / f.get_pixel_scale(file=science, astropy_units=True)[1]).to(f.pix).value if eso_back: background_eso = background_origin_eso + fil + "/" + file_name.replace("SCIENCE_REDUCED", "PHOT_BACKGROUND_SCI") ff.subtract_file(file=science, sub_file=background_eso, output=new_path) science_image = new_path if method != "ESO backgrounds only": print(ra, dec) print("Science image:", science) science_image = fits.open(science) print("Science file:", science_image) wcs_this = WCS(header=science_image[0].header) if method == "SExtractor backgrounds only": background = background_origin + fil + "/" + file_name + "_back.fits" print("Background image:", background) else: if method == "median value": print(science_image[0].data.shape) _, background_value, _ = sigma_clipped_stats(science_image[0].data) background = deepcopy(science_image) background[0].data = np.full(shape=science_image[0].data.shape, fill_value=background_value) background_path = background_origin + fil + "/" + file_name.replace("SCIENCE_REDUCED", "PHOT_BACKGROUND_MEDIAN") # Next do background fitting. else: background = deepcopy(science_image) background[0].data = np.zeros(background[0].data.shape) background_path = background_origin + fil + "/" + file_name.replace("SCIENCE_REDUCED", "PHOT_BACKGROUND_FITTED") for i, ra in enumerate(ras): dec = decs[i] x, y = wcs_this.all_world2pix(ra, dec, 0) print(x, y) bottom, top, left, right = ff.subimage_edges(data=science_image[0].data, x=x, y=y, frame=frame) if do_mask: # Produce a pixel mask that roughly masks out the true sources in the image so that # they don't get fitted. mask_max = 10 _, pixel_scale = ff.get_pixel_scale(science_image) sextractor = Table.read( f"{data_dir}analysis/sextractor/4-divided_by_exp_time/{fil}/{file_name.replace('.fits', '_psf-fit.cat')}", format='ascii.sextractor') weights = np.ones(shape=science_image[0].data.shape) for obj in filter( lambda o: left < o["X_IMAGE"] < right and bottom < o["Y_IMAGE"] < top, sextractor): mask_rad = min(int(obj["A_WORLD"] * obj["KRON_RADIUS"] / pixel_scale), mask_max) x_prime = int(np.round(obj["X_IMAGE"])) y_prime = int(np.round(obj["Y_IMAGE"])) weights[y_prime - mask_rad:y_prime + mask_rad, x_prime - mask_rad:x_prime + mask_rad] = 0.0 plt.imshow(weights, origin="lower") plt.savefig( background_origin + fil + "/" + file_name.replace("norm.fits", "mask.png")) else: weights = None background_this = fit_background_fits(image=science_image, model_type=method[:method.find(" ")], deg=degree, local=local, global_sub=global_sub, centre_x=x, centre_y=y, frame=frame, weights=weights) background[0].data += background_this[0].data if recorrect_subbed: offset = get_median_background(image=science, ra=epoch_params["renormalise_centre_ra"], dec=epoch_params["renormalise_centre_dec"], frame=50, show=False, output=new_path[ :new_path.find("bg_sub")] + "renorm_patch_") print("RECORRECT_SUBBED:", recorrect_subbed) print("SUBTRACTING FROM BACKGROUND:", offset) print(bottom, top, left, right) print(background[0].data[bottom:top, left:right].shape) print(np.median(background[0].data[bottom:top, left:right])) background[0].data[bottom:top, left:right] -= offset print(np.median(background[0].data[bottom:top, left:right])) if trim_image: print("TRIMMED_PATH_FIL:", trimmed_path_fil) science_image = ff.trim_file(path=science_image, left=left, right=right, top=top, bottom=bottom, new_path=trimmed_path_fil + file_name.replace( "norm.fits", "trimmed_to_back.fits")) print("Science after trim:", science_image) background = ff.trim_file(path=background, left=left, right=right, top=top, bottom=bottom, new_path=background_path) print("Writing background to:") print(background_path) background.writeto(background_path, overwrite=True) print("SCIENCE:", science_image) print("BACKGROUND:", background) subbed = ff.subtract_file(file=science_image, sub_file=background, output=new_path) # # TODO: check if regions overlap # # plt.hist(subbed[0].data[int(y - frame + 1):int(y + frame - 1), # int(x - frame + 1):int(x + frame - 1)].flatten(), # bins=10) # plt.savefig(new_path[:new_path.find("bg_sub")] + "histplot.png") # plt.close() copyfile(data_dir + "/" + origin + "/" + data_title + ".log", destination + data_title + ".log") u.write_log(path=destination + data_title + ".log", action=f'Backgrounds subtracted using 4-background_subtract.py with method {method}\n')
def main(data_title: str, show: bool = False): properties = p.object_params_xshooter(data_title) path = properties['data_dir'] master_path = path + '/1-master_calibs/' reduced_path = path + '/2-reduced/' defringed_path = path + '/3-defringed/' u.mkdir_check(defringed_path) # Define fringe measurement points. # high_xs = [267, 267, 267, 267, 267, 267, 267, 267] # high_ys = [279, 279, 279, 279, 279, 279, 279, 279] # low_xs = [266, 266, 266, 267, 267, 270, 274, 273] # low_ys = [293, 295, 298, 301, 303, 305, 303, 292] high_xs = [219, 380, 426, 515, 156, 495, 310] high_ys = [166, 369, 185, 33, 59, 195, 70] low_xs = [219, 380, 424, 474, 160, 500, 315] low_ys = [120, 342, 213, 39, 34, 160, 35] # n_random = 1000 # # high_xs = np.random.random(n_random) # high_xs *= 507 # high_xs += 29 # high_xs = np.round(high_xs) # high_xs = high_xs.astype(int) # # high_ys = np.random.random(n_random) # high_ys *= 200 # high_ys += 20 # high_ys = np.round(high_ys) # high_ys = high_ys.astype(int) # # low_xs = np.random.random(n_random) # low_xs *= 507 # low_xs += 29 # low_xs = np.round(low_xs) # low_xs = low_xs.astype(int) # # low_ys = np.random.random(n_random) # low_ys *= 200 # low_ys += 20 # low_ys = np.round(low_ys) # low_ys = low_ys.astype(int) filters = filter(lambda f: os.path.isdir(reduced_path + f), os.listdir(reduced_path)) for f in filters: print('Constructing fringe map for', f) filter_path = reduced_path + f + '/' defringed_filter_path = defringed_path + f + '/' master_filter_path = master_path + f + '/' u.mkdir_check(defringed_filter_path) files = list( filter(lambda file: file[-5:] == '.fits', os.listdir(filter_path))) # Construct fringe map by median-combining science images. fringe_map = ff.stack(files, directory=filter_path, output=master_filter_path + 'fringe_map.fits', stack_type='median', inherit=False, show=show, normalise=True) fringe_map = fringe_map[0].data map_differences = [] for i in range(len(high_xs)): # Take high_y = high_ys[i] high_x = high_xs[i] high_cut = fringe_map[high_y - 1:high_y + 1, high_x - 1:high_x + 1] high = np.nanmedian(high_cut) low_y = low_ys[i] low_x = low_xs[i] low_cut = fringe_map[low_y - 1:low_y + 1, low_x - 1:low_x + 1] low = np.nanmedian(low_cut) map_differences.append(high - low) for file in os.listdir(filter_path): print(file) hdu = fits.open(filter_path + file) data = hdu[0].data image_differences = [] factors = [] for i in range(len(high_xs)): high_y = high_ys[i] high_x = high_xs[i] high_cut = data[high_y - 2:high_y + 2, high_x - 2:high_x + 2] high = np.nanmedian(high_cut) low_y = low_ys[i] low_x = low_xs[i] low_cut = data[low_y - 2:low_y + 2, low_x - 2:low_x + 2] low = np.nanmedian(low_cut) difference = high - low image_differences.append(difference) factor = difference / map_differences[i] factors.append(factor) used_factor = np.nanmedian(factors) adjusted_map = fringe_map * used_factor data = data - adjusted_map hdu[0].data = data norm = pl.nice_norm(data) if show: plt.imshow(data, norm=norm, origin='lower') plt.show() hdu.writeto(defringed_filter_path + file, overwrite=True) if show: norm = pl.nice_norm(fringe_map) plt.imshow(fringe_map, norm=norm, origin='lower') plt.scatter(high_xs, high_ys) plt.scatter(low_xs, low_ys) plt.show() copyfile(reduced_path + data_title + ".log", defringed_path + data_title + ".log") u.write_log(path=defringed_path + data_title + ".log", action='Edges trimmed using 4-trim.py\n')
def main(data_title: str, show: bool = False): properties = p.object_params_xshooter(data_title) path = properties['data_dir'] raw_path = path + '/0-data_with_raw_calibs/' master_path = path + '/1-master_calibs/' reduced_path = path + '/2-reduced/' u.mkdir_check(raw_path) u.mkdir_check(master_path) u.mkdir_check(reduced_path) files = os.listdir(raw_path) biases = [] flats = {} science = {} airmasses = {} print('Creating lists of files.') for file in files: if file[-5:] == '.fits': hdu = fits.open(raw_path + file) header = hdu[0].header obj = header['OBJECT'] f = header['ESO INS FILT1 NAME'] if 'BIAS' in obj: biases.append(file) elif 'FLAT' in obj: if f not in flats: flats[f] = [] flats[f].append(file) else: if f not in science: science[f] = [] airmasses[f] = [] science[f].append(file) bias_hdus = [] for bias in biases: bias_hdus.append(bias) # Stack biases. print(f'Processing biases.') ff.stack(bias_hdus, output=master_path + f'master_bias.fits', stack_type='median', directory=raw_path, inherit=False) master_bias = CCDData.read(master_path + f'master_bias.fits', unit='du') # Loop through filters. for f in science: flats_filter = flats[f] master_path_filter = master_path + f + '/' u.mkdir_check(master_path_filter) print(f'Processing flats for filter {f}.') flats_ccds = [] for flat in flats_filter: flat_ccd = CCDData.read(raw_path + flat, unit='du') # Subtract master bias from each flat. flat_ccd = ccdproc.subtract_bias(ccd=flat_ccd, master=master_bias) flats_ccds.append(flat_ccd) # Stack debiased flats. master_flat = ff.stack(flats_ccds, output=None, stack_type='median', inherit=False) master_flat.writeto(master_path_filter + f'master_flat.fits', overwrite=True) master_flat = CCDData.read(master_path_filter + f'master_flat.fits', unit='du') science_filter = science[f] reduced_path_filter = reduced_path + f + '/' u.mkdir_check(reduced_path_filter) # Loop through the science images. for image in science_filter: print(f'Reducing {image}.') image_ccd = CCDData.read(raw_path + image, unit='du') if show: norm = ImageNormalize(image_ccd.data, interval=ZScaleInterval(), stretch=SqrtStretch()) plt.imshow(image_ccd.data, origin='lower', norm=norm) plt.title('Unreduced image') plt.show() # Subtract master bias from science image. image_ccd = ccdproc.subtract_bias(image_ccd, master_bias) if show: norm = ImageNormalize(image_ccd.data, interval=ZScaleInterval(), stretch=SqrtStretch()) plt.imshow(image_ccd.data, origin='lower', norm=norm) plt.title('After debiasing') plt.show() # Divide by master flat. image_ccd = ccdproc.flat_correct(image_ccd, master_flat) if show: norm = ImageNormalize(image_ccd.data, interval=ZScaleInterval(), stretch=SqrtStretch()) plt.imshow(image_ccd.data, origin='lower', norm=norm) plt.title('After flatfielding') plt.show() # Convert back to HDU object for saving. image_ccd = image_ccd.to_hdu() image_ccd.writeto(reduced_path_filter + image, overwrite=True) u.write_log(path=reduced_path + data_title + ".log", action=f'Reduced using 1-reduce.py')
def main(data_title: str, show: bool = False): properties = p.object_params_imacs(data_title) path = properties['data_dir'] raw_path = path + '/0-raw_data_with_calibs/' master_path = path + '/1-master_calibs/' reduced_path = path + '/2-reduced/' u.mkdir_check(raw_path) u.mkdir_check(master_path) u.mkdir_check(reduced_path) for file in os.listdir(path): if os.path.isfile(path + file): shutil.move(path + file, raw_path + file) files = os.listdir(raw_path) biases = [] flats = {} science = {} airmasses = {} print('Creating lists of files.') param_dict = {} for file in files: if file[-5:] == '.fits': hdu = fits.open(raw_path + file) header = hdu[0].header obj = header['OBJECT'] f = header['FILTER'] if 'bias' in obj: biases.append(file) elif 'flat' in obj: if f not in flats: flats[f] = [] flats[f].append(file) else: if f not in science: science[f] = [] airmasses[f] = [] science[f].append(file) airmasses[f].append(header['AIRMASS']) # Loop through chip numbers. for chip in range(1, 9): print(f'Processing biases for chip {chip}.') # Get only those bias frames relevant to the current chip. biases_chip = list(filter(lambda name: name[-6] == str(chip), biases)) bias_hdus = [] for bias in biases_chip: # Correct NAXIS1 & 2 in headers; they are too large in the unreduced files, leading to padding by astropy. bias = ff.correct_naxis(file=raw_path + bias, x=2048, y=4096, write=False) bias_hdus.append(bias) # Stack biases. print(biases_chip) ff.stack(bias_hdus, output=master_path + f'master_bias_c{chip}.fits', stack_type='median', directory=raw_path) master_bias = CCDData.read(master_path + f'master_bias_c{chip}.fits', unit='du') # Loop through filters. for f in science: flats_filter = flats[f] master_path_filter = master_path + f + '/' u.mkdir_check(master_path_filter) print(f'Processing flats for chip {chip}, filter {f}.') # Get only those flats relevant to the current chip and filter. flats_chip = list(filter(lambda name: name[-6] == str(chip), flats_filter)) flats_chip_ccds = [] for flat in flats_chip: flat_ccd = CCDData.read(raw_path + flat, unit='du') # Correct NAXIS1 & 2 in headers; they are too large in the unreduced files, leading to padding by astropy. flat_ccd = ff.correct_naxis(file=flat_ccd, x=2048, y=4096, write=False) # Subtract master bias from each flat. flat_ccd = ccdproc.subtract_bias(ccd=flat_ccd, master=master_bias) flats_chip_ccds.append(flat_ccd) # Stack debiased flats. master_flat = ff.stack(flats_chip_ccds, output=None, stack_type='median') master_flat.writeto(master_path_filter + f'master_flat_c{chip}.fits', overwrite=True) master_flat = CCDData.read(master_path_filter + f'master_flat_c{chip}.fits', unit='du') science_filter = science[f] param_dict[f + '_airmass_mean'] = float(np.nanmean(airmasses[f])) param_dict[f + '_airmass_err'] = float(2 * np.nanstd(airmasses[f])) param_dict[f + '_n_frames'] = len(science_filter) param_dict[f + '_n_exposures'] = len(science_filter) / 8 reduced_path_filter = reduced_path + f + '/' u.mkdir_check(reduced_path_filter) # Loop through the science images taken on the current chip. for image in filter(lambda name: name[-6] == str(chip), science_filter): print(f'Reducing {image}.') image_ccd = CCDData.read(raw_path + image, unit='du') if show: norm = ImageNormalize(image_ccd.data, interval=ZScaleInterval(), stretch=SqrtStretch()) plt.imshow(image_ccd.data, origin='lower', norm=norm) plt.title('Unreduced image') plt.show() # Correct NAXIS1 & 2 in header; they are too large in the unreduced files, leading to padding by astropy. image_ccd = ff.correct_naxis(file=image_ccd, x=2048, y=4096, write=False) if show: norm = ImageNormalize(image_ccd.data, interval=ZScaleInterval(), stretch=SqrtStretch()) plt.imshow(image_ccd.data, origin='lower', norm=norm) plt.title('After NAXIS correction') plt.show() # Subtract master bias from science image. image_ccd = ccdproc.subtract_bias(image_ccd, master_bias) if show: norm = ImageNormalize(image_ccd.data, interval=ZScaleInterval(), stretch=SqrtStretch()) plt.imshow(image_ccd.data, origin='lower', norm=norm) plt.title('After debiasing') plt.show() # Divide by master flat. image_ccd = ccdproc.flat_correct(image_ccd, master_flat) if show: norm = ImageNormalize(image_ccd.data, interval=ZScaleInterval(), stretch=SqrtStretch()) plt.imshow(image_ccd.data, origin='lower', norm=norm) plt.title('After flatfielding') plt.show() # Convert back to HDU object for saving. image_ccd = image_ccd.to_hdu() image_ccd.writeto(reduced_path_filter + image, overwrite=True) u.write_log(path=reduced_path + data_title + ".log", action=f'Reduced using 1-reduce.py') p.add_params(path + '/output_values', param_dict)
def main(origin_dir, output_dir, data_title, sextractor_path): print("\nExecuting Python script pipeline_fors2/3-trim.py, with:") print(f"\tepoch {data_title}") print(f"\torigin directory {origin_dir}") print(f"\toutput directory {output_dir}") print() # If this is None, we don't want the SExtractor components to be performed. if sextractor_path is not None: if not os.path.isdir(sextractor_path): os.mkdir(sextractor_path) do_sextractor = True print(os.getcwd()) else: do_sextractor = False if not os.path.isdir(output_dir): os.mkdir(output_dir) if not os.path.isdir(output_dir + "backgrounds/"): os.mkdir(output_dir + "backgrounds/") if not os.path.isdir(output_dir + "science/"): os.mkdir(output_dir + "science/") wdir = origin_dir + "backgrounds/" epoch_params = p.object_params_fors2(obj=data_title) outputs = p.object_output_params(obj=data_title) fils = outputs["filters"] edged = False up_left = 0 up_right = 0 up_bottom = 0 up_top = 0 dn_left = 0 dn_right = 0 dn_bottom = 0 dn_top = 0 for fil in fils: print(output_dir + "backgrounds/" + fil) if not os.path.isdir(output_dir + "backgrounds/" + fil): os.mkdir(output_dir + "backgrounds/" + fil) print('HERE:') print(wdir + fil) files = os.listdir(wdir + fil) files.sort() if not edged: # Find borders of noise frame using backgrounds. # First, make sure that the background we're using is for the top chip. i = 0 while f.get_chip_num(wdir + fil + "/" + files[i]) != 1: i += 1 up_left, up_right, up_bottom, up_top = f.detect_edges(wdir + fil + "/" + files[i]) # Ditto for the bottom chip. i = 0 while f.get_chip_num(wdir + fil + "/" + files[i]) != 2: i += 1 dn_left, dn_right, dn_bottom, dn_top = f.detect_edges(wdir + fil + "/" + files[i]) up_left = up_left + 5 up_right = up_right - 5 up_top = up_top - 5 dn_left = dn_left + 5 dn_right = dn_right - 5 dn_bottom = dn_bottom + 5 print('Upper chip:') print(up_left, up_right, up_top, up_bottom) print('Lower:') print(dn_left, dn_right, dn_top, dn_bottom) edged = True for i, file in enumerate(files): print(f'{i} {file}') for i, file in enumerate(files): new_path = output_dir + "backgrounds/" + fil + "/" + file.replace( ".fits", "_trim.fits") # Add GAIN and SATURATE keywords to headers. path = wdir + fil + "/" + file print(f'{i} {file}') # Split the files into upper CCD and lower CCD if f.get_chip_num(path) == 1: print('Upper Chip:') f.trim_file(path, left=up_left, right=up_right, top=up_top, bottom=up_bottom, new_path=new_path) elif f.get_chip_num(path) == 2: print('Lower Chip:') f.trim_file(path, left=dn_left, right=dn_right, top=dn_top, bottom=dn_bottom, new_path=new_path) else: raise ValueError( 'Invalid chip ID; could not trim based on upper or lower chip.' ) # Repeat for science images wdir = origin_dir + "science/" fils = os.listdir(wdir) for fil in fils: print(output_dir + "science/" + fil) if do_sextractor: if not os.path.isdir(sextractor_path + fil): os.mkdir(sextractor_path + fil) if not os.path.isdir(output_dir + "science/" + fil): os.mkdir(output_dir + "science/" + fil) files = os.listdir(wdir + fil) files.sort() for i, file in enumerate(files): print(f'{i} {file}') for i, file in enumerate(files): # Split the files into upper CCD and lower CCD, with even-numbered being upper and odd-numbered being lower new_file = file.replace(".fits", "_trim.fits") new_path = output_dir + "science/" + fil + "/" + new_file path = wdir + fil + "/" + file f.change_header(file=path, key='GAIN', value=0.8) f.change_header(file=path, key='SATURATE', value=65535.) if f.get_chip_num(path) == 1: print('Upper Chip:') f.trim_file(path, left=up_left, right=up_right, top=up_top, bottom=up_bottom, new_path=new_path) if do_sextractor: copyfile(new_path, sextractor_path + fil + "/" + new_file) elif f.get_chip_num(path) == 2: print('Lower Chip:') f.trim_file(path, left=dn_left, right=dn_right, top=dn_top, bottom=dn_bottom, new_path=new_path) try: copyfile(origin_dir + data_title + ".log", output_dir + data_title + ".log") except FileNotFoundError: print("Previous log not found.") u.write_log(path=output_dir + data_title + ".log", action='Edges trimmed using 3-trim.py\n')
def main(data_title, sextractor_path, origin, destination): print( "\nExecuting Python script pipeline_fors2/4-divide_by_exp_time.py, with:" ) print(f"\tepoch {data_title}") print(f"\tsextractor path {sextractor_path}") print(f"\torigin directory {origin}") print(f"\tdestination directory {destination}") print() properties = p.object_params_fors2(data_title) outputs = p.object_output_params(obj=data_title, instrument='FORS2') data_dir = properties['data_dir'] if sextractor_path is not None: if not os.path.isdir(sextractor_path): os.mkdir(sextractor_path) do_sextractor = True else: do_sextractor = False origin_path = data_dir + origin destination_path = data_dir + destination u.mkdir_check(destination_path) u.mkdir_check(destination_path + "science/") u.mkdir_check(destination_path + "backgrounds/") filters = outputs['filters'] for fil in filters: u.mkdir_check(destination_path + "science/" + fil) u.mkdir_check(destination_path + "backgrounds/" + fil) if do_sextractor: if not os.path.isdir(sextractor_path + fil): os.mkdir(sextractor_path + fil) files = os.listdir(origin_path + "science/" + fil + "/") for file_name in files: if file_name[-5:] == '.fits': science_origin = origin_path + "science/" + fil + "/" + file_name science_destination = destination_path + "science/" + fil + "/" + file_name.replace( "trim", "norm") background_origin = origin_path + "backgrounds/" + fil + "/" + file_name.replace( "SCIENCE_REDUCED", "PHOT_BACKGROUND_SCI") background_destination = destination_path + "backgrounds/" + fil + "/" + \ file_name.replace("SCIENCE_REDUCED", "PHOT_BACKGROUND_SCI").replace("trim", "norm") print(science_origin) # Divide by exposure time to get an image in counts/second. ff.divide_by_exp_time(file=science_origin, output=science_destination) ff.divide_by_exp_time(file=background_origin, output=background_destination) if do_sextractor: copyfile( science_destination, sextractor_path + fil + "/" + file_name.replace("trim", "norm")) if os.path.isfile(origin_path + data_title + '.log'): copyfile(origin_path + data_title + '.log', destination_path + data_title + ".log") u.write_log(path=destination_path + data_title + ".log", action=f'Divided by exposure time.')
def main(data_title: 'str', delete_output: bool = True): print( "\nExecuting Python script pipeline_fors2/2-sort_after_esoreflex.py, with:" ) print(f"\tepoch {data_title}") print() eso_dir = p.config['esoreflex_output_dir'] if os.path.isdir(eso_dir): obj_params = p.object_params_fors2(data_title) data_dir = obj_params['data_dir'] destination = data_dir + "2-sorted/" date = None output_values = p.object_output_params(obj=data_title, instrument='FORS2') mjd = int(output_values['mjd_obs']) obj = output_values['object'] print( f"Looking for data with object '{obj}' and MJD of observation {mjd} inside {eso_dir}" ) # Look for files with the appropriate object and MJD, as recorded in output_values # List directories in eso_output_dir; these are dates on which data was reduced using ESOReflex. eso_dirs = filter(lambda d: os.path.isdir(eso_dir + "/" + d), os.listdir(eso_dir)) for directory in eso_dirs: directory += "/" # List directories within 'reduction date' directories. # These should represent individual images reduced. print(f"Searching {eso_dir + directory}") eso_subdirs = filter( lambda d: os.path.isdir(eso_dir + directory + d) and "FORS2" in d, os.listdir(eso_dir + directory)) for subdirectory in eso_subdirs: subdirectory += "/" subpath = eso_dir + "/" + directory + subdirectory print(f"\tSearching {subpath}") # Get the files within the image directory. files = filter(lambda d: os.path.isfile(subpath + d), os.listdir(subpath)) for file_name in files: # Retrieve the target object name from the fits file. file_path = subpath + file_name file = fits.open(file_path) file_obj = ff.get_object(file) file_mjd = int(ff.get_header_attribute(file, 'MJD-OBS')) file_filter = ff.get_filter(file) # Check the object name and observation date against those of the epoch we're concerned with. if file_obj == obj and file_mjd == mjd: # Check which type of file we have. date = directory[:-1] if file_name[-28:] == "PHOT_BACKGROUND_SCI_IMG.fits": file_destination = f"{destination}/backgrounds/" suffix = "PHOT_BACKGROUND_SCI_IMG.fits" elif file_name[-25:] == "OBJECT_TABLE_SCI_IMG.fits": file_destination = f"{destination}/obj_tbls/" suffix = "OBJECT_TABLE_SCI_IMG.fits" elif file_name[-24:] == "SCIENCE_REDUCED_IMG.fits": file_destination = f"{destination}/science/" suffix = "SCIENCE_REDUCED_IMG.fits" else: file_destination = f"{destination}/sources/" suffix = "SOURCES_SCI_IMG.fits" # Make this directory, if it doesn't already exist. u.mkdir_check(file_destination) # Make a subdirectory by filter. file_destination += file_filter + "/" u.mkdir_check(file_destination) # Title new file. file_destination += f"{data_title}_{subdirectory[:-1]}_{suffix}" # Copy file to new location. print( f"Copying: {file_path} to \n\t {file_destination}") file.writeto(file_destination, overwrite=True) if delete_output and os.path.isfile(file_destination): os.remove(file_path) if not os.path.isfile(f"{destination}/{data_title}.log"): sh.copy(f"{data_dir}0-data_with_raw_calibs/{data_title}.log", f"{destination}/{data_title}.log") u.write_log(path=destination + data_title + ".log", action=f'Data reduced with ESOReflex.', date=date) u.write_log(path=destination + data_title + ".log", action='Files sorted using 2-sort_after_esoreflex.sh') else: print(f"ESO output directory '{eso_dir}' not found.") exit(1)