def main(obj, test, magnitude): print('Got here.') properties = p.object_params_fors2(obj) synth_path = properties['data_dir'] + 'synthetic/' u.mkdir_check(synth_path) synth_path = synth_path + 'frb_position/' u.mkdir_check(synth_path) now = time.Time.now() now.format = 'isot' test = str(now) + '_' + test test_path = synth_path + test + '/' print(properties['filters']) ph.insert_synthetic_at_frb(obj=obj, test_path=test_path, filters=properties['filters'], magnitudes=[magnitude, magnitude], add_path=True)
def main(obj, test, curves_path): properties = p.object_params_fors2(obj) if curves_path[-1] != '/': curves_path += '/' mag_table_file = filter(lambda file: file[-4:] == '.csv', os.listdir(curves_path)).__next__() mag_table = table.Table.read(curves_path + mag_table_file) filters = mag_table.colnames.copy() filters.remove('model') synth_path = properties['data_dir'] + 'synthetic/' u.mkdir_check(synth_path) synth_path = synth_path + 'frb_position/' u.mkdir_check(synth_path) now = time.Time.now() now.format = 'isot' synth_path += f'sn_models_{now}/' u.mkdir_check(synth_path) for row in mag_table: model = row['model'] magnitudes = [] for f in filters: magnitudes.append(row[f]) test_spec = test + '_' + model test_path = synth_path + test_spec + '/' ph.insert_synthetic_at_frb(obj=obj, test_path=test_path, filters=filters, magnitudes=magnitudes, add_path=False) for f in filters: p.add_output_path(obj=obj, key=f[0] + '_subtraction_image_synth_frb_sn_models', path=synth_path)
def main(epoch, origin, destination): print("\nExecuting Python script pipeline_fors2/4.1-insert_test_synth.py, with:") print(f"\tepoch {epoch}") print(f"\torigin directory {origin}") print(f"\tdestination directory {destination}") print() epoch_params = p.object_params_fors2(obj=epoch) outputs = p.object_output_params(obj=epoch, instrument='FORS2') data_dir = epoch_params['data_dir'] insert = epoch_params['test_synths'] origin_path = data_dir + "analysis/sextractor/" + origin destination_path = data_dir + destination u.mkdir_check(destination_path) u.mkdir_check(destination_path + "science/") u.mkdir_check(destination_path + "backgrounds/") filters = outputs['filters'] for fil in filters: f = fil[0] path_fil_output = destination_path + "science/" + fil + "/" path_fil_input = origin_path + fil + "/" u.mkdir_check(path_fil_output) u.mkdir_check(destination_path + "backgrounds/" + fil) zeropoint, _, airmass, _, extinction, _ = ph.select_zeropoint(obj=epoch, filt=fil, instrument='fors2', outputs=outputs) print(path_fil_input) # print(os.listdir(path_fil_input)) for fits_file in filter(lambda f: f.endswith("_norm.fits"), os.listdir(path_fil_input)): print(fits_file) path_fits_file_input = path_fil_input + fits_file path_fits_file_output = path_fil_output + fits_file path_psf_model = path_fits_file_input.replace(".fits", "_psfex.psf") try: ph.insert_point_sources_to_file(file=path_fits_file_input, x=array(insert["ra"]), y=array(insert["dec"]), mag=insert[f"{f}_mag"], output=path_fits_file_output, zeropoint=zeropoint, extinction=extinction, airmass=airmass, world_coordinates=True, psf_model=path_psf_model ) except ValueError: ph.insert_point_sources_to_file(file=path_fits_file_input, x=array(insert["ra"]), y=array(insert["dec"]), mag=insert[f"{f}_mag"], output=path_fits_file_output, zeropoint=zeropoint, extinction=extinction, airmass=airmass, world_coordinates=True, fwhm=fits.open(path_psf_model)[1].header['PSF_FWHM'] ) if os.path.isfile(origin_path + epoch + '.log'): copyfile(origin_path + epoch + '.log', destination_path + epoch + ".log") u.write_log(path=destination_path + epoch + ".log", action=f'Divided by exposure time.')
def main(data_dir, data_title, origin, destination, all_synths): print("\nExecuting Python script pipeline_fors2/5-background_subtract.py, with:") print(f"\tepoch {data_title}") print(f"\torigin directory {origin}") print(f"\tdestination directory {destination}") print() methods = ["ESO backgrounds only", "SExtractor backgrounds only", "polynomial fit", "Gaussian fit", "median value"] if all_synths: frame = 56 method = "polynomial fit" degree = 5 do_mask = True local = True global_sub = False trim_image = False recorrect_subbed = True eso_back = False else: frame = 200 # frame_arcsec = 30 * units.arcsec # frame_deg = frame_arcsec.to(units.deg) eso_back = False _, method = u.select_option(message="Please select the background subtraction method.", options=methods, default="polynomial fit") degree = None if method == "polynomial fit": degree = u.user_input(message=f"Please enter the degree of {method} to use:", typ=int, default=3) elif method == "ESO backgrounds only": eso_back = True do_mask = False if method not in ["ESO backgrounds only", "SExtractor backgrounds only", "median value"]: do_mask = u.select_yn(message="Mask sources using SExtractor catalogue?", default=True) if method in ["polynomial fit", "Gaussian fit"]: local = u.select_yn(message="Use a local fit?", default=True) else: local = False global_sub = False trim_image = False recorrect_subbed = False if local: global_sub = u.select_yn(message="Subtract local fit from entire image?", default="n") if not global_sub: trim_image = u.select_yn(message="Trim images to subtracted region?", default="y") recorrect_subbed = u.select_yn(message="Re-normalise background of subtracted region?", default="y") # if not eso_back and method != "SExtractor backgrounds only": # eso_back = u.select_yn(message="Subtract ESO Reflex fitted backgrounds first?", default=False) outputs = p.object_output_params(data_title, instrument='FORS2') data_dir = u.check_trailing_slash(data_dir) destination = u.check_trailing_slash(destination) destination = data_dir + destination u.mkdir_check_nested(destination) origin = u.check_trailing_slash(origin) science_origin = data_dir + origin + "science/" print(science_origin) filters = outputs['filters'] frb_params = p.object_params_frb(obj=data_title[:-2]) epoch_params = p.object_params_fors2(obj=data_title) background_origin_eso = "" if eso_back: background_origin_eso = data_dir + "/" + origin + "/backgrounds/" if method == "SExtractor backgrounds only": background_origin = f"{data_dir}{origin}backgrounds_sextractor/" elif method == "polynomial fit": background_origin = f"{destination}backgrounds/" # f"{destination}backgrounds_{method.replace(' ', '')}_degree_{degree}_local_{local}_globalsub_{global_sub}/" else: background_origin = f"{destination}backgrounds/" # f"{destination}backgrounds_{method.replace(' ', '')}_local_{local}_globalsub_{global_sub}/" trimmed_path = "" if trim_image: trimmed_path = f"{data_dir}{origin}trimmed_to_background/" u.mkdir_check_nested(trimmed_path) ra = frb_params["burst_ra"] dec = frb_params["burst_dec"] if all_synths: ras = epoch_params["test_synths"]["ra"] decs = epoch_params["test_synths"]["dec"] else: ras = [ra] decs = [dec] for fil in filters: trimmed_path_fil = "" if trim_image: trimmed_path_fil = f"{trimmed_path}{fil}/" u.mkdir_check(trimmed_path_fil) background_fil_dir = f"{background_origin}{fil}/" u.mkdir_check_nested(background_fil_dir) science_destination_fil = f"{destination}science/{fil}/" u.mkdir_check_nested(science_destination_fil) files = os.listdir(science_origin + fil + "/") for file_name in files: if file_name.endswith('.fits'): new_file = file_name.replace("norm", "bg_sub") new_path = f"{science_destination_fil}/{new_file}" print("NEW_PATH:", new_path) science = science_origin + fil + "/" + file_name # First subtract ESO Reflex background images # frame = (frame_deg / f.get_pixel_scale(file=science, astropy_units=True)[1]).to(f.pix).value if eso_back: background_eso = background_origin_eso + fil + "/" + file_name.replace("SCIENCE_REDUCED", "PHOT_BACKGROUND_SCI") ff.subtract_file(file=science, sub_file=background_eso, output=new_path) science_image = new_path if method != "ESO backgrounds only": print(ra, dec) print("Science image:", science) science_image = fits.open(science) print("Science file:", science_image) wcs_this = WCS(header=science_image[0].header) if method == "SExtractor backgrounds only": background = background_origin + fil + "/" + file_name + "_back.fits" print("Background image:", background) else: if method == "median value": print(science_image[0].data.shape) _, background_value, _ = sigma_clipped_stats(science_image[0].data) background = deepcopy(science_image) background[0].data = np.full(shape=science_image[0].data.shape, fill_value=background_value) background_path = background_origin + fil + "/" + file_name.replace("SCIENCE_REDUCED", "PHOT_BACKGROUND_MEDIAN") # Next do background fitting. else: background = deepcopy(science_image) background[0].data = np.zeros(background[0].data.shape) background_path = background_origin + fil + "/" + file_name.replace("SCIENCE_REDUCED", "PHOT_BACKGROUND_FITTED") for i, ra in enumerate(ras): dec = decs[i] x, y = wcs_this.all_world2pix(ra, dec, 0) print(x, y) bottom, top, left, right = ff.subimage_edges(data=science_image[0].data, x=x, y=y, frame=frame) if do_mask: # Produce a pixel mask that roughly masks out the true sources in the image so that # they don't get fitted. mask_max = 10 _, pixel_scale = ff.get_pixel_scale(science_image) sextractor = Table.read( f"{data_dir}analysis/sextractor/4-divided_by_exp_time/{fil}/{file_name.replace('.fits', '_psf-fit.cat')}", format='ascii.sextractor') weights = np.ones(shape=science_image[0].data.shape) for obj in filter( lambda o: left < o["X_IMAGE"] < right and bottom < o["Y_IMAGE"] < top, sextractor): mask_rad = min(int(obj["A_WORLD"] * obj["KRON_RADIUS"] / pixel_scale), mask_max) x_prime = int(np.round(obj["X_IMAGE"])) y_prime = int(np.round(obj["Y_IMAGE"])) weights[y_prime - mask_rad:y_prime + mask_rad, x_prime - mask_rad:x_prime + mask_rad] = 0.0 plt.imshow(weights, origin="lower") plt.savefig( background_origin + fil + "/" + file_name.replace("norm.fits", "mask.png")) else: weights = None background_this = fit_background_fits(image=science_image, model_type=method[:method.find(" ")], deg=degree, local=local, global_sub=global_sub, centre_x=x, centre_y=y, frame=frame, weights=weights) background[0].data += background_this[0].data if recorrect_subbed: offset = get_median_background(image=science, ra=epoch_params["renormalise_centre_ra"], dec=epoch_params["renormalise_centre_dec"], frame=50, show=False, output=new_path[ :new_path.find("bg_sub")] + "renorm_patch_") print("RECORRECT_SUBBED:", recorrect_subbed) print("SUBTRACTING FROM BACKGROUND:", offset) print(bottom, top, left, right) print(background[0].data[bottom:top, left:right].shape) print(np.median(background[0].data[bottom:top, left:right])) background[0].data[bottom:top, left:right] -= offset print(np.median(background[0].data[bottom:top, left:right])) if trim_image: print("TRIMMED_PATH_FIL:", trimmed_path_fil) science_image = ff.trim_file(path=science_image, left=left, right=right, top=top, bottom=bottom, new_path=trimmed_path_fil + file_name.replace( "norm.fits", "trimmed_to_back.fits")) print("Science after trim:", science_image) background = ff.trim_file(path=background, left=left, right=right, top=top, bottom=bottom, new_path=background_path) print("Writing background to:") print(background_path) background.writeto(background_path, overwrite=True) print("SCIENCE:", science_image) print("BACKGROUND:", background) subbed = ff.subtract_file(file=science_image, sub_file=background, output=new_path) # # TODO: check if regions overlap # # plt.hist(subbed[0].data[int(y - frame + 1):int(y + frame - 1), # int(x - frame + 1):int(x + frame - 1)].flatten(), # bins=10) # plt.savefig(new_path[:new_path.find("bg_sub")] + "histplot.png") # plt.close() copyfile(data_dir + "/" + origin + "/" + data_title + ".log", destination + data_title + ".log") u.write_log(path=destination + data_title + ".log", action=f'Backgrounds subtracted using 4-background_subtract.py with method {method}\n')
def main(data_title: 'str'): print("\nExecuting Python script pipeline_fors2/1-initial.py, with:") print(f"\tepoch {data_title}") print() epoch_params = p.object_params_fors2(obj=data_title) data_dir = epoch_params['data_dir'] output_dir = data_dir + "/0-data_with_raw_calibs/" # Write tables of fits files to main directory; firstly, science images only: table = fits_table(input_path=output_dir, output_path=data_dir + data_title + "_fits_table_science.csv", science_only=True) # Then including all calibration files table_full = fits_table(input_path=output_dir, output_path=data_dir + data_title + "_fits_table_all.csv", science_only=False) fits_table_all(input_path=output_dir, output_path=data_dir + data_title + "_fits_table_detailled.csv", science_only=False) # Clear output files for fresh start. u.rm_check(data_dir + '/output_values.yaml') u.rm_check(data_dir + '/output_values.json') # Collect list of filters used: filters = [] columns = [] for j in [1, 2, 3, 4, 5]: column = 'filter' + str(j) for name in table[column]: if name != 'free': if name not in filters: filters.append(name) columns.append(column) # Collect pointings of standard-star observations. std_ras = [] std_decs = [] std_pointings = [] # TODO: This is a horrible way to do this. Take the time to find a better one. for ra in table_full[table_full['object'] == 'STD']['ref_ra']: if ra not in std_ras: std_ras.append(ra) for dec in table_full[table_full['object'] == 'STD']['ref_dec']: if dec not in std_decs: std_decs.append(dec) for i, ra in enumerate(std_ras): std_pointings.append(f'RA{ra}_DEC{std_decs[i]}') print(std_ras) print(std_decs) print(std_pointings) # Collect and save some stats on those filters: param_dict = {} exp_times = [] ns_exposures = [] param_dict['filters'] = filters param_dict['object'] = table['object'][0] param_dict['obs_name'] = table['obs_name'][0] mjd = param_dict['mjd_obs'] = float(table['mjd_obs'][0]) for i, f in enumerate(filters): f_0 = f[0] exp_time = table['exp_time'][table[columns[i]] == f] exp_times.append(exp_time) airmass_col = table['airmass'][table[columns[i]] == f] n_frames = sum(table[columns[i]] == f) n_exposures = n_frames / 2 ns_exposures.append(n_exposures) airmass = float(np.nanmean(airmass_col)) param_dict[f_0 + '_exp_time_mean'] = float(np.nanmean(exp_time)) param_dict[f_0 + '_exp_time_err'] = float(2 * np.nanstd(exp_time)) param_dict[f_0 + '_airmass_mean'] = airmass param_dict[f_0 + '_airmass_err'] = float( max(np.nanmax(airmass_col) - airmass, airmass - np.nanmin(airmass_col))) param_dict[f_0 + '_n_frames'] = float(n_frames) param_dict[f_0 + '_n_exposures'] = float(n_exposures) param_dict[f_0 + '_mjd_obs'] = float(np.nanmean(table['mjd_obs'][table[columns[i]] == f])) std_filter_dir = f'{data_dir}calibration/std_star/{f}/' u.mkdir_check(std_filter_dir) print(f'Copying {f} calibration data to std_star folder...') # Sort the STD files by filter, and within that by pointing. for j, ra in enumerate(std_ras): at_pointing = False pointing = std_pointings[j] pointing_dir = std_filter_dir + pointing + '/' for file in \ table_full[ (table_full['object'] == 'STD') & (table_full['ref_ra'] == ra) & (table_full[columns[i]] == f)]['identifier']: at_pointing = True u.mkdir_check(pointing_dir) shutil.copyfile(output_dir + file, pointing_dir + file) if at_pointing: for file in table_full[table_full['object'] == 'BIAS']['identifier']: shutil.copyfile(output_dir + file, pointing_dir + file) for file in table_full[(table_full['object'] == 'FLAT,SKY') & (table_full[columns[i]] == f)][ 'identifier']: shutil.copyfile(output_dir + file, pointing_dir + file) p.add_output_values(obj=data_title, params=param_dict) if "new_epoch" in data_dir: mjd = f"MJD{int(float(mjd))}" new_data_dir = data_dir.replace("new_epoch", mjd) p.add_epoch_param(obj=data_title, params={"data_dir": new_data_dir})
def main(obj, test, n, mag_lower, mag_upper, colour_upper, colour_lower): properties = p.object_params_fors2(obj) output = p.object_output_params(obj=obj, instrument='FORS2') paths = p.object_output_paths(obj) burst_properties = p.object_params_frb(obj[:-2]) synth_path = properties['data_dir'] + 'synthetic/' u.mkdir_check(synth_path) synth_path = synth_path + 'random/' u.mkdir_check(synth_path) now = time.Time.now() now.format = 'isot' test = str(now) + '_' + test test_path = synth_path + test + '/' u.mkdir_check(test_path) filters = {} bluest = None bluest_lambda = np.inf for f in output['filters']: filter_properties = p.filter_params(f=f, instrument='FORS2') filters[f] = filter_properties lambda_eff = filter_properties['lambda_eff'] if lambda_eff < bluest_lambda: bluest_lambda = lambda_eff bluest = f # Insert random sources in the bluest filter. f_0 = bluest[0] output_properties = p.object_output_params(obj) fwhm = output_properties[f_0 + '_fwhm_pix'] zeropoint, _, airmass, _ = ph.select_zeropoint(obj, bluest, instrument='fors2') base_path = paths[f_0 + '_subtraction_image'] output_path = test_path + f_0 + '_random_sources.fits' _, sources = ph.insert_random_point_sources_to_file(file=base_path, fwhm=fwhm, output=output_path, n=n, airmass=airmass, zeropoint=zeropoint) p.add_output_path(obj=obj, key=f_0 + '_subtraction_image_synth_random', path=output_path) # Now insert sources at the same positions in other filters, but with magnitudes randomised. for f in filters: if f != bluest: f_0 = f[0] output_properties = p.object_output_params(obj) fwhm = output_properties[f_0 + '_fwhm_pix'] zeropoint, _, airmass, _ = ph.select_zeropoint(obj, f, instrument='fors2') base_path = paths[f_0 + '_subtraction_image'] mag = np.random.uniform(mag_lower, mag_upper, size=n) output_path = test_path + f_0 + '_random_sources.fits' ph.insert_point_sources_to_file(file=base_path, fwhm=fwhm, output=output_path, x=sources['x_0'], y=sources['y_0'], mag=mag, airmass=airmass, zeropoint=zeropoint) p.add_output_path(obj=obj, key=f_0 + '_subtraction_image_synth_random', path=output_path)
def main(origin_dir, output_dir, data_title, sextractor_path): print("\nExecuting Python script pipeline_fors2/3-trim.py, with:") print(f"\tepoch {data_title}") print(f"\torigin directory {origin_dir}") print(f"\toutput directory {output_dir}") print() # If this is None, we don't want the SExtractor components to be performed. if sextractor_path is not None: if not os.path.isdir(sextractor_path): os.mkdir(sextractor_path) do_sextractor = True print(os.getcwd()) else: do_sextractor = False if not os.path.isdir(output_dir): os.mkdir(output_dir) if not os.path.isdir(output_dir + "backgrounds/"): os.mkdir(output_dir + "backgrounds/") if not os.path.isdir(output_dir + "science/"): os.mkdir(output_dir + "science/") wdir = origin_dir + "backgrounds/" epoch_params = p.object_params_fors2(obj=data_title) outputs = p.object_output_params(obj=data_title) fils = outputs["filters"] edged = False up_left = 0 up_right = 0 up_bottom = 0 up_top = 0 dn_left = 0 dn_right = 0 dn_bottom = 0 dn_top = 0 for fil in fils: print(output_dir + "backgrounds/" + fil) if not os.path.isdir(output_dir + "backgrounds/" + fil): os.mkdir(output_dir + "backgrounds/" + fil) print('HERE:') print(wdir + fil) files = os.listdir(wdir + fil) files.sort() if not edged: # Find borders of noise frame using backgrounds. # First, make sure that the background we're using is for the top chip. i = 0 while f.get_chip_num(wdir + fil + "/" + files[i]) != 1: i += 1 up_left, up_right, up_bottom, up_top = f.detect_edges(wdir + fil + "/" + files[i]) # Ditto for the bottom chip. i = 0 while f.get_chip_num(wdir + fil + "/" + files[i]) != 2: i += 1 dn_left, dn_right, dn_bottom, dn_top = f.detect_edges(wdir + fil + "/" + files[i]) up_left = up_left + 5 up_right = up_right - 5 up_top = up_top - 5 dn_left = dn_left + 5 dn_right = dn_right - 5 dn_bottom = dn_bottom + 5 print('Upper chip:') print(up_left, up_right, up_top, up_bottom) print('Lower:') print(dn_left, dn_right, dn_top, dn_bottom) edged = True for i, file in enumerate(files): print(f'{i} {file}') for i, file in enumerate(files): new_path = output_dir + "backgrounds/" + fil + "/" + file.replace( ".fits", "_trim.fits") # Add GAIN and SATURATE keywords to headers. path = wdir + fil + "/" + file print(f'{i} {file}') # Split the files into upper CCD and lower CCD if f.get_chip_num(path) == 1: print('Upper Chip:') f.trim_file(path, left=up_left, right=up_right, top=up_top, bottom=up_bottom, new_path=new_path) elif f.get_chip_num(path) == 2: print('Lower Chip:') f.trim_file(path, left=dn_left, right=dn_right, top=dn_top, bottom=dn_bottom, new_path=new_path) else: raise ValueError( 'Invalid chip ID; could not trim based on upper or lower chip.' ) # Repeat for science images wdir = origin_dir + "science/" fils = os.listdir(wdir) for fil in fils: print(output_dir + "science/" + fil) if do_sextractor: if not os.path.isdir(sextractor_path + fil): os.mkdir(sextractor_path + fil) if not os.path.isdir(output_dir + "science/" + fil): os.mkdir(output_dir + "science/" + fil) files = os.listdir(wdir + fil) files.sort() for i, file in enumerate(files): print(f'{i} {file}') for i, file in enumerate(files): # Split the files into upper CCD and lower CCD, with even-numbered being upper and odd-numbered being lower new_file = file.replace(".fits", "_trim.fits") new_path = output_dir + "science/" + fil + "/" + new_file path = wdir + fil + "/" + file f.change_header(file=path, key='GAIN', value=0.8) f.change_header(file=path, key='SATURATE', value=65535.) if f.get_chip_num(path) == 1: print('Upper Chip:') f.trim_file(path, left=up_left, right=up_right, top=up_top, bottom=up_bottom, new_path=new_path) if do_sextractor: copyfile(new_path, sextractor_path + fil + "/" + new_file) elif f.get_chip_num(path) == 2: print('Lower Chip:') f.trim_file(path, left=dn_left, right=dn_right, top=dn_top, bottom=dn_bottom, new_path=new_path) try: copyfile(origin_dir + data_title + ".log", output_dir + data_title + ".log") except FileNotFoundError: print("Previous log not found.") u.write_log(path=output_dir + data_title + ".log", action='Edges trimmed using 3-trim.py\n')
def main(obj, astrometry_path, sextractor_path, template, show): print("\nExecuting Python script pipeline_fors2/8-astrometry.py, with:") print(f"\tepoch {obj}") print(f"\tastrometry path {astrometry_path}") print(f"\tsextractor path {sextractor_path}") print(f"\ttemplate {template}") print(f"\tshow {show}") print() files = os.listdir(astrometry_path) template_file = template[0] + '_astrometry.fits' params = p.object_params_fors2(obj) for file in files: if 'coadded.fits' in file: astrometry_file = file.replace('_coadded', '_astrometry') coadd = fits.open(astrometry_path + file) header = coadd[0].header if os.path.isfile(astrometry_path + astrometry_file): astrometry = fits.open(astrometry_path + astrometry_file, mode='update') astrometry[0].header['AIRMASS'] = header['AIRMASS'] astrometry[0].header['FILTER'] = header['FILTER'] astrometry[0].header['OBJECT'] = header['OBJECT'] gain = header['GAIN'] astrometry[0].header['GAIN'] = gain astrometry[0].header['EXPTIME'] = header['EXPTIME'] astrometry[0].header['RDNOISE'] = 0 astrometry[0].header['MJD-OBS'] = header['MJD-OBS'] ff.add_log(astrometry, 'Refined astrometric solution using Astrometry.net') coadd.close() astrometry.close() print() p.add_output_path(obj=obj, key=file[0] + '_astrometry_image', path=astrometry_path + astrometry_file) if file != template_file: shutil.copy(astrometry_path + astrometry_file, astrometry_path + astrometry_file.replace('.fits', '_orig.fits')) shutil.copy(astrometry_path + template_file, astrometry_path + template_file.replace('.fits', '_orig.fits')) ff.align(comparison=astrometry_path + astrometry_file, template=astrometry_path + template_file, comparison_output=astrometry_path + astrometry_file, template_output=astrometry_path + template_file) else: print(f'No astrometry file corresponding to {file} found.') # TODO: This will break if more than two filters have been used on this observation. # Rewrite align to work with one template and multiple comparisons. if params['astrometry_tweak']: tweak_final(epoch=obj, sextractor_path=sextractor_path, destination=astrometry_path, instrument='FORS2', show=show, output_suffix='astrometry_tweaked', input_suffix='astrometry', stars_only=True, specific_star=False, path_add='tweaked_image', manual=False) tweak_final(epoch=obj, sextractor_path=sextractor_path, destination=astrometry_path, instrument='FORS2', show=show, output_suffix='astrometry_tweaked_hg', input_suffix='astrometry', stars_only=True, specific_star=True, path_add='tweaked_image_hg', manual=False) if params['manual_astrometry']: tweak_final(epoch=obj, sextractor_path=sextractor_path, destination=astrometry_path, instrument='FORS2', show=show, output_suffix='astrometry_tweaked_manual', input_suffix='astrometry', stars_only=True, path_add='tweaked_image_manual', manual=True)
def main(epoch, test_name, sex_x_col, sex_y_col, sex_ra_col, sex_dec_col, sex_flux_col, stars_only, show_plots, mag_range_sex_lower, mag_range_sex_upper, pix_tol, instrument): print("\nExecuting Python script pipeline_fors2/9-zeropoint.py, with:") print(f"\tepoch {epoch}") print() properties = p.object_params_fors2(epoch) outputs = p.object_output_params(obj=epoch, instrument='fors2') proj_paths = p.config output = properties['data_dir'] + '9-zeropoint/' mkdir_check(output) output_std = output + 'std/' mkdir_check(output_std) mkdir_check(output_std) std_path = properties['data_dir'] + 'calibration/std_star/' filters = outputs['filters'] std_field_path = proj_paths['top_data_dir'] + "std_fields/" std_fields = list( filter(lambda path: os.path.isdir(path), os.listdir(std_field_path))) print('Standard fields with available catalogues:') print(std_fields) print() cat_names = photometry_catalogues for fil in filters: print('Doing filter', fil) f = fil[0] # Obtain zeropoints from FRB field, if data is available. print( f"\nDetermining science-field zeropoints for {epoch}, filter {fil}:\n" ) if f"{f}_zeropoints" in outputs: zeropoints = outputs[f"{f}_zeropoints"] else: zeropoints = {} zeropoints["science_field"] = {} for cat_name in cat_names: zeropoints["science_field"][ cat_name], _ = photometry.zeropoint_science_field( epoch=epoch, instrument=instrument, test_name=None, sex_x_col='XPSF_IMAGE', sex_y_col='YPSF_IMAGE', sex_ra_col='ALPHAPSF_SKY', sex_dec_col='DELTAPSF_SKY', sex_flux_col='FLUX_PSF', stars_only=True, star_class_col='CLASS_STAR', star_class_tol=0.95, show_plots=False, mag_range_sex_lower=-100. * units.mag, mag_range_sex_upper=100. * units.mag, pix_tol=5. * units.pixel, separate_chips=True, cat_name=cat_name) # Obtain zeropoints from available standard fields and available data. print( f"\nDetermining standard-field zeropoints for {epoch}, filter {fil}\n" ) zeropoints["standard_field"] = {} fil_path = std_path + fil + '/' if os.path.isdir(fil_path): fields = filter(lambda d: os.path.isdir(fil_path + d), os.listdir(fil_path)) output_path_fil_std = output_std + '/' + fil + '/' mkdir_check(output_path_fil_std) for field in fields: zeropoints["standard_field"][field] = {} ra = float(field[field.find("RA") + 2:field.find("_")]) dec = float(field[field.find("DEC") + 3:]) print("Looking for photometry data in field " + field + ":") mkdir_check(std_field_path + field) field_path = fil_path + field + '/' output_path = output_path_fil_std + field + '/' std_cat_path = std_field_path + field + '/' std_properties = p.load_params(field_path + 'params.yaml') use_sex_star_class = std_properties['use_sex_star_class'] # Cycle through the three catalogues used to determine zeropoint, in order of preference. for cat_name in cat_names: # Check for photometry on-disk in the relevant catalogue; if none present, attempt to retrieve from # online archive. print(f"In {cat_name}:") output_path_cat = output_path + cat_name cat_path = f"{std_cat_path}{cat_name}/{cat_name}.csv" if not (os.path.isdir(std_cat_path + cat_name) ) or os.path.isfile(cat_path): print( "None found on disk. Attempting retrieval from archive..." ) if update_std_photometry(ra=ra, dec=dec, cat=cat_name) is None: print("\t\tNo data found in archive.") zeropoints["standard_field"][field][ cat_name] = None continue column_names = cat_columns(cat=cat_name, f=f) cat_ra_col = column_names['ra'] cat_dec_col = column_names['dec'] cat_mag_col = column_names['mag_psf'] if not use_sex_star_class: star_class_col = column_names['class_star'] else: star_class_col = 'CLASS_STAR' cat_type = 'csv' sextractor_path = field_path + 'sextractor/_psf-fit.cat' image_path = field_path + '3-trimmed/standard_trimmed_img_up.fits' star_class_tol = std_properties['star_class_tol'] now = time.Time.now() now.format = 'isot' test_name = str(now) + '_' + test_name mkdir_check(properties['data_dir'] + '/analysis/zeropoint/') mkdir_check(output_path) exp_time = ff.get_exp_time(image_path) print('SExtractor catalogue path:', sextractor_path) print('Image path:', image_path) print('Catalogue name:', cat_name) print('Catalogue path:', cat_path) print('Class star column:', star_class_col) print('Output:', output_path_cat) print('Exposure time:', exp_time) print("Use sextractor class star:", use_sex_star_class) zeropoints["standard_field"][field][ cat_name] = photometry.determine_zeropoint_sextractor( sextractor_cat=sextractor_path, image=image_path, cat_path=cat_path, cat_name=cat_name, output_path=output_path_cat, show=show_plots, cat_ra_col=cat_ra_col, cat_dec_col=cat_dec_col, cat_mag_col=cat_mag_col, sex_ra_col=sex_ra_col, sex_dec_col=sex_dec_col, sex_x_col=sex_x_col, sex_y_col=sex_y_col, dist_tol=pix_tol, flux_column=sex_flux_col, mag_range_sex_upper=mag_range_sex_upper, mag_range_sex_lower=mag_range_sex_lower, stars_only=stars_only, star_class_tol=star_class_tol, star_class_col=star_class_col, exp_time=exp_time, y_lower=0, cat_type=cat_type, ) output_dict = {f + '_zeropoints': zeropoints} p.add_output_values(obj=epoch, instrument='FORS2', params=output_dict) outputs = p.object_output_params(obj=epoch, instrument='fors2') output_path_final = output + "collated/" mkdir_check(output_path_final) print("Collating zeropoints...") for fil in filters: print(f"For {fil}:") f = fil[0] output_path_final_f = output_path_final + fil + "/" mkdir_check(output_path_final_f) zeropoints = outputs[f"{f}_zeropoints"] airmass_sci = outputs[f"{f}_airmass_mean"] airmass_sci_err = outputs[f"{f}_airmass_err"] extinction = outputs[f"{f}_extinction"] extinction_err = outputs[f"{f}_extinction_err"] zeropoint_tbl = table.Table( dtype=[("type", 'S15'), ("field", 'S25'), ( "cat", 'S10'), ("zeropoint", float), ("zeropoint_err", float), ("airmass", float), ("airmass_err", float), ("extinction", float), ("extinction_err", float), ("zeropoint_ext_corr", float), ("zeropoint_ext_corr_err", float), ("n_matches", float)]) if f"provided" in zeropoints: print(f"\tProvided:") zeropoint_prov = zeropoints["provided"] zeropoint_tbl.add_row([ "provided", "N/A", "N/A", zeropoint_prov["zeropoint"], zeropoint_prov["zeropoint_err"], 0.0, 0.0, extinction, extinction_err, zeropoint_prov["zeropoint"], zeropoint_prov["zeropoint_err"], 0 ]) print(f"\tScience field:") for cat_name in zeropoints["science_field"]: print(f"\t\t{cat_name}") zeropoint_sci = zeropoints["science_field"][cat_name] if zeropoint_sci is not None: zeropoint_corrected = zeropoint_sci[ "zeropoint"] + extinction * airmass_sci print("\t\t\t", zeropoint_sci["zeropoint"], extinction, airmass_sci) zeropoint_corrected_err = zeropoint_sci[ "zeropoint_err"] + error_product( value=extinction * airmass_sci, measurements=[extinction, airmass_sci], errors=[extinction_err, airmass_sci_err]) zp_cat = table.Table.read(zeropoint_sci["matches_cat_path"]) zeropoint_tbl.add_row([ "science_field", epoch[:-2], cat_name, zeropoint_sci["zeropoint"], zeropoint_sci["zeropoint_err"], airmass_sci, airmass_sci_err, extinction, extinction_err, zeropoint_corrected, zeropoint_corrected_err, len(zp_cat) ]) plt.scatter(zp_cat["mag_cat"], zp_cat["mag"], c='green') plt.errorbar(zp_cat["mag_cat"], zp_cat["mag"], yerr=zp_cat["mag_err"], linestyle="None", c='black') plt.plot(zp_cat["mag_cat"], zp_cat["mag_cat"] - zeropoint_sci["zeropoint"], c="violet") plt.title(f"{fil}: science-field, {cat_name}") plt.figtext( 0, 0.01, f'zeropoint - kX: {zeropoint_sci["zeropoint"]} ± {zeropoint_sci["zeropoint_err"]}\n' f'zeropoint: {zeropoint_corrected} ± {zeropoint_corrected_err}\n' f'num stars: {len(zp_cat)}') plt.gca().set_aspect('equal', adjustable='box') plt.savefig(output_path_final_f + f"zeropoint_science_{cat_name}.pdf") plt.show() plt.close() print("\tStandard fields:") for field in zeropoints["standard_field"]: print(f"\t\t{field}") for cat_name in zeropoints["standard_field"][field]: print(f"\t\t\t{cat_name}") zeropoint_std = zeropoints["standard_field"][field][cat_name] if zeropoint_std is not None: zeropoint_corrected = zeropoint_std[ "zeropoint"] + extinction * zeropoint_std["airmass"] print("\t\t\t\t", zeropoint_std["zeropoint"], extinction, zeropoint_std["airmass"]) zeropoint_corrected_err = zeropoint_std[ "zeropoint_err"] + error_product( value=extinction * zeropoint_std["airmass"], measurements=[ extinction, zeropoint_std["airmass"] ], errors=[extinction_err, 0.0]) zp_cat = table.Table.read( zeropoint_std["matches_cat_path"]) zeropoint_tbl.add_row([ "standard_field", field, cat_name, zeropoint_std["zeropoint"], zeropoint_std["zeropoint_err"], zeropoint_std["airmass"], 0.0, extinction, extinction_err, zeropoint_corrected, zeropoint_corrected_err, len(zp_cat) ]) plt.scatter(zp_cat["mag_cat"], zp_cat["mag"], c='green') plt.errorbar(zp_cat["mag_cat"], zp_cat["mag"], yerr=zp_cat["mag_err"], linestyle="None", c='black') plt.plot(zp_cat["mag_cat"], zp_cat["mag_cat"] - zeropoint_std["zeropoint"], c="violet") plt.title(f"{fil}: standard-field {field}, {cat_name}") plt.gca().set_aspect('equal', adjustable='box') plt.figtext( 0, 0.01, f'zeropoint - kX: {zeropoint_std["zeropoint"]} ± {zeropoint_std["zeropoint_err"]}\n' f'zeropoint: {zeropoint_corrected} ± {zeropoint_corrected_err}\n' f'num stars: {len(zp_cat)}') plt.savefig(output_path_final_f + f"zeropoint_standard_{field}_{cat_name}.pdf") plt.show() plt.close() zeropoint_tbl["selection_index"] = zeropoint_tbl[ "n_matches"] / zeropoint_tbl["zeropoint_ext_corr_err"] best_arg = np.argmax(zeropoint_tbl["selection_index"]) print("Best zeropoint:") best_zeropoint = zeropoint_tbl[best_arg] print(best_zeropoint) zeropoints = outputs[f + '_zeropoints'] zeropoints['best'] = { "zeropoint": float(best_zeropoint['zeropoint']), "zeropoint_err": float(best_zeropoint["zeropoint_err"]), "airmass": float(best_zeropoint["airmass"]), "airmass_err": float(best_zeropoint["airmass_err"]), "type": str(best_zeropoint["type"]), "catalogue": str(best_zeropoint["cat"]) } output_dict = {f + '_zeropoints': zeropoints} p.add_output_values(obj=epoch, instrument='FORS2', params=output_dict) zeropoint_tbl.write(output_path_final_f + "zeropoints.csv", format="ascii.csv", overwrite=True)
def main(data_title, sextractor_path, origin, destination): print( "\nExecuting Python script pipeline_fors2/4-divide_by_exp_time.py, with:" ) print(f"\tepoch {data_title}") print(f"\tsextractor path {sextractor_path}") print(f"\torigin directory {origin}") print(f"\tdestination directory {destination}") print() properties = p.object_params_fors2(data_title) outputs = p.object_output_params(obj=data_title, instrument='FORS2') data_dir = properties['data_dir'] if sextractor_path is not None: if not os.path.isdir(sextractor_path): os.mkdir(sextractor_path) do_sextractor = True else: do_sextractor = False origin_path = data_dir + origin destination_path = data_dir + destination u.mkdir_check(destination_path) u.mkdir_check(destination_path + "science/") u.mkdir_check(destination_path + "backgrounds/") filters = outputs['filters'] for fil in filters: u.mkdir_check(destination_path + "science/" + fil) u.mkdir_check(destination_path + "backgrounds/" + fil) if do_sextractor: if not os.path.isdir(sextractor_path + fil): os.mkdir(sextractor_path + fil) files = os.listdir(origin_path + "science/" + fil + "/") for file_name in files: if file_name[-5:] == '.fits': science_origin = origin_path + "science/" + fil + "/" + file_name science_destination = destination_path + "science/" + fil + "/" + file_name.replace( "trim", "norm") background_origin = origin_path + "backgrounds/" + fil + "/" + file_name.replace( "SCIENCE_REDUCED", "PHOT_BACKGROUND_SCI") background_destination = destination_path + "backgrounds/" + fil + "/" + \ file_name.replace("SCIENCE_REDUCED", "PHOT_BACKGROUND_SCI").replace("trim", "norm") print(science_origin) # Divide by exposure time to get an image in counts/second. ff.divide_by_exp_time(file=science_origin, output=science_destination) ff.divide_by_exp_time(file=background_origin, output=background_destination) if do_sextractor: copyfile( science_destination, sextractor_path + fil + "/" + file_name.replace("trim", "norm")) if os.path.isfile(origin_path + data_title + '.log'): copyfile(origin_path + data_title + '.log', destination_path + data_title + ".log") u.write_log(path=destination_path + data_title + ".log", action=f'Divided by exposure time.')
def main(data_title: 'str', delete_output: bool = True): print( "\nExecuting Python script pipeline_fors2/2-sort_after_esoreflex.py, with:" ) print(f"\tepoch {data_title}") print() eso_dir = p.config['esoreflex_output_dir'] if os.path.isdir(eso_dir): obj_params = p.object_params_fors2(data_title) data_dir = obj_params['data_dir'] destination = data_dir + "2-sorted/" date = None output_values = p.object_output_params(obj=data_title, instrument='FORS2') mjd = int(output_values['mjd_obs']) obj = output_values['object'] print( f"Looking for data with object '{obj}' and MJD of observation {mjd} inside {eso_dir}" ) # Look for files with the appropriate object and MJD, as recorded in output_values # List directories in eso_output_dir; these are dates on which data was reduced using ESOReflex. eso_dirs = filter(lambda d: os.path.isdir(eso_dir + "/" + d), os.listdir(eso_dir)) for directory in eso_dirs: directory += "/" # List directories within 'reduction date' directories. # These should represent individual images reduced. print(f"Searching {eso_dir + directory}") eso_subdirs = filter( lambda d: os.path.isdir(eso_dir + directory + d) and "FORS2" in d, os.listdir(eso_dir + directory)) for subdirectory in eso_subdirs: subdirectory += "/" subpath = eso_dir + "/" + directory + subdirectory print(f"\tSearching {subpath}") # Get the files within the image directory. files = filter(lambda d: os.path.isfile(subpath + d), os.listdir(subpath)) for file_name in files: # Retrieve the target object name from the fits file. file_path = subpath + file_name file = fits.open(file_path) file_obj = ff.get_object(file) file_mjd = int(ff.get_header_attribute(file, 'MJD-OBS')) file_filter = ff.get_filter(file) # Check the object name and observation date against those of the epoch we're concerned with. if file_obj == obj and file_mjd == mjd: # Check which type of file we have. date = directory[:-1] if file_name[-28:] == "PHOT_BACKGROUND_SCI_IMG.fits": file_destination = f"{destination}/backgrounds/" suffix = "PHOT_BACKGROUND_SCI_IMG.fits" elif file_name[-25:] == "OBJECT_TABLE_SCI_IMG.fits": file_destination = f"{destination}/obj_tbls/" suffix = "OBJECT_TABLE_SCI_IMG.fits" elif file_name[-24:] == "SCIENCE_REDUCED_IMG.fits": file_destination = f"{destination}/science/" suffix = "SCIENCE_REDUCED_IMG.fits" else: file_destination = f"{destination}/sources/" suffix = "SOURCES_SCI_IMG.fits" # Make this directory, if it doesn't already exist. u.mkdir_check(file_destination) # Make a subdirectory by filter. file_destination += file_filter + "/" u.mkdir_check(file_destination) # Title new file. file_destination += f"{data_title}_{subdirectory[:-1]}_{suffix}" # Copy file to new location. print( f"Copying: {file_path} to \n\t {file_destination}") file.writeto(file_destination, overwrite=True) if delete_output and os.path.isfile(file_destination): os.remove(file_path) if not os.path.isfile(f"{destination}/{data_title}.log"): sh.copy(f"{data_dir}0-data_with_raw_calibs/{data_title}.log", f"{destination}/{data_title}.log") u.write_log(path=destination + data_title + ".log", action=f'Data reduced with ESOReflex.', date=date) u.write_log(path=destination + data_title + ".log", action='Files sorted using 2-sort_after_esoreflex.sh') else: print(f"ESO output directory '{eso_dir}' not found.") exit(1)