def main(data_dir, data_title, destination, fil, object): table = (destination + '/' + fil, science_only=False) fil = fil.replace('/', '') header_file = destination + '/' + fil + '_template.hdr' header_stream = open(header_file, 'r') header = header_stream.readlines() header_stream.close() # Transfer and/or modify header information params = p.object_output_params(data_title, instrument='IMACS') airmass = table['airmass'].mean() saturate = table['saturate'].mean() old_gain = table['gain'].mean() n_frames = params[fil + '_n_exposures'] gain = gain_median_combine(old_gain=old_gain, n_frames=n_frames) header.insert(-1, f'AIRMASS = {airmass}\n') header.insert(-1, f'FILTER = {fil}\n') header.insert(-1, f'OBJECT = {object}\n') header.insert(-1, f'EXPTIME = 1.\n') header.insert(-1, f'GAIN = {gain}\n') header.insert(-1, f'SATURATE= {saturate}\n') header.insert(-1, f'MJD-OBS = {float(np.nanmean(table["mjd_obs"]))}\n') os.remove(header_file) with open(header_file, 'w') as file: file.writelines(header) p.add_output_path(obj=data_title, key=fil + '_subtraction_image', path=destination + '/' + fil + '_coadded.fits', instrument='IMACS')
def main(ob, path): print("\nExecuting Python script pipeline_fors2/9-esorex_zeropoint_prep.py, with:") print(f"\tepoch {ob}") print(f"\tcalibration data path {path}") print() output = object_output_params(obj=ob, instrument='FORS2') filters = output['filters'] for f in filters: mkdir_check(path + '/' + f)
def main(obj, test, mag_min, mag_max, increment, instrument): properties = p.object_params_instrument(obj, instrument=instrument) output = p.object_output_params(obj=obj, instrument=instrument) paths = p.object_output_paths(obj=obj, instrument=instrument) synth_path = properties['data_dir'] + 'synthetic/' u.mkdir_check(synth_path) synth_path = synth_path + 'frb_position/' u.mkdir_check(synth_path) now = time.Time.now() now.format = 'isot' synth_path += f'range_{now}/' u.mkdir_check(synth_path) filters = output['filters'] for magnitude in np.arange(mag_min, mag_max, increment): magnitudes = [] for i in range(len(filters)): magnitudes.append(magnitude) test_spec = test + '_' + str(u.round_to_sig_fig(magnitude, 4)) test_path = synth_path + test_spec + '/' ph.insert_synthetic_at_frb(obj=properties, test_path=test_path, filters=filters, magnitudes=magnitudes, add_path=False, psf=True, output_properties=output, instrument=instrument, paths=paths) p.add_output_path(obj=obj, key='subtraction_image_synth_frb_range', path=synth_path, instrument=instrument)
def main(obj, test, n, filter_dist, instrument, limit): properties = p.object_params_instrument(obj, instrument=instrument) burst_properties = p.object_params_frb(obj=obj[:-2]) output = p.object_output_params(obj=obj, instrument=instrument) paths = p.object_output_paths(obj=obj, instrument=instrument) z = burst_properties['z'] mjd_burst = burst_properties['mjd_burst'] ebv_mw = burst_properties['dust_ebv'] mjd_obs = properties['mjd'] synth_path = properties['data_dir'] + 'synthetic/' u.mkdir_check(synth_path) synth_path = synth_path + 'sn_random/' u.mkdir_check(synth_path) epoch = mjd_obs - mjd_burst f_0 = filter_dist[0] hg_ra = burst_properties['hg_ra'] hg_dec = burst_properties['hg_dec'] burst_ra = burst_properties['burst_ra'] burst_dec = burst_properties['burst_dec'] image_path = paths[f_0 + '_' + properties['subtraction_image']] image = fits.open(image_path) wcs_info = wcs.WCS(image[0].header) burst_x, burst_y = wcs_info.all_world2pix(burst_ra, burst_dec, 0) now = time.Time.now() now.format = 'isot' synth_path += test + '_' + str(now) + '/' u.mkdir_check(synth_path) filters = output['filters'] psf_models = [] for i, f in enumerate(filters): sn.register_filter(f=f, instrument=instrument) psf_model = fits.open(paths[f[0] + '_psf_model']) psf_models.append(psf_model) for i in range(n): test_spec = test + '_' + str(i) test_path = synth_path + test_spec + '/' magnitudes = [] mags_filters, model, x, y, tbl = sn.random_light_curves_type_ia( filters=filters, image=image, hg_ra=hg_ra, hg_dec=hg_dec, z=z, ebv_mw=ebv_mw, output_path=test_path, output_title=test, limit=limit, x=burst_x, y=burst_y, ra=burst_ra, dec=burst_dec) days = mags_filters['days'] for f in filters: magnitude = sn.magnitude_at_epoch(epoch=epoch, days=days, mags=mags_filters[f]) print(f, 'mag:', magnitude) magnitudes.append(magnitude) ph.insert_synthetic(x=float(x), y=float(y), obj=properties, test_path=test_path, filters=filters, magnitudes=magnitudes, suffix='sn_random_random_ia', extra_values=tbl, paths=paths, output_properties=output, psf_models=psf_models, instrument=instrument) p.add_output_path(obj=obj, key='subtraction_image_synth_sn_random_ia', path=synth_path, instrument=instrument)
def main(epoch, origin, destination): print("\nExecuting Python script pipeline_fors2/4.1-insert_test_synth.py, with:") print(f"\tepoch {epoch}") print(f"\torigin directory {origin}") print(f"\tdestination directory {destination}") print() epoch_params = p.object_params_fors2(obj=epoch) outputs = p.object_output_params(obj=epoch, instrument='FORS2') data_dir = epoch_params['data_dir'] insert = epoch_params['test_synths'] origin_path = data_dir + "analysis/sextractor/" + origin destination_path = data_dir + destination u.mkdir_check(destination_path) u.mkdir_check(destination_path + "science/") u.mkdir_check(destination_path + "backgrounds/") filters = outputs['filters'] for fil in filters: f = fil[0] path_fil_output = destination_path + "science/" + fil + "/" path_fil_input = origin_path + fil + "/" u.mkdir_check(path_fil_output) u.mkdir_check(destination_path + "backgrounds/" + fil) zeropoint, _, airmass, _, extinction, _ = ph.select_zeropoint(obj=epoch, filt=fil, instrument='fors2', outputs=outputs) print(path_fil_input) # print(os.listdir(path_fil_input)) for fits_file in filter(lambda f: f.endswith("_norm.fits"), os.listdir(path_fil_input)): print(fits_file) path_fits_file_input = path_fil_input + fits_file path_fits_file_output = path_fil_output + fits_file path_psf_model = path_fits_file_input.replace(".fits", "_psfex.psf") try: ph.insert_point_sources_to_file(file=path_fits_file_input, x=array(insert["ra"]), y=array(insert["dec"]), mag=insert[f"{f}_mag"], output=path_fits_file_output, zeropoint=zeropoint, extinction=extinction, airmass=airmass, world_coordinates=True, psf_model=path_psf_model ) except ValueError: ph.insert_point_sources_to_file(file=path_fits_file_input, x=array(insert["ra"]), y=array(insert["dec"]), mag=insert[f"{f}_mag"], output=path_fits_file_output, zeropoint=zeropoint, extinction=extinction, airmass=airmass, world_coordinates=True, fwhm=fits.open(path_psf_model)[1].header['PSF_FWHM'] ) if os.path.isfile(origin_path + epoch + '.log'): copyfile(origin_path + epoch + '.log', destination_path + epoch + ".log") u.write_log(path=destination_path + epoch + ".log", action=f'Divided by exposure time.')
def main(data_dir, data_title, origin, destination, all_synths): print("\nExecuting Python script pipeline_fors2/5-background_subtract.py, with:") print(f"\tepoch {data_title}") print(f"\torigin directory {origin}") print(f"\tdestination directory {destination}") print() methods = ["ESO backgrounds only", "SExtractor backgrounds only", "polynomial fit", "Gaussian fit", "median value"] if all_synths: frame = 56 method = "polynomial fit" degree = 5 do_mask = True local = True global_sub = False trim_image = False recorrect_subbed = True eso_back = False else: frame = 200 # frame_arcsec = 30 * units.arcsec # frame_deg = frame_arcsec.to(units.deg) eso_back = False _, method = u.select_option(message="Please select the background subtraction method.", options=methods, default="polynomial fit") degree = None if method == "polynomial fit": degree = u.user_input(message=f"Please enter the degree of {method} to use:", typ=int, default=3) elif method == "ESO backgrounds only": eso_back = True do_mask = False if method not in ["ESO backgrounds only", "SExtractor backgrounds only", "median value"]: do_mask = u.select_yn(message="Mask sources using SExtractor catalogue?", default=True) if method in ["polynomial fit", "Gaussian fit"]: local = u.select_yn(message="Use a local fit?", default=True) else: local = False global_sub = False trim_image = False recorrect_subbed = False if local: global_sub = u.select_yn(message="Subtract local fit from entire image?", default="n") if not global_sub: trim_image = u.select_yn(message="Trim images to subtracted region?", default="y") recorrect_subbed = u.select_yn(message="Re-normalise background of subtracted region?", default="y") # if not eso_back and method != "SExtractor backgrounds only": # eso_back = u.select_yn(message="Subtract ESO Reflex fitted backgrounds first?", default=False) outputs = p.object_output_params(data_title, instrument='FORS2') data_dir = u.check_trailing_slash(data_dir) destination = u.check_trailing_slash(destination) destination = data_dir + destination u.mkdir_check_nested(destination) origin = u.check_trailing_slash(origin) science_origin = data_dir + origin + "science/" print(science_origin) filters = outputs['filters'] frb_params = p.object_params_frb(obj=data_title[:-2]) epoch_params = p.object_params_fors2(obj=data_title) background_origin_eso = "" if eso_back: background_origin_eso = data_dir + "/" + origin + "/backgrounds/" if method == "SExtractor backgrounds only": background_origin = f"{data_dir}{origin}backgrounds_sextractor/" elif method == "polynomial fit": background_origin = f"{destination}backgrounds/" # f"{destination}backgrounds_{method.replace(' ', '')}_degree_{degree}_local_{local}_globalsub_{global_sub}/" else: background_origin = f"{destination}backgrounds/" # f"{destination}backgrounds_{method.replace(' ', '')}_local_{local}_globalsub_{global_sub}/" trimmed_path = "" if trim_image: trimmed_path = f"{data_dir}{origin}trimmed_to_background/" u.mkdir_check_nested(trimmed_path) ra = frb_params["burst_ra"] dec = frb_params["burst_dec"] if all_synths: ras = epoch_params["test_synths"]["ra"] decs = epoch_params["test_synths"]["dec"] else: ras = [ra] decs = [dec] for fil in filters: trimmed_path_fil = "" if trim_image: trimmed_path_fil = f"{trimmed_path}{fil}/" u.mkdir_check(trimmed_path_fil) background_fil_dir = f"{background_origin}{fil}/" u.mkdir_check_nested(background_fil_dir) science_destination_fil = f"{destination}science/{fil}/" u.mkdir_check_nested(science_destination_fil) files = os.listdir(science_origin + fil + "/") for file_name in files: if file_name.endswith('.fits'): new_file = file_name.replace("norm", "bg_sub") new_path = f"{science_destination_fil}/{new_file}" print("NEW_PATH:", new_path) science = science_origin + fil + "/" + file_name # First subtract ESO Reflex background images # frame = (frame_deg / f.get_pixel_scale(file=science, astropy_units=True)[1]).to(f.pix).value if eso_back: background_eso = background_origin_eso + fil + "/" + file_name.replace("SCIENCE_REDUCED", "PHOT_BACKGROUND_SCI") ff.subtract_file(file=science, sub_file=background_eso, output=new_path) science_image = new_path if method != "ESO backgrounds only": print(ra, dec) print("Science image:", science) science_image = fits.open(science) print("Science file:", science_image) wcs_this = WCS(header=science_image[0].header) if method == "SExtractor backgrounds only": background = background_origin + fil + "/" + file_name + "_back.fits" print("Background image:", background) else: if method == "median value": print(science_image[0].data.shape) _, background_value, _ = sigma_clipped_stats(science_image[0].data) background = deepcopy(science_image) background[0].data = np.full(shape=science_image[0].data.shape, fill_value=background_value) background_path = background_origin + fil + "/" + file_name.replace("SCIENCE_REDUCED", "PHOT_BACKGROUND_MEDIAN") # Next do background fitting. else: background = deepcopy(science_image) background[0].data = np.zeros(background[0].data.shape) background_path = background_origin + fil + "/" + file_name.replace("SCIENCE_REDUCED", "PHOT_BACKGROUND_FITTED") for i, ra in enumerate(ras): dec = decs[i] x, y = wcs_this.all_world2pix(ra, dec, 0) print(x, y) bottom, top, left, right = ff.subimage_edges(data=science_image[0].data, x=x, y=y, frame=frame) if do_mask: # Produce a pixel mask that roughly masks out the true sources in the image so that # they don't get fitted. mask_max = 10 _, pixel_scale = ff.get_pixel_scale(science_image) sextractor = Table.read( f"{data_dir}analysis/sextractor/4-divided_by_exp_time/{fil}/{file_name.replace('.fits', '_psf-fit.cat')}", format='ascii.sextractor') weights = np.ones(shape=science_image[0].data.shape) for obj in filter( lambda o: left < o["X_IMAGE"] < right and bottom < o["Y_IMAGE"] < top, sextractor): mask_rad = min(int(obj["A_WORLD"] * obj["KRON_RADIUS"] / pixel_scale), mask_max) x_prime = int(np.round(obj["X_IMAGE"])) y_prime = int(np.round(obj["Y_IMAGE"])) weights[y_prime - mask_rad:y_prime + mask_rad, x_prime - mask_rad:x_prime + mask_rad] = 0.0 plt.imshow(weights, origin="lower") plt.savefig( background_origin + fil + "/" + file_name.replace("norm.fits", "mask.png")) else: weights = None background_this = fit_background_fits(image=science_image, model_type=method[:method.find(" ")], deg=degree, local=local, global_sub=global_sub, centre_x=x, centre_y=y, frame=frame, weights=weights) background[0].data += background_this[0].data if recorrect_subbed: offset = get_median_background(image=science, ra=epoch_params["renormalise_centre_ra"], dec=epoch_params["renormalise_centre_dec"], frame=50, show=False, output=new_path[ :new_path.find("bg_sub")] + "renorm_patch_") print("RECORRECT_SUBBED:", recorrect_subbed) print("SUBTRACTING FROM BACKGROUND:", offset) print(bottom, top, left, right) print(background[0].data[bottom:top, left:right].shape) print(np.median(background[0].data[bottom:top, left:right])) background[0].data[bottom:top, left:right] -= offset print(np.median(background[0].data[bottom:top, left:right])) if trim_image: print("TRIMMED_PATH_FIL:", trimmed_path_fil) science_image = ff.trim_file(path=science_image, left=left, right=right, top=top, bottom=bottom, new_path=trimmed_path_fil + file_name.replace( "norm.fits", "trimmed_to_back.fits")) print("Science after trim:", science_image) background = ff.trim_file(path=background, left=left, right=right, top=top, bottom=bottom, new_path=background_path) print("Writing background to:") print(background_path) background.writeto(background_path, overwrite=True) print("SCIENCE:", science_image) print("BACKGROUND:", background) subbed = ff.subtract_file(file=science_image, sub_file=background, output=new_path) # # TODO: check if regions overlap # # plt.hist(subbed[0].data[int(y - frame + 1):int(y + frame - 1), # int(x - frame + 1):int(x + frame - 1)].flatten(), # bins=10) # plt.savefig(new_path[:new_path.find("bg_sub")] + "histplot.png") # plt.close() copyfile(data_dir + "/" + origin + "/" + data_title + ".log", destination + data_title + ".log") u.write_log(path=destination + data_title + ".log", action=f'Backgrounds subtracted using 4-background_subtract.py with method {method}\n')
def main(field, destination, epoch, instrument, template_instrument, comparison_type): if destination[-1] != '/': destination = destination + '/' p.refresh_params_frbs() types = ['normal', 'synth_random', 'synth_frb'] if comparison_type not in types: raise ValueError(comparison_type + ' is not a valid synthetic argument; choose from ' + str(types)) if comparison_type == 'normal': comparison_type = '' else: comparison_type = '_' + comparison_type params = p.object_params_frb(field) u.mkdir_check(f'{params["data_dir"]}subtraction/') destination_path = f'{params["data_dir"]}subtraction/{destination}/' u.mkdir_check(destination_path) comparison_title = f'{field}_{epoch}' comparison_paths = p.object_output_paths(obj=comparison_title, instrument=instrument) comparison_params = p.object_params_instrument(obj=comparison_title, instrument=instrument) params = p.object_params_frb(field) template_epoch = params['template_epoch_' + template_instrument.lower()] template_title = f'{field}_{template_epoch}' template_paths = p.object_output_paths(obj=template_title, instrument=template_instrument) template_outputs = p.object_output_params(obj=template_title, instrument=template_instrument) template_params = p.object_params_instrument( obj=template_title, instrument=template_instrument) filters = params['filters'] for f in filters: values = {} f_0 = f[0] destination_path_filter = f'{destination_path}{f}/' u.mkdir_check(destination_path_filter) # COMPARISON IMAGE: comparison_image_name = comparison_params[ 'subtraction_image'] + comparison_type # Get path to comparison image from parameter .yaml file if f'{f_0}_{comparison_image_name}' in comparison_paths: comparison_origin = comparison_paths[ f'{f_0}_{comparison_image_name}'] elif f'{f_0.lower()}_{comparison_image_name}' in comparison_paths: comparison_origin = comparison_paths[ f'{f_0.lower()}_{comparison_image_name}'] else: raise ValueError( f'{f_0.lower()}_{comparison_image_name} not found in {comparison_title} paths' ) comparison_destination = f'{comparison_title}_comparison.fits' if comparison_type != '': shutil.copyfile( comparison_origin.replace('.fits', '.csv'), destination_path_filter + comparison_destination.replace('.fits', '.csv')) print('Copying comparison image') print('From:') print('\t', comparison_origin) print('To:') print(f'\t {destination_path}{f}/{comparison_destination}') shutil.copy( comparison_params['data_dir'] + 'output_values.yaml', f'{destination_path}{f}/{comparison_title}_comparison_output_values.yaml' ) shutil.copy( comparison_params['data_dir'] + 'output_values.json', f'{destination_path}{f}/{comparison_title}_comparison_output_values.json' ) shutil.copy(comparison_origin, f'{destination_path}{f}/{comparison_destination}') values['comparison_file'] = comparison_origin # TEMPLATE IMAGE if template_instrument != 'FORS2' and template_instrument != 'XSHOOTER': f_0 = f_0.lower() template_image_name = template_params[ 'subtraction_image'] + comparison_type if f'{f_0}_{template_image_name}' in template_paths: template_origin = template_paths[f'{f_0}_{template_image_name}'] elif f'{f_0.lower()}_{template_image_name}' in template_paths: template_origin = template_paths[ f'{f_0.lower()}_{template_image_name}'] else: raise ValueError( f'{f_0.lower()}_{template_image_name} not found in {template_title} paths' ) fwhm_template = template_outputs[f_0 + '_fwhm_pix'] template_destination = f'{template_title}_template.fits' print('Copying template') print('From:') print('\t', template_origin) print('To:') print(f'\t {destination_path}{f}/{template_destination}') shutil.copy( template_params['data_dir'] + 'output_values.yaml', f'{destination_path}{f}/{template_title}_template_output_values.yaml' ) shutil.copy( template_params['data_dir'] + 'output_values.json', f'{destination_path}{f}/{template_title}_template_output_values.json' ) shutil.copy(template_origin, f'{destination_path}{f}/{template_destination}') values['template_file'] = comparison_origin p.add_params(f'{destination_path}{f}/output_values.yaml', values)
def main(obj, test, n, filter_dist, sn_type, instrument, limit): properties = p.object_params_instrument(obj, instrument=instrument) burst_properties = p.object_params_frb(obj=obj[:-2]) output = p.object_output_params(obj=obj, instrument=instrument) paths = p.object_output_paths(obj=obj, instrument=instrument) z = burst_properties['z'] mjd_burst = burst_properties['mjd_burst'] ebv_mw = burst_properties['dust_ebv'] mjd_obs = properties['mjd'] synth_path = properties['data_dir'] + 'synthetic/' u.mkdir_check(synth_path) synth_path = synth_path + 'sn_random/' u.mkdir_check(synth_path) epoch = mjd_obs - mjd_burst f_0 = filter_dist[0] hg_ra = burst_properties['hg_ra'] hg_dec = burst_properties['hg_dec'] image_path = paths[f_0 + '_' + properties['subtraction_image']] image = fits.open(image_path) now = time.Time.now() now.format = 'isot' test += sn_type + '_' synth_path += test + str(now) + '/' u.mkdir_check(synth_path) filters = output['filters'] if 'FRB190608' in obj: limit = 45 psf_models = [] for f in filters: sn.register_filter(f=f) psf_model = fits.open(paths[f[0] + '_psf_model']) psf_models.append(psf_model) for i in range(n): test_spec = test + str(i) test_path = synth_path + test_spec + '/' magnitudes = [] mags_filters, model, x, y, tbl = sn.random_light_curves( sn_type=sn_type, filters=filters, image=image, hg_ra=hg_ra, hg_dec=hg_dec, z=z, ebv_mw=ebv_mw, output_path=test_path, output_title=test, limit=limit) days = mags_filters['days'] for f in filters: magnitude = sn.magnitude_at_epoch(epoch=epoch, days=days, mags=mags_filters[f]) print(f, 'mag:', magnitude) magnitudes.append(magnitude) ph.insert_synthetic(x=x, y=y, obj=properties, test_path=test_path, filters=filters, magnitudes=magnitudes, suffix='sn_random_random_' + sn_type, extra_values=tbl, paths=paths, output_properties=output, psf_models=psf_models, instrument=instrument) p.add_output_path(obj=obj, key='subtraction_image_synth_sn_random_' + sn_type, path=synth_path, instrument=instrument)
def main(obj, test, n, mag_lower, mag_upper, colour_upper, colour_lower): properties = p.object_params_fors2(obj) output = p.object_output_params(obj=obj, instrument='FORS2') paths = p.object_output_paths(obj) burst_properties = p.object_params_frb(obj[:-2]) synth_path = properties['data_dir'] + 'synthetic/' u.mkdir_check(synth_path) synth_path = synth_path + 'random/' u.mkdir_check(synth_path) now = time.Time.now() now.format = 'isot' test = str(now) + '_' + test test_path = synth_path + test + '/' u.mkdir_check(test_path) filters = {} bluest = None bluest_lambda = np.inf for f in output['filters']: filter_properties = p.filter_params(f=f, instrument='FORS2') filters[f] = filter_properties lambda_eff = filter_properties['lambda_eff'] if lambda_eff < bluest_lambda: bluest_lambda = lambda_eff bluest = f # Insert random sources in the bluest filter. f_0 = bluest[0] output_properties = p.object_output_params(obj) fwhm = output_properties[f_0 + '_fwhm_pix'] zeropoint, _, airmass, _ = ph.select_zeropoint(obj, bluest, instrument='fors2') base_path = paths[f_0 + '_subtraction_image'] output_path = test_path + f_0 + '_random_sources.fits' _, sources = ph.insert_random_point_sources_to_file(file=base_path, fwhm=fwhm, output=output_path, n=n, airmass=airmass, zeropoint=zeropoint) p.add_output_path(obj=obj, key=f_0 + '_subtraction_image_synth_random', path=output_path) # Now insert sources at the same positions in other filters, but with magnitudes randomised. for f in filters: if f != bluest: f_0 = f[0] output_properties = p.object_output_params(obj) fwhm = output_properties[f_0 + '_fwhm_pix'] zeropoint, _, airmass, _ = ph.select_zeropoint(obj, f, instrument='fors2') base_path = paths[f_0 + '_subtraction_image'] mag = np.random.uniform(mag_lower, mag_upper, size=n) output_path = test_path + f_0 + '_random_sources.fits' ph.insert_point_sources_to_file(file=base_path, fwhm=fwhm, output=output_path, x=sources['x_0'], y=sources['y_0'], mag=mag, airmass=airmass, zeropoint=zeropoint) p.add_output_path(obj=obj, key=f_0 + '_subtraction_image_synth_random', path=output_path)
def main(origin_dir, output_dir, data_title, sextractor_path): print("\nExecuting Python script pipeline_fors2/3-trim.py, with:") print(f"\tepoch {data_title}") print(f"\torigin directory {origin_dir}") print(f"\toutput directory {output_dir}") print() # If this is None, we don't want the SExtractor components to be performed. if sextractor_path is not None: if not os.path.isdir(sextractor_path): os.mkdir(sextractor_path) do_sextractor = True print(os.getcwd()) else: do_sextractor = False if not os.path.isdir(output_dir): os.mkdir(output_dir) if not os.path.isdir(output_dir + "backgrounds/"): os.mkdir(output_dir + "backgrounds/") if not os.path.isdir(output_dir + "science/"): os.mkdir(output_dir + "science/") wdir = origin_dir + "backgrounds/" epoch_params = p.object_params_fors2(obj=data_title) outputs = p.object_output_params(obj=data_title) fils = outputs["filters"] edged = False up_left = 0 up_right = 0 up_bottom = 0 up_top = 0 dn_left = 0 dn_right = 0 dn_bottom = 0 dn_top = 0 for fil in fils: print(output_dir + "backgrounds/" + fil) if not os.path.isdir(output_dir + "backgrounds/" + fil): os.mkdir(output_dir + "backgrounds/" + fil) print('HERE:') print(wdir + fil) files = os.listdir(wdir + fil) files.sort() if not edged: # Find borders of noise frame using backgrounds. # First, make sure that the background we're using is for the top chip. i = 0 while f.get_chip_num(wdir + fil + "/" + files[i]) != 1: i += 1 up_left, up_right, up_bottom, up_top = f.detect_edges(wdir + fil + "/" + files[i]) # Ditto for the bottom chip. i = 0 while f.get_chip_num(wdir + fil + "/" + files[i]) != 2: i += 1 dn_left, dn_right, dn_bottom, dn_top = f.detect_edges(wdir + fil + "/" + files[i]) up_left = up_left + 5 up_right = up_right - 5 up_top = up_top - 5 dn_left = dn_left + 5 dn_right = dn_right - 5 dn_bottom = dn_bottom + 5 print('Upper chip:') print(up_left, up_right, up_top, up_bottom) print('Lower:') print(dn_left, dn_right, dn_top, dn_bottom) edged = True for i, file in enumerate(files): print(f'{i} {file}') for i, file in enumerate(files): new_path = output_dir + "backgrounds/" + fil + "/" + file.replace( ".fits", "_trim.fits") # Add GAIN and SATURATE keywords to headers. path = wdir + fil + "/" + file print(f'{i} {file}') # Split the files into upper CCD and lower CCD if f.get_chip_num(path) == 1: print('Upper Chip:') f.trim_file(path, left=up_left, right=up_right, top=up_top, bottom=up_bottom, new_path=new_path) elif f.get_chip_num(path) == 2: print('Lower Chip:') f.trim_file(path, left=dn_left, right=dn_right, top=dn_top, bottom=dn_bottom, new_path=new_path) else: raise ValueError( 'Invalid chip ID; could not trim based on upper or lower chip.' ) # Repeat for science images wdir = origin_dir + "science/" fils = os.listdir(wdir) for fil in fils: print(output_dir + "science/" + fil) if do_sextractor: if not os.path.isdir(sextractor_path + fil): os.mkdir(sextractor_path + fil) if not os.path.isdir(output_dir + "science/" + fil): os.mkdir(output_dir + "science/" + fil) files = os.listdir(wdir + fil) files.sort() for i, file in enumerate(files): print(f'{i} {file}') for i, file in enumerate(files): # Split the files into upper CCD and lower CCD, with even-numbered being upper and odd-numbered being lower new_file = file.replace(".fits", "_trim.fits") new_path = output_dir + "science/" + fil + "/" + new_file path = wdir + fil + "/" + file f.change_header(file=path, key='GAIN', value=0.8) f.change_header(file=path, key='SATURATE', value=65535.) if f.get_chip_num(path) == 1: print('Upper Chip:') f.trim_file(path, left=up_left, right=up_right, top=up_top, bottom=up_bottom, new_path=new_path) if do_sextractor: copyfile(new_path, sextractor_path + fil + "/" + new_file) elif f.get_chip_num(path) == 2: print('Lower Chip:') f.trim_file(path, left=dn_left, right=dn_right, top=dn_top, bottom=dn_bottom, new_path=new_path) try: copyfile(origin_dir + data_title + ".log", output_dir + data_title + ".log") except FileNotFoundError: print("Previous log not found.") u.write_log(path=output_dir + data_title + ".log", action='Edges trimmed using 3-trim.py\n')
def main(field, destination, epoch, instrument, template_instrument, comparison_type): if destination[-1] != '/': destination = destination + '/' # p.refresh_params_frbs() # types = ['multi_frb_range', 'multi_sn_models', 'multi_sn_random', 'multi_sn_random_ia', 'multi_sn_random_ib'] # if comparison_type not in types: # raise ValueError(comparison_type + ' is not a valid multi-synthetic argument; choose from ' + str(types)) comparison_type = '_' + comparison_type type_suffix = comparison_type[7:] params = p.object_params_frb(field) u.mkdir_check(f'{params["data_dir"]}subtraction/') destination_path = f'{params["data_dir"]}subtraction/{destination}/' u.mkdir_check(destination_path) comparison_title = f'{field}_{epoch}' comparison_paths = p.object_output_paths(obj=comparison_title, instrument=instrument) comparison_origin_top = comparison_paths['subtraction_image_synth_' + type_suffix] comparison_tests = os.listdir(comparison_origin_top) comparison_params = p.object_params_instrument(obj=comparison_title, instrument=instrument) params = p.object_params_frb(field) template_epoch = params['template_epoch_' + template_instrument.lower()] template_title = f'{field}_{template_epoch}' template_paths = p.object_output_paths(obj=template_title, instrument=template_instrument) template_outputs = p.object_output_params(obj=template_title, instrument=template_instrument) template_params = p.object_params_instrument( obj=template_title, instrument=template_instrument) filters = params['filters'] for test in comparison_tests: destination_test = destination_path + test + '/' origin_test = comparison_origin_top + test + '/' u.mkdir_check(destination_test) test_files = os.listdir(origin_test) for f in filters: for file in filter(lambda fil: fil[:2] != f + '_', os.listdir(origin_test)): shutil.copyfile(origin_test + file, destination_test + file) values = {} f_0 = f[0] destination_path_filter = f'{destination_test}{f}/' u.mkdir_check(destination_path_filter) # COMPARISON IMAGE: # Get path to comparison image from parameter .yaml file comparison_origin = origin_test + filter( lambda file: file[0] == f_0 and file[-5:] == '.fits', test_files).__next__() comparison_destination = f'{comparison_title}_comparison.fits' shutil.copyfile(comparison_origin, destination_path_filter + comparison_destination) shutil.copyfile( comparison_origin.replace('.fits', '.csv'), destination_path_filter + comparison_destination.replace('.fits', '.csv')) print('Copying comparison image from:') print('\t', comparison_origin) print('To:') print(f'\t {destination_test}{f}/{comparison_destination}') shutil.copy( comparison_params['data_dir'] + 'output_values.yaml', f'{destination_test}{f}/{comparison_title}_comparison_output_values.yaml' ) shutil.copy( comparison_params['data_dir'] + 'output_values.json', f'{destination_test}{f}/{comparison_title}_comparison_output_values.json' ) shutil.copy(comparison_origin, f'{destination_test}{f}/{comparison_destination}') values['comparison_file'] = comparison_origin if template_instrument != 'FORS2': # TEMPLATE IMAGE if template_instrument != 'XSHOOTER': f_0 = f_0.lower() template_image_name = template_params['subtraction_image'] if f'{f_0}_{template_image_name}' in template_paths: template_origin = template_paths[ f'{f_0}_{template_image_name}'] elif f'{f_0.lower()}_{template_image_name}' in template_paths: template_origin = template_paths[ f'{f_0.lower()}_{template_image_name}'] else: raise ValueError( f'{f_0.lower()}_{template_image_name} not found in {template_title} paths' ) fwhm_template = template_outputs[f_0 + '_fwhm_pix'] template_destination = f'{template_title}_template.fits' print('Copying template from:') print('\t', template_origin) print('To:') print(f'\t {destination_test}{f}/{template_destination}') shutil.copy( template_params['data_dir'] + 'output_values.yaml', f'{destination_test}{f}/{template_title}_template_output_values.yaml' ) shutil.copy( template_params['data_dir'] + 'output_values.json', f'{destination_test}{f}/{template_title}_template_output_values.json' ) shutil.copy(template_origin, f'{destination_test}{f}/{template_destination}') values['template_file'] = comparison_origin p.add_params(f'{destination_test}{f}/output_values.yaml', values)
def tweak_final(sextractor_path: str, destination: str, epoch: str, instrument: str, show: bool, tolerance: float = 10., output_suffix: str = 'astrometry', input_suffix='coadded', stars_only: bool = False, path_add: str = 'subtraction_image', manual: bool = False, specific_star: bool = False): """ A wrapper for tweak, to interface with the .yaml param files and provide offsets to all filters used in an observation. :param sextractor_path: Path to SExtractor-generated catalogue. :param destination: Directory to write tweaked image to. :param epoch: The epoch number of the observation to be tweaked. :param instrument: The instrument on which the observation was taken. :param show: Plot matches onscreen? :param tolerance: Tolerance, in pixels, within which matches will be accepted. :param output_suffix: String to append to filename of output file. :param input_suffix: Suffix appended to filenmae of input file. :param stars_only: Only match using stars, determined using SExtractor's 'class_star' output. :param path_add: Key under which to add the output path in the 'output_paths.yaml' file. :param manual: Use manual offset? :param specific_star: Use a specific star to tweak? This means, instead of finding the closest matches for many stars, alignment is attempted with a single star nearest the given position. :return: None """ u.mkdir_check(destination) properties = p.object_params_instrument(obj=epoch, instrument=instrument) frb_properties = p.object_params_frb(obj=epoch[:-2]) cat_name = properties['cat_field_name'] manual = properties['manual_astrometry'] and manual cat_path = frb_properties['data_dir'] + cat_name.upper( ) + "/" + cat_name.upper() + ".csv" if cat_path is not None: outputs = p.object_output_params(obj=epoch, instrument=instrument) filters = outputs['filters'] param_dict = {} for filt in filters: print(filt) f = filt[0] if manual: offset_x = properties[f + '_offset_x'] offset_y = properties[f + '_offset_y'] else: offset_x = None offset_y = None if specific_star: burst_properties = p.object_params_frb(epoch[:-2]) star_ra = burst_properties['alignment_ra'] star_dec = burst_properties['alignment_dec'] else: star_ra = star_dec = None param = tweak( sextractor_path=sextractor_path + f + '_psf-fit.cat', destination=destination + f + '_' + output_suffix + '.fits', image_path=destination + f + '_' + input_suffix + '.fits', cat_path=cat_path, cat_name=cat_name, tolerance=tolerance, show=show, stars_only=stars_only, manual=manual, offset_x=offset_x, offset_y=offset_y, specific_star=specific_star, star_ra=star_ra, star_dec=star_dec) for par in param: param_dict[f + '_' + par] = param[par] p.add_output_path(obj=epoch, key=f + '_' + path_add, path=destination + f + '_' + output_suffix + '.fits', instrument=instrument) p.add_params(properties['data_dir'] + 'output_values.yaml', params=param_dict) else: print('No catalogue found for this alignment.')
def main(obj, test_name, sex_x_col, sex_y_col, sex_ra_col, sex_dec_col, sex_flux_col, get_sextractor_names, mag_tol, stars_only, star_class_col, star_class_tol, show_plots, mag_range_sex_lower, mag_range_sex_upper, pix_tol, separate_chips, write): print(separate_chips) sextractor_names = p.sextractor_names_psf() # None to auto-detect properties = p.object_params_des(obj=obj) outputs = p.object_output_params(obj=obj, instrument='des') paths = p.object_output_paths(obj=obj) cat_name = 'DES' cat_path = properties['data_dir'] + 'des_objects.csv' output = properties['data_dir'] + '/3-zeropoint/' mkdir_check(output) output = output + 'field/' mkdir_check(output) for fil in properties['filters']: f_0 = fil[0] f_up = f_0.upper() output_path = output + f_0 cat_zeropoint = 0. cat_zeropoint_err = 0. now = time.Time.now() now.format = 'isot' test_name = str(now) + '_' + test_name mkdir_check(output_path) output_path = output_path + '/' + f_0 + '/' mkdir_check(output_path) output_path = output_path + '/' + test_name + '/' mkdir_check(output_path) if not os.path.isdir(output_path): os.mkdir(output_path) output_path = output_path + test_name + '/' if not os.path.isdir(output_path): os.mkdir(output_path) image_path = properties[ 'data_dir'] + '2-sextractor/' + f_0 + '_cutout.fits' sextractor_path = properties[ 'data_dir'] + '2-sextractor/' + f_0 + '_psf-fit.cat' # We override here because there's no need to separate the chips if we're using the DES image. separate_chips = False # pix_tol = 1. mag_range_lower = 16 mag_range_upper = 25 exp_time = 1. print('SExtractor catalogue path:', sextractor_path) print('Image path:', image_path) print('Catalogue path:', cat_path) print('Output:', output_path + test_name) print() print(cat_zeropoint) up = photometry.determine_zeropoint_sextractor( sextractor_cat=sextractor_path, image=image_path, image_name='DES', cat_path=cat_path, cat_name=cat_name, output_path=output_path + "/chip_1/", show=show_plots, cat_ra_col='RA', cat_dec_col='DEC', cat_mag_col='WAVG_MAG_PSF_' + f_up, sex_ra_col=sex_ra_col, sex_dec_col=sex_dec_col, sex_x_col=sex_x_col, sex_y_col=sex_y_col, dist_tol=pix_tol, mag_tol=mag_tol, flux_column=sex_flux_col, mag_range_cat_upper=mag_range_upper, mag_range_cat_lower=mag_range_lower, mag_range_sex_upper=mag_range_sex_upper, mag_range_sex_lower=mag_range_sex_lower, stars_only=stars_only, star_class_tol=star_class_tol, star_class_col=star_class_col, exp_time=exp_time, get_sextractor_names=get_sextractor_names, sextractor_names=sextractor_names, cat_type='csv', cat_zeropoint=cat_zeropoint, cat_zeropoint_err=cat_zeropoint_err, ) if write: update_dict = { f_0 + '_zeropoint_derived': float(up['zeropoint_sub_outliers']), f_0 + '_zeropoint_derived_err': float(up['zeropoint_err']) } p.add_output_values(obj=obj, params=update_dict, instrument='des')
def main(epoch, test_name, sex_x_col, sex_y_col, sex_ra_col, sex_dec_col, sex_flux_col, stars_only, show_plots, mag_range_sex_lower, mag_range_sex_upper, pix_tol, instrument): print("\nExecuting Python script pipeline_fors2/9-zeropoint.py, with:") print(f"\tepoch {epoch}") print() properties = p.object_params_fors2(epoch) outputs = p.object_output_params(obj=epoch, instrument='fors2') proj_paths = p.config output = properties['data_dir'] + '9-zeropoint/' mkdir_check(output) output_std = output + 'std/' mkdir_check(output_std) mkdir_check(output_std) std_path = properties['data_dir'] + 'calibration/std_star/' filters = outputs['filters'] std_field_path = proj_paths['top_data_dir'] + "std_fields/" std_fields = list( filter(lambda path: os.path.isdir(path), os.listdir(std_field_path))) print('Standard fields with available catalogues:') print(std_fields) print() cat_names = photometry_catalogues for fil in filters: print('Doing filter', fil) f = fil[0] # Obtain zeropoints from FRB field, if data is available. print( f"\nDetermining science-field zeropoints for {epoch}, filter {fil}:\n" ) if f"{f}_zeropoints" in outputs: zeropoints = outputs[f"{f}_zeropoints"] else: zeropoints = {} zeropoints["science_field"] = {} for cat_name in cat_names: zeropoints["science_field"][ cat_name], _ = photometry.zeropoint_science_field( epoch=epoch, instrument=instrument, test_name=None, sex_x_col='XPSF_IMAGE', sex_y_col='YPSF_IMAGE', sex_ra_col='ALPHAPSF_SKY', sex_dec_col='DELTAPSF_SKY', sex_flux_col='FLUX_PSF', stars_only=True, star_class_col='CLASS_STAR', star_class_tol=0.95, show_plots=False, mag_range_sex_lower=-100. * units.mag, mag_range_sex_upper=100. * units.mag, pix_tol=5. * units.pixel, separate_chips=True, cat_name=cat_name) # Obtain zeropoints from available standard fields and available data. print( f"\nDetermining standard-field zeropoints for {epoch}, filter {fil}\n" ) zeropoints["standard_field"] = {} fil_path = std_path + fil + '/' if os.path.isdir(fil_path): fields = filter(lambda d: os.path.isdir(fil_path + d), os.listdir(fil_path)) output_path_fil_std = output_std + '/' + fil + '/' mkdir_check(output_path_fil_std) for field in fields: zeropoints["standard_field"][field] = {} ra = float(field[field.find("RA") + 2:field.find("_")]) dec = float(field[field.find("DEC") + 3:]) print("Looking for photometry data in field " + field + ":") mkdir_check(std_field_path + field) field_path = fil_path + field + '/' output_path = output_path_fil_std + field + '/' std_cat_path = std_field_path + field + '/' std_properties = p.load_params(field_path + 'params.yaml') use_sex_star_class = std_properties['use_sex_star_class'] # Cycle through the three catalogues used to determine zeropoint, in order of preference. for cat_name in cat_names: # Check for photometry on-disk in the relevant catalogue; if none present, attempt to retrieve from # online archive. print(f"In {cat_name}:") output_path_cat = output_path + cat_name cat_path = f"{std_cat_path}{cat_name}/{cat_name}.csv" if not (os.path.isdir(std_cat_path + cat_name) ) or os.path.isfile(cat_path): print( "None found on disk. Attempting retrieval from archive..." ) if update_std_photometry(ra=ra, dec=dec, cat=cat_name) is None: print("\t\tNo data found in archive.") zeropoints["standard_field"][field][ cat_name] = None continue column_names = cat_columns(cat=cat_name, f=f) cat_ra_col = column_names['ra'] cat_dec_col = column_names['dec'] cat_mag_col = column_names['mag_psf'] if not use_sex_star_class: star_class_col = column_names['class_star'] else: star_class_col = 'CLASS_STAR' cat_type = 'csv' sextractor_path = field_path + 'sextractor/_psf-fit.cat' image_path = field_path + '3-trimmed/standard_trimmed_img_up.fits' star_class_tol = std_properties['star_class_tol'] now = time.Time.now() now.format = 'isot' test_name = str(now) + '_' + test_name mkdir_check(properties['data_dir'] + '/analysis/zeropoint/') mkdir_check(output_path) exp_time = ff.get_exp_time(image_path) print('SExtractor catalogue path:', sextractor_path) print('Image path:', image_path) print('Catalogue name:', cat_name) print('Catalogue path:', cat_path) print('Class star column:', star_class_col) print('Output:', output_path_cat) print('Exposure time:', exp_time) print("Use sextractor class star:", use_sex_star_class) zeropoints["standard_field"][field][ cat_name] = photometry.determine_zeropoint_sextractor( sextractor_cat=sextractor_path, image=image_path, cat_path=cat_path, cat_name=cat_name, output_path=output_path_cat, show=show_plots, cat_ra_col=cat_ra_col, cat_dec_col=cat_dec_col, cat_mag_col=cat_mag_col, sex_ra_col=sex_ra_col, sex_dec_col=sex_dec_col, sex_x_col=sex_x_col, sex_y_col=sex_y_col, dist_tol=pix_tol, flux_column=sex_flux_col, mag_range_sex_upper=mag_range_sex_upper, mag_range_sex_lower=mag_range_sex_lower, stars_only=stars_only, star_class_tol=star_class_tol, star_class_col=star_class_col, exp_time=exp_time, y_lower=0, cat_type=cat_type, ) output_dict = {f + '_zeropoints': zeropoints} p.add_output_values(obj=epoch, instrument='FORS2', params=output_dict) outputs = p.object_output_params(obj=epoch, instrument='fors2') output_path_final = output + "collated/" mkdir_check(output_path_final) print("Collating zeropoints...") for fil in filters: print(f"For {fil}:") f = fil[0] output_path_final_f = output_path_final + fil + "/" mkdir_check(output_path_final_f) zeropoints = outputs[f"{f}_zeropoints"] airmass_sci = outputs[f"{f}_airmass_mean"] airmass_sci_err = outputs[f"{f}_airmass_err"] extinction = outputs[f"{f}_extinction"] extinction_err = outputs[f"{f}_extinction_err"] zeropoint_tbl = table.Table( dtype=[("type", 'S15'), ("field", 'S25'), ( "cat", 'S10'), ("zeropoint", float), ("zeropoint_err", float), ("airmass", float), ("airmass_err", float), ("extinction", float), ("extinction_err", float), ("zeropoint_ext_corr", float), ("zeropoint_ext_corr_err", float), ("n_matches", float)]) if f"provided" in zeropoints: print(f"\tProvided:") zeropoint_prov = zeropoints["provided"] zeropoint_tbl.add_row([ "provided", "N/A", "N/A", zeropoint_prov["zeropoint"], zeropoint_prov["zeropoint_err"], 0.0, 0.0, extinction, extinction_err, zeropoint_prov["zeropoint"], zeropoint_prov["zeropoint_err"], 0 ]) print(f"\tScience field:") for cat_name in zeropoints["science_field"]: print(f"\t\t{cat_name}") zeropoint_sci = zeropoints["science_field"][cat_name] if zeropoint_sci is not None: zeropoint_corrected = zeropoint_sci[ "zeropoint"] + extinction * airmass_sci print("\t\t\t", zeropoint_sci["zeropoint"], extinction, airmass_sci) zeropoint_corrected_err = zeropoint_sci[ "zeropoint_err"] + error_product( value=extinction * airmass_sci, measurements=[extinction, airmass_sci], errors=[extinction_err, airmass_sci_err]) zp_cat = table.Table.read(zeropoint_sci["matches_cat_path"]) zeropoint_tbl.add_row([ "science_field", epoch[:-2], cat_name, zeropoint_sci["zeropoint"], zeropoint_sci["zeropoint_err"], airmass_sci, airmass_sci_err, extinction, extinction_err, zeropoint_corrected, zeropoint_corrected_err, len(zp_cat) ]) plt.scatter(zp_cat["mag_cat"], zp_cat["mag"], c='green') plt.errorbar(zp_cat["mag_cat"], zp_cat["mag"], yerr=zp_cat["mag_err"], linestyle="None", c='black') plt.plot(zp_cat["mag_cat"], zp_cat["mag_cat"] - zeropoint_sci["zeropoint"], c="violet") plt.title(f"{fil}: science-field, {cat_name}") plt.figtext( 0, 0.01, f'zeropoint - kX: {zeropoint_sci["zeropoint"]} ± {zeropoint_sci["zeropoint_err"]}\n' f'zeropoint: {zeropoint_corrected} ± {zeropoint_corrected_err}\n' f'num stars: {len(zp_cat)}') plt.gca().set_aspect('equal', adjustable='box') plt.savefig(output_path_final_f + f"zeropoint_science_{cat_name}.pdf") plt.show() plt.close() print("\tStandard fields:") for field in zeropoints["standard_field"]: print(f"\t\t{field}") for cat_name in zeropoints["standard_field"][field]: print(f"\t\t\t{cat_name}") zeropoint_std = zeropoints["standard_field"][field][cat_name] if zeropoint_std is not None: zeropoint_corrected = zeropoint_std[ "zeropoint"] + extinction * zeropoint_std["airmass"] print("\t\t\t\t", zeropoint_std["zeropoint"], extinction, zeropoint_std["airmass"]) zeropoint_corrected_err = zeropoint_std[ "zeropoint_err"] + error_product( value=extinction * zeropoint_std["airmass"], measurements=[ extinction, zeropoint_std["airmass"] ], errors=[extinction_err, 0.0]) zp_cat = table.Table.read( zeropoint_std["matches_cat_path"]) zeropoint_tbl.add_row([ "standard_field", field, cat_name, zeropoint_std["zeropoint"], zeropoint_std["zeropoint_err"], zeropoint_std["airmass"], 0.0, extinction, extinction_err, zeropoint_corrected, zeropoint_corrected_err, len(zp_cat) ]) plt.scatter(zp_cat["mag_cat"], zp_cat["mag"], c='green') plt.errorbar(zp_cat["mag_cat"], zp_cat["mag"], yerr=zp_cat["mag_err"], linestyle="None", c='black') plt.plot(zp_cat["mag_cat"], zp_cat["mag_cat"] - zeropoint_std["zeropoint"], c="violet") plt.title(f"{fil}: standard-field {field}, {cat_name}") plt.gca().set_aspect('equal', adjustable='box') plt.figtext( 0, 0.01, f'zeropoint - kX: {zeropoint_std["zeropoint"]} ± {zeropoint_std["zeropoint_err"]}\n' f'zeropoint: {zeropoint_corrected} ± {zeropoint_corrected_err}\n' f'num stars: {len(zp_cat)}') plt.savefig(output_path_final_f + f"zeropoint_standard_{field}_{cat_name}.pdf") plt.show() plt.close() zeropoint_tbl["selection_index"] = zeropoint_tbl[ "n_matches"] / zeropoint_tbl["zeropoint_ext_corr_err"] best_arg = np.argmax(zeropoint_tbl["selection_index"]) print("Best zeropoint:") best_zeropoint = zeropoint_tbl[best_arg] print(best_zeropoint) zeropoints = outputs[f + '_zeropoints'] zeropoints['best'] = { "zeropoint": float(best_zeropoint['zeropoint']), "zeropoint_err": float(best_zeropoint["zeropoint_err"]), "airmass": float(best_zeropoint["airmass"]), "airmass_err": float(best_zeropoint["airmass_err"]), "type": str(best_zeropoint["type"]), "catalogue": str(best_zeropoint["cat"]) } output_dict = {f + '_zeropoints': zeropoints} p.add_output_values(obj=epoch, instrument='FORS2', params=output_dict) zeropoint_tbl.write(output_path_final_f + "zeropoints.csv", format="ascii.csv", overwrite=True)
def main(data_title, sextractor_path, origin, destination): print( "\nExecuting Python script pipeline_fors2/4-divide_by_exp_time.py, with:" ) print(f"\tepoch {data_title}") print(f"\tsextractor path {sextractor_path}") print(f"\torigin directory {origin}") print(f"\tdestination directory {destination}") print() properties = p.object_params_fors2(data_title) outputs = p.object_output_params(obj=data_title, instrument='FORS2') data_dir = properties['data_dir'] if sextractor_path is not None: if not os.path.isdir(sextractor_path): os.mkdir(sextractor_path) do_sextractor = True else: do_sextractor = False origin_path = data_dir + origin destination_path = data_dir + destination u.mkdir_check(destination_path) u.mkdir_check(destination_path + "science/") u.mkdir_check(destination_path + "backgrounds/") filters = outputs['filters'] for fil in filters: u.mkdir_check(destination_path + "science/" + fil) u.mkdir_check(destination_path + "backgrounds/" + fil) if do_sextractor: if not os.path.isdir(sextractor_path + fil): os.mkdir(sextractor_path + fil) files = os.listdir(origin_path + "science/" + fil + "/") for file_name in files: if file_name[-5:] == '.fits': science_origin = origin_path + "science/" + fil + "/" + file_name science_destination = destination_path + "science/" + fil + "/" + file_name.replace( "trim", "norm") background_origin = origin_path + "backgrounds/" + fil + "/" + file_name.replace( "SCIENCE_REDUCED", "PHOT_BACKGROUND_SCI") background_destination = destination_path + "backgrounds/" + fil + "/" + \ file_name.replace("SCIENCE_REDUCED", "PHOT_BACKGROUND_SCI").replace("trim", "norm") print(science_origin) # Divide by exposure time to get an image in counts/second. ff.divide_by_exp_time(file=science_origin, output=science_destination) ff.divide_by_exp_time(file=background_origin, output=background_destination) if do_sextractor: copyfile( science_destination, sextractor_path + fil + "/" + file_name.replace("trim", "norm")) if os.path.isfile(origin_path + data_title + '.log'): copyfile(origin_path + data_title + '.log', destination_path + data_title + ".log") u.write_log(path=destination_path + data_title + ".log", action=f'Divided by exposure time.')
def main(data_title: 'str', delete_output: bool = True): print( "\nExecuting Python script pipeline_fors2/2-sort_after_esoreflex.py, with:" ) print(f"\tepoch {data_title}") print() eso_dir = p.config['esoreflex_output_dir'] if os.path.isdir(eso_dir): obj_params = p.object_params_fors2(data_title) data_dir = obj_params['data_dir'] destination = data_dir + "2-sorted/" date = None output_values = p.object_output_params(obj=data_title, instrument='FORS2') mjd = int(output_values['mjd_obs']) obj = output_values['object'] print( f"Looking for data with object '{obj}' and MJD of observation {mjd} inside {eso_dir}" ) # Look for files with the appropriate object and MJD, as recorded in output_values # List directories in eso_output_dir; these are dates on which data was reduced using ESOReflex. eso_dirs = filter(lambda d: os.path.isdir(eso_dir + "/" + d), os.listdir(eso_dir)) for directory in eso_dirs: directory += "/" # List directories within 'reduction date' directories. # These should represent individual images reduced. print(f"Searching {eso_dir + directory}") eso_subdirs = filter( lambda d: os.path.isdir(eso_dir + directory + d) and "FORS2" in d, os.listdir(eso_dir + directory)) for subdirectory in eso_subdirs: subdirectory += "/" subpath = eso_dir + "/" + directory + subdirectory print(f"\tSearching {subpath}") # Get the files within the image directory. files = filter(lambda d: os.path.isfile(subpath + d), os.listdir(subpath)) for file_name in files: # Retrieve the target object name from the fits file. file_path = subpath + file_name file = fits.open(file_path) file_obj = ff.get_object(file) file_mjd = int(ff.get_header_attribute(file, 'MJD-OBS')) file_filter = ff.get_filter(file) # Check the object name and observation date against those of the epoch we're concerned with. if file_obj == obj and file_mjd == mjd: # Check which type of file we have. date = directory[:-1] if file_name[-28:] == "PHOT_BACKGROUND_SCI_IMG.fits": file_destination = f"{destination}/backgrounds/" suffix = "PHOT_BACKGROUND_SCI_IMG.fits" elif file_name[-25:] == "OBJECT_TABLE_SCI_IMG.fits": file_destination = f"{destination}/obj_tbls/" suffix = "OBJECT_TABLE_SCI_IMG.fits" elif file_name[-24:] == "SCIENCE_REDUCED_IMG.fits": file_destination = f"{destination}/science/" suffix = "SCIENCE_REDUCED_IMG.fits" else: file_destination = f"{destination}/sources/" suffix = "SOURCES_SCI_IMG.fits" # Make this directory, if it doesn't already exist. u.mkdir_check(file_destination) # Make a subdirectory by filter. file_destination += file_filter + "/" u.mkdir_check(file_destination) # Title new file. file_destination += f"{data_title}_{subdirectory[:-1]}_{suffix}" # Copy file to new location. print( f"Copying: {file_path} to \n\t {file_destination}") file.writeto(file_destination, overwrite=True) if delete_output and os.path.isfile(file_destination): os.remove(file_path) if not os.path.isfile(f"{destination}/{data_title}.log"): sh.copy(f"{data_dir}0-data_with_raw_calibs/{data_title}.log", f"{destination}/{data_title}.log") u.write_log(path=destination + data_title + ".log", action=f'Data reduced with ESOReflex.', date=date) u.write_log(path=destination + data_title + ".log", action='Files sorted using 2-sort_after_esoreflex.sh') else: print(f"ESO output directory '{eso_dir}' not found.") exit(1)