def main(data_dir, data_title, destination, fil, object): table = (destination + '/' + fil, science_only=False) fil = fil.replace('/', '') header_file = destination + '/' + fil + '_template.hdr' header_stream = open(header_file, 'r') header = header_stream.readlines() header_stream.close() # Transfer and/or modify header information params = p.object_output_params(data_title, instrument='IMACS') airmass = table['airmass'].mean() saturate = table['saturate'].mean() old_gain = table['gain'].mean() n_frames = params[fil + '_n_exposures'] gain = gain_median_combine(old_gain=old_gain, n_frames=n_frames) header.insert(-1, f'AIRMASS = {airmass}\n') header.insert(-1, f'FILTER = {fil}\n') header.insert(-1, f'OBJECT = {object}\n') header.insert(-1, f'EXPTIME = 1.\n') header.insert(-1, f'GAIN = {gain}\n') header.insert(-1, f'SATURATE= {saturate}\n') header.insert(-1, f'MJD-OBS = {float(np.nanmean(table["mjd_obs"]))}\n') os.remove(header_file) with open(header_file, 'w') as file: file.writelines(header) p.add_output_path(obj=data_title, key=fil + '_subtraction_image', path=destination + '/' + fil + '_coadded.fits', instrument='IMACS')
def main(obj, input_path, output, path, prefix): print( "\nExecuting Python script pipeline_fors2/9-esorex_zeropoint_trim.py, with:" ) print(f"\tepoch {obj}") print(f"\tinput path {input_path}") print(f"\toutput path {output}") print(f"\tstandard star data path {path}") print(f"\tfilter prefix {prefix}") print() ff.trim_file(path=input_path, bottom=0, top=950, left=195, right=1825, new_path=output) airmass = ff.get_airmass(input_path) exptime = ff.get_exp_time(input_path) p.add_params( path + '/output_values', { prefix + '_airmass': airmass, prefix + '_airmass_err': 0, prefix + '_exptime': exptime }) p.add_output_path(obj=obj, instrument='FORS2', key=prefix + '_std_image', path=output)
def main(comb_path, output_dir, obj, sextractor_path, path_suffix): print("\nExecuting Python script pipeline_fors2/7-trim_combined.py, with:") print(f"\tepoch {obj}") print(f"\toutput directory {output_dir}") print(f"\tsextractor directory {sextractor_path}") print() if sextractor_path is not None: u.mkdir_check_nested(sextractor_path) do_sextractor = True else: do_sextractor = False # Build a list of the filter prefixes used. fils = [] files = list(filter(lambda x: x[-4:] == '.tbl', os.listdir(comb_path))) for file in files: if file[0] not in fils: fils.append(str(file[0])) for fil in fils: if do_sextractor: u.mkdir_check(sextractor_path) area_file = fil + "_coadded_area.fits" comb_file = fil + "_coadded.fits" left, right, bottom, top = f.detect_edges_area(comb_path + area_file) # Trim a little extra to be safe. left = left + 5 right = right - 5 top = top - 5 bottom = bottom + 5 f.trim_file(comb_path + comb_file, left=left, right=right, top=top, bottom=bottom, new_path=output_dir + "/" + comb_file) # Keep a trimmed version of the area file, it comes in handy later. f.trim_file(comb_path + area_file, left=left, right=right, top=top, bottom=bottom, new_path=output_dir + "/" + area_file) if do_sextractor: copyfile(output_dir + "/" + comb_file, sextractor_path + comb_file) if path_suffix is None: path_suffix = output_dir.split("/")[-2] path_suffix = u.remove_trailing_slash(path_suffix) p.add_output_path(obj=obj, instrument='fors2', key=fil[0] + '_trimmed_image' + path_suffix, path=output_dir + "/" + comb_file)
def main(data_title: str): properties = p.object_params_des(data_title) path = properties['data_dir'] data_path = path + '/0-data/' u.mkdir_check(data_path) os.listdir(path) print(path) for file in filter(lambda fil: fil[-5:] == '.fits', os.listdir(path)): f = file[-6] shutil.copy(path + file, data_path + str(f) + '_cutout.fits') p.add_output_path(obj=data_title, instrument='DES', key=f + '_subtraction_image', path=data_path + str(f) + '_cutout.fits')
def main(obj, test, curves_path): properties = p.object_params_fors2(obj) if curves_path[-1] != '/': curves_path += '/' mag_table_file = filter(lambda file: file[-4:] == '.csv', os.listdir(curves_path)).__next__() mag_table = table.Table.read(curves_path + mag_table_file) filters = mag_table.colnames.copy() filters.remove('model') synth_path = properties['data_dir'] + 'synthetic/' u.mkdir_check(synth_path) synth_path = synth_path + 'frb_position/' u.mkdir_check(synth_path) now = time.Time.now() now.format = 'isot' synth_path += f'sn_models_{now}/' u.mkdir_check(synth_path) for row in mag_table: model = row['model'] magnitudes = [] for f in filters: magnitudes.append(row[f]) test_spec = test + '_' + model test_path = synth_path + test_spec + '/' ph.insert_synthetic_at_frb(obj=obj, test_path=test_path, filters=filters, magnitudes=magnitudes, add_path=False) for f in filters: p.add_output_path(obj=obj, key=f[0] + '_subtraction_image_synth_frb_sn_models', path=synth_path)
def main(obj, test, mag_min, mag_max, increment, instrument): properties = p.object_params_instrument(obj, instrument=instrument) output = p.object_output_params(obj=obj, instrument=instrument) paths = p.object_output_paths(obj=obj, instrument=instrument) synth_path = properties['data_dir'] + 'synthetic/' u.mkdir_check(synth_path) synth_path = synth_path + 'frb_position/' u.mkdir_check(synth_path) now = time.Time.now() now.format = 'isot' synth_path += f'range_{now}/' u.mkdir_check(synth_path) filters = output['filters'] for magnitude in np.arange(mag_min, mag_max, increment): magnitudes = [] for i in range(len(filters)): magnitudes.append(magnitude) test_spec = test + '_' + str(u.round_to_sig_fig(magnitude, 4)) test_path = synth_path + test_spec + '/' ph.insert_synthetic_at_frb(obj=properties, test_path=test_path, filters=filters, magnitudes=magnitudes, add_path=False, psf=True, output_properties=output, instrument=instrument, paths=paths) p.add_output_path(obj=obj, key='subtraction_image_synth_frb_range', path=synth_path, instrument=instrument)
def main(obj, test, n, filter_dist, instrument, limit): properties = p.object_params_instrument(obj, instrument=instrument) burst_properties = p.object_params_frb(obj=obj[:-2]) output = p.object_output_params(obj=obj, instrument=instrument) paths = p.object_output_paths(obj=obj, instrument=instrument) z = burst_properties['z'] mjd_burst = burst_properties['mjd_burst'] ebv_mw = burst_properties['dust_ebv'] mjd_obs = properties['mjd'] synth_path = properties['data_dir'] + 'synthetic/' u.mkdir_check(synth_path) synth_path = synth_path + 'sn_random/' u.mkdir_check(synth_path) epoch = mjd_obs - mjd_burst f_0 = filter_dist[0] hg_ra = burst_properties['hg_ra'] hg_dec = burst_properties['hg_dec'] burst_ra = burst_properties['burst_ra'] burst_dec = burst_properties['burst_dec'] image_path = paths[f_0 + '_' + properties['subtraction_image']] image = fits.open(image_path) wcs_info = wcs.WCS(image[0].header) burst_x, burst_y = wcs_info.all_world2pix(burst_ra, burst_dec, 0) now = time.Time.now() now.format = 'isot' synth_path += test + '_' + str(now) + '/' u.mkdir_check(synth_path) filters = output['filters'] psf_models = [] for i, f in enumerate(filters): sn.register_filter(f=f, instrument=instrument) psf_model = fits.open(paths[f[0] + '_psf_model']) psf_models.append(psf_model) for i in range(n): test_spec = test + '_' + str(i) test_path = synth_path + test_spec + '/' magnitudes = [] mags_filters, model, x, y, tbl = sn.random_light_curves_type_ia( filters=filters, image=image, hg_ra=hg_ra, hg_dec=hg_dec, z=z, ebv_mw=ebv_mw, output_path=test_path, output_title=test, limit=limit, x=burst_x, y=burst_y, ra=burst_ra, dec=burst_dec) days = mags_filters['days'] for f in filters: magnitude = sn.magnitude_at_epoch(epoch=epoch, days=days, mags=mags_filters[f]) print(f, 'mag:', magnitude) magnitudes.append(magnitude) ph.insert_synthetic(x=float(x), y=float(y), obj=properties, test_path=test_path, filters=filters, magnitudes=magnitudes, suffix='sn_random_random_ia', extra_values=tbl, paths=paths, output_properties=output, psf_models=psf_models, instrument=instrument) p.add_output_path(obj=obj, key='subtraction_image_synth_sn_random_ia', path=synth_path, instrument=instrument)
def main(obj, test, n, mag_lower, mag_upper, colour_upper, colour_lower): properties = p.object_params_fors2(obj) output = p.object_output_params(obj=obj, instrument='FORS2') paths = p.object_output_paths(obj) burst_properties = p.object_params_frb(obj[:-2]) synth_path = properties['data_dir'] + 'synthetic/' u.mkdir_check(synth_path) synth_path = synth_path + 'random/' u.mkdir_check(synth_path) now = time.Time.now() now.format = 'isot' test = str(now) + '_' + test test_path = synth_path + test + '/' u.mkdir_check(test_path) filters = {} bluest = None bluest_lambda = np.inf for f in output['filters']: filter_properties = p.filter_params(f=f, instrument='FORS2') filters[f] = filter_properties lambda_eff = filter_properties['lambda_eff'] if lambda_eff < bluest_lambda: bluest_lambda = lambda_eff bluest = f # Insert random sources in the bluest filter. f_0 = bluest[0] output_properties = p.object_output_params(obj) fwhm = output_properties[f_0 + '_fwhm_pix'] zeropoint, _, airmass, _ = ph.select_zeropoint(obj, bluest, instrument='fors2') base_path = paths[f_0 + '_subtraction_image'] output_path = test_path + f_0 + '_random_sources.fits' _, sources = ph.insert_random_point_sources_to_file(file=base_path, fwhm=fwhm, output=output_path, n=n, airmass=airmass, zeropoint=zeropoint) p.add_output_path(obj=obj, key=f_0 + '_subtraction_image_synth_random', path=output_path) # Now insert sources at the same positions in other filters, but with magnitudes randomised. for f in filters: if f != bluest: f_0 = f[0] output_properties = p.object_output_params(obj) fwhm = output_properties[f_0 + '_fwhm_pix'] zeropoint, _, airmass, _ = ph.select_zeropoint(obj, f, instrument='fors2') base_path = paths[f_0 + '_subtraction_image'] mag = np.random.uniform(mag_lower, mag_upper, size=n) output_path = test_path + f_0 + '_random_sources.fits' ph.insert_point_sources_to_file(file=base_path, fwhm=fwhm, output=output_path, x=sources['x_0'], y=sources['y_0'], mag=mag, airmass=airmass, zeropoint=zeropoint) p.add_output_path(obj=obj, key=f_0 + '_subtraction_image_synth_random', path=output_path)
def main(obj, test, n, filter_dist, sn_type, instrument, limit): properties = p.object_params_instrument(obj, instrument=instrument) burst_properties = p.object_params_frb(obj=obj[:-2]) output = p.object_output_params(obj=obj, instrument=instrument) paths = p.object_output_paths(obj=obj, instrument=instrument) z = burst_properties['z'] mjd_burst = burst_properties['mjd_burst'] ebv_mw = burst_properties['dust_ebv'] mjd_obs = properties['mjd'] synth_path = properties['data_dir'] + 'synthetic/' u.mkdir_check(synth_path) synth_path = synth_path + 'sn_random/' u.mkdir_check(synth_path) epoch = mjd_obs - mjd_burst f_0 = filter_dist[0] hg_ra = burst_properties['hg_ra'] hg_dec = burst_properties['hg_dec'] image_path = paths[f_0 + '_' + properties['subtraction_image']] image = fits.open(image_path) now = time.Time.now() now.format = 'isot' test += sn_type + '_' synth_path += test + str(now) + '/' u.mkdir_check(synth_path) filters = output['filters'] if 'FRB190608' in obj: limit = 45 psf_models = [] for f in filters: sn.register_filter(f=f) psf_model = fits.open(paths[f[0] + '_psf_model']) psf_models.append(psf_model) for i in range(n): test_spec = test + str(i) test_path = synth_path + test_spec + '/' magnitudes = [] mags_filters, model, x, y, tbl = sn.random_light_curves( sn_type=sn_type, filters=filters, image=image, hg_ra=hg_ra, hg_dec=hg_dec, z=z, ebv_mw=ebv_mw, output_path=test_path, output_title=test, limit=limit) days = mags_filters['days'] for f in filters: magnitude = sn.magnitude_at_epoch(epoch=epoch, days=days, mags=mags_filters[f]) print(f, 'mag:', magnitude) magnitudes.append(magnitude) ph.insert_synthetic(x=x, y=y, obj=properties, test_path=test_path, filters=filters, magnitudes=magnitudes, suffix='sn_random_random_' + sn_type, extra_values=tbl, paths=paths, output_properties=output, psf_models=psf_models, instrument=instrument) p.add_output_path(obj=obj, key='subtraction_image_synth_sn_random_' + sn_type, path=synth_path, instrument=instrument)
def tweak_final(sextractor_path: str, destination: str, epoch: str, instrument: str, show: bool, tolerance: float = 10., output_suffix: str = 'astrometry', input_suffix='coadded', stars_only: bool = False, path_add: str = 'subtraction_image', manual: bool = False, specific_star: bool = False): """ A wrapper for tweak, to interface with the .yaml param files and provide offsets to all filters used in an observation. :param sextractor_path: Path to SExtractor-generated catalogue. :param destination: Directory to write tweaked image to. :param epoch: The epoch number of the observation to be tweaked. :param instrument: The instrument on which the observation was taken. :param show: Plot matches onscreen? :param tolerance: Tolerance, in pixels, within which matches will be accepted. :param output_suffix: String to append to filename of output file. :param input_suffix: Suffix appended to filenmae of input file. :param stars_only: Only match using stars, determined using SExtractor's 'class_star' output. :param path_add: Key under which to add the output path in the 'output_paths.yaml' file. :param manual: Use manual offset? :param specific_star: Use a specific star to tweak? This means, instead of finding the closest matches for many stars, alignment is attempted with a single star nearest the given position. :return: None """ u.mkdir_check(destination) properties = p.object_params_instrument(obj=epoch, instrument=instrument) frb_properties = p.object_params_frb(obj=epoch[:-2]) cat_name = properties['cat_field_name'] manual = properties['manual_astrometry'] and manual cat_path = frb_properties['data_dir'] + cat_name.upper( ) + "/" + cat_name.upper() + ".csv" if cat_path is not None: outputs = p.object_output_params(obj=epoch, instrument=instrument) filters = outputs['filters'] param_dict = {} for filt in filters: print(filt) f = filt[0] if manual: offset_x = properties[f + '_offset_x'] offset_y = properties[f + '_offset_y'] else: offset_x = None offset_y = None if specific_star: burst_properties = p.object_params_frb(epoch[:-2]) star_ra = burst_properties['alignment_ra'] star_dec = burst_properties['alignment_dec'] else: star_ra = star_dec = None param = tweak( sextractor_path=sextractor_path + f + '_psf-fit.cat', destination=destination + f + '_' + output_suffix + '.fits', image_path=destination + f + '_' + input_suffix + '.fits', cat_path=cat_path, cat_name=cat_name, tolerance=tolerance, show=show, stars_only=stars_only, manual=manual, offset_x=offset_x, offset_y=offset_y, specific_star=specific_star, star_ra=star_ra, star_dec=star_dec) for par in param: param_dict[f + '_' + par] = param[par] p.add_output_path(obj=epoch, key=f + '_' + path_add, path=destination + f + '_' + output_suffix + '.fits', instrument=instrument) p.add_params(properties['data_dir'] + 'output_values.yaml', params=param_dict) else: print('No catalogue found for this alignment.')
def main(obj, astrometry_path, sextractor_path, template, show): print("\nExecuting Python script pipeline_fors2/8-astrometry.py, with:") print(f"\tepoch {obj}") print(f"\tastrometry path {astrometry_path}") print(f"\tsextractor path {sextractor_path}") print(f"\ttemplate {template}") print(f"\tshow {show}") print() files = os.listdir(astrometry_path) template_file = template[0] + '_astrometry.fits' params = p.object_params_fors2(obj) for file in files: if 'coadded.fits' in file: astrometry_file = file.replace('_coadded', '_astrometry') coadd = fits.open(astrometry_path + file) header = coadd[0].header if os.path.isfile(astrometry_path + astrometry_file): astrometry = fits.open(astrometry_path + astrometry_file, mode='update') astrometry[0].header['AIRMASS'] = header['AIRMASS'] astrometry[0].header['FILTER'] = header['FILTER'] astrometry[0].header['OBJECT'] = header['OBJECT'] gain = header['GAIN'] astrometry[0].header['GAIN'] = gain astrometry[0].header['EXPTIME'] = header['EXPTIME'] astrometry[0].header['RDNOISE'] = 0 astrometry[0].header['MJD-OBS'] = header['MJD-OBS'] ff.add_log(astrometry, 'Refined astrometric solution using Astrometry.net') coadd.close() astrometry.close() print() p.add_output_path(obj=obj, key=file[0] + '_astrometry_image', path=astrometry_path + astrometry_file) if file != template_file: shutil.copy(astrometry_path + astrometry_file, astrometry_path + astrometry_file.replace('.fits', '_orig.fits')) shutil.copy(astrometry_path + template_file, astrometry_path + template_file.replace('.fits', '_orig.fits')) ff.align(comparison=astrometry_path + astrometry_file, template=astrometry_path + template_file, comparison_output=astrometry_path + astrometry_file, template_output=astrometry_path + template_file) else: print(f'No astrometry file corresponding to {file} found.') # TODO: This will break if more than two filters have been used on this observation. # Rewrite align to work with one template and multiple comparisons. if params['astrometry_tweak']: tweak_final(epoch=obj, sextractor_path=sextractor_path, destination=astrometry_path, instrument='FORS2', show=show, output_suffix='astrometry_tweaked', input_suffix='astrometry', stars_only=True, specific_star=False, path_add='tweaked_image', manual=False) tweak_final(epoch=obj, sextractor_path=sextractor_path, destination=astrometry_path, instrument='FORS2', show=show, output_suffix='astrometry_tweaked_hg', input_suffix='astrometry', stars_only=True, specific_star=True, path_add='tweaked_image_hg', manual=False) if params['manual_astrometry']: tweak_final(epoch=obj, sextractor_path=sextractor_path, destination=astrometry_path, instrument='FORS2', show=show, output_suffix='astrometry_tweaked_manual', input_suffix='astrometry', stars_only=True, path_add='tweaked_image_manual', manual=True)
def main(obj, key, path, instrument): p.add_output_path(obj=obj, key=key, path=path, instrument=instrument)