def runpipe5(cluster): # This portion of the pypeline combined_dir = cluster.combined_directory io.make_directory(combined_dir) while not io.file_exists(cluster.master_crop_file): print("Master crop file not found") run_ds9_for_master_crop(cluster) for observation in cluster.observations: # clean data infile = "{}[sky=region({})]".format(observation.clean, cluster.master_crop_file) outfile = cluster.temp_acisI_comb clobber = True rt.dmcopy.punlearn() rt.dmcopy(infile=infile, outfile=outfile, clobber=clobber) infile = "{}[bin sky=4][energy=700:8000]".format( cluster.temp_acisI_comb) outfile = observation.acisI_comb_img clobber = True print("ObsID: {}\t- Extracting just 0.7keV - 8keV.".format( observation.id)) rt.dmcopy.punlearn() rt.dmcopy(infile=infile, outfile=outfile, clobber=clobber) # background infile = "{}[sky=region({})]".format(observation.back, cluster.master_crop_file) outfile = cluster.temp_backI_comb clobber = True rt.dmcopy.punlearn() rt.dmcopy(infile=infile, outfile=outfile, clobber=clobber) infile = "{}[bin sky=4][energy=700:8000]".format( cluster.temp_backI_comb) outfile = observation.backI_comb_img clobber = True rt.dmcopy.punlearn() rt.dmcopy(infile=infile, outfile=outfile, clobber=clobber) io.delete(cluster.temp_acisI_comb) io.delete(cluster.temp_backI_comb) make_mask_file(observation) make_cumulative_mask_file(cluster, observation) create_combined_images(cluster) make_nosrc_cropped_xray_sb(cluster)
def get_cluster(config_file_arg): if io.file_exists(config_file_arg): cluster_obj = cluster.read_cluster_data(config_file_arg) else: config_file = cluster.get_cluster_config(config_file_arg) if config_file: cluster_obj = cluster.read_cluster_data(config_file) else: print("Error finding cluster configuration file. Try passing the full path to the file.") return return cluster_obj
def runpipe5(cluster): print("runpipe5") from astropy.io import fits # This portion of the pypeline combined_dir = cluster.combined_directory io.make_directory(combined_dir) while not io.file_exists(cluster.master_crop_file): print("Master crop file not found") run_ds9_for_master_crop(cluster) # the contents of this for loop should be refactored/replaced with the make_acisI_and_back function for observation in cluster.observations: infile = "{}[sky=region({})]".format(observation.clean, cluster.master_crop_file) outfile = cluster.temp_acisI_comb clobber = True rt.dmcopy.punlearn() rt.dmcopy(infile=infile, outfile=outfile, clobber=clobber) print("{} shape: {}".format(outfile, fits.open(outfile)[0].shape)) infile = "{}[bin sky=4][energy=700:8000]".format(cluster.temp_acisI_comb) outfile = observation.acisI_comb_img clobber = True print("ObsID: {}\t- Extracting just 0.7keV - 8keV.".format(observation.id)) rt.dmcopy.punlearn() rt.dmcopy(infile=infile, outfile=outfile, clobber=clobber) # background infile = "{}[sky=region({})]".format(observation.back, cluster.master_crop_file) outfile = cluster.temp_backI_comb clobber = True rt.dmcopy.punlearn() rt.dmcopy(infile=infile, outfile=outfile, clobber=clobber) infile = "{}[bin sky=4][energy=700:8000]".format(cluster.temp_backI_comb) outfile = observation.backI_comb_img clobber = True rt.dmcopy.punlearn() rt.dmcopy(infile=infile, outfile=outfile, clobber=clobber) io.delete(cluster.temp_acisI_comb) io.delete(cluster.temp_backI_comb) make_mask_file(observation) make_cumulative_mask_file(cluster, observation) create_combined_images(cluster) make_nosrc_cropped_xray_sb(cluster)
def process_commandline_arguments(cluster_obj): print("Processing commandline arguments") args = get_arguments() cluster_obj = None if io.file_exists(args.batch_file): ciao.automated_cluster_init(args.batch_file) if None not in [args.name, args.abundance, args.nH, args.redshift, args.obsids]: ciao.initialize_cluster(name=args.name, obsids=args.obsids, abundance=args.abundance, redshift=args.redshift, nH=args.nH) if args.init_cluster: cluster_obj.initialize_cluster() if args.config_file != "": if io.file_exists(args.config_file): cluster_obj = cluster.read_cluster_data(args.config_file) else: config_file = cluster.get_cluster_config(args.config_file) if config_file: cluster_obj = cluster.read_cluster_data(config_file) else: print("Error finding cluster configuration file. Try passing the full path to the file.") return if args.cont: if cluster_obj is None: cluster_obj = config.current_cluster() if cluster_obj is None: print("Cannot find a current working cluster.") exit(-1) print("Continuing {}.".format(cluster_obj.name)) ciao.start_from_last(cluster_obj) if args.download_data: if 0 == cluster_obj.last_step_completed: cluster_obj.get_cluster_info_from_user() ciao.download_data(cluster_obj) if args.remove_sources: ciao.remove_sources(cluster_obj) return cluster_obj
def make_response_files(cluster): for obsid in cluster.observation_ids: print("Making response files for observation {}".format(obsid)) obs_analysis_dir = cluster.obs_analysis_directory(obsid) region_file = io.get_path("{}/acisI_region_0.reg".format(obs_analysis_dir)) if (not io.file_exists(region_file)) or (io.file_size(region_file) == 0): print("Region file {} does not exist.".format(region_file)) print("When DS9 opens, draw a small circle that covers a piece of each ACIS-I chip (~20 pixels) and save it as:\n" \ "{}".format(region_file)) print("Opening SAO DS9") io.write_contents_to_file("", region_file, False) ds9_arguments = "ds9 -regions system physical -regions shape circle -regions format ciao -zoom 0.5 " \ "-bin factor 4 {}/acisI_clean.fits".format(obs_analysis_dir) subprocess.run([ds9_arguments], shell=True) print('Creating global response file.') create_global_response_file_for(cluster, obsid, region_file)
def process_commandline_arguments(cluster_obj): print("Processing commandline arguments") args = get_arguments() cluster_obj = None if None not in [ args.name, args.abundance, args.nH, args.redshift, args.obsids ]: ciao.initialize_cluster(name=args.name, obsids=args.obsids, abundance=args.abundance, redshift=args.redshift, nH=args.nH) if args.init_cluster: cluster_obj.initialize_cluster() if args.config_file != "": if io.file_exists(args.config_file): cluster_obj = cluster.read_cluster_data(args.config_file) else: config_file = cluster.get_cluster_config(args.config_file) if config_file: cluster_obj = cluster.read_cluster_data(config_file) else: print( "Error finding cluster configuration file. Try passing the full path to the file." ) return if args.find_sources: ciao.find_sources(cluster_obj, ecf=args.ecf, energy=args.energy) if args.cont: if cluster_obj is None: cluster_obj = config.current_cluster() if cluster_obj is None: print("Cannot find a current working cluster.") exit(-1) if args.parallel and cluster_obj.last_step_completed == '1': ciao.run_stage_2_parallel(cluster_obj, args) else: ciao.start_from_last(cluster_obj, args) return cluster_obj
def process_commandline_arguments(cluster_obj): print("Processing commandline arguments") args = get_arguments() if io.file_exists(args.batch_file): ciao.automated_cluster_init(args.batch_file) if None not in [args.name, args.abundance, args.nH, args.redshift, args.obsids]: ciao.initialize_cluster(name=args.name, obsids=args.obsids, abundance=args.abundance, redshift=args.redshift, nH=args.nH) if args.init_cluster: cluster_obj.initialize_cluster() if args.config_file != "": cluster_obj = get_cluster(args.config_file) if args.cont: ciao.start_from_last(cluster_obj) if args.download_data: if 0 == cluster_obj.last_step_completed: cluster_obj.get_cluster_info_from_user() ciao.download_data(cluster_obj) if args.remove_sources: ciao.remove_sources(cluster_obj) return cluster_obj
def make_cumulative_mask_file(cluster, observation): from astropy.io import fits cumulative_mask_filename = cluster.combined_mask current_obs_mask_filename = observation.acisI_combined_mask if not io.file_exists(cumulative_mask_filename): print("Cumulative mask file not found. Creating it.") cumulative_mask = fits.open(current_obs_mask_filename) cumulative_mask.writeto(cumulative_mask_filename) else: current_mask = fits.open(current_obs_mask_filename) cumulative_mask = fits.open(cumulative_mask_filename) current_mask[0].data = current_mask[0].data + cumulative_mask[0].data # from idl # newcumulativemask = currentmask + cumulativemask # newcumulativemask[where(newcumulativemask gt 1)] = 1 # all values that are greater than 1 make 1? current_mask[0].data[np.where(current_mask[0].data > 1)] = 1 current_mask.writeto(cumulative_mask_filename, overwrite=True)
def make_cumulative_mask_file(cluster, observation): from astropy.io import fits cumulative_mask_filename = cluster.combined_mask current_obs_mask_filename = observation.acisI_combined_mask if not io.file_exists(cumulative_mask_filename): print("Cumulative mask file not found. Creating it.") cumulative_mask = fits.open(current_obs_mask_filename) cumulative_mask.writeto(cumulative_mask_filename) else: current_mask = fits.open(current_obs_mask_filename) cumulative_mask = fits.open(cumulative_mask_filename) print("Cumulative mask {} shape:{}".format(cumulative_mask_filename, cumulative_mask[0].shape)) print("current mask {} shape:{}".format(current_obs_mask_filename, current_mask[0].shape)) current_mask[0].data = current_mask[0].data + cumulative_mask[0].data current_mask[0].data[np.where(current_mask[0].data > 1)] = 1 current_mask.writeto(cumulative_mask_filename, overwrite=True)
def prepare_efftime_circle(cluster): try: from ciao_contrib import runtool as rt except ImportError: print("Failed to import CIAO python scripts. ") raise for observation in cluster.observations: io.delete_if_exists(observation.effbtime) io.delete_if_exists(observation.effdtime) if not io.file_exists(observation.acisI_nosrc_combined_mask_file): print("Removing point sources from the observations combined mask file.") print("dmcopy infile='{}[exclude sky=region({})]' outfile={} clobber=True".format( observation.acisI_combined_mask_file, cluster.sources_file, observation.acisI_nosrc_combined_mask_file )) rt.dmcopy.punlearn() rt.dmcopy( infile="{fits_file}[exclude sky=region({source_file})]".format( fits_file=observation.acisI_combined_mask_file, source_file=cluster.sources_file ), outfile=observation.acisI_nosrc_combined_mask_file, clobber=True ) else: print("{acis} already exists.".format( acis=observation.acisI_nosrc_combined_mask_file )) if not io.file_exists(observation.acisI_high_energy_combined_image_file): print("Creating high band (9.5-12 keV) source image cropped to combined region.") rt.dmcopy.punlearn() rt.dmcopy( infile="{fits_file}[sky=region({crop_file})]".format( fits_file=observation.clean, crop_file=cluster.master_crop_file ), outfile=observation.acisI_high_energy_temp_image, clobber=True ) rt.dmcopy.punlearn() rt.dmcopy( infile="{fits_file}[EVENTS][bin sky=4][energy=9500:12000]".format( fits_file=observation.acisI_high_energy_temp_image ), outfile=observation.acisI_high_energy_combined_image_file, option="image", clobber=True ) else: print("{fits_file} already exists.".format( fits_file=observation.acisI_high_energy_combined_image_file )) io.delete_if_exists(observation.acisI_high_energy_temp_image) if not io.file_exists(observation.backI_high_energy_combined_image_file): print("Creating high band (9.5-12 keV) background image cropped to combined region.") rt.dmcopy.punlearn() rt.dmcopy( infile="{fits_file}[sky=region({crop_file})]".format( fits_file=observation.back, crop_file=cluster.master_crop_file ), outfile=observation.backI_high_energy_temp_image, clobber=True ) rt.dmcopy.punlearn() rt.dmcopy( infile="{fits_file}[EVENTS][bin sky=4][energy=9500:12000]".format( fits_file=observation.backI_high_energy_temp_image ), outfile=observation.backI_high_energy_combined_image_file, option="image", clobber=True ) else: print("{fits_file} already exists.".format( fits_file=observation.backI_high_energy_combined_image_file )) io.delete_if_exists(observation.backI_high_energy_temp_image)
def prepare_effective_time_circles_for(observation: cluster.Observation): io.delete_if_exists(observation.effbtime) io.delete_if_exists(observation.effdtime) if not io.file_exists(observation.acisI_nosrc_combined_mask_file): print("Removing point sources from the observations combined mask file.") print("dmcopy infile='{}[exclude sky=region({})]' outfile={} clobber=True".format( observation.acisI_combined_mask_file, observation.cluster.sources_file, observation.acisI_nosrc_combined_mask_file )) rt.dmcopy.punlearn() rt.dmcopy( infile="{fits_file}[exclude sky=region({source_file})]".format( fits_file=observation.acisI_combined_mask_file, source_file=observation.cluster.sources_file ), outfile=observation.acisI_nosrc_combined_mask_file, clobber=True ) else: print("{acis} already exists.".format( acis=observation.acisI_nosrc_combined_mask_file )) # if not io.file_exists(observation.acisI_high_energy_combined_image_file): print("Creating high band (9.5-12 keV) source image cropped to combined region.") rt.dmcopy.punlearn() rt.dmcopy( infile="{fits_file}[sky=region({crop_file})]".format( fits_file=observation.clean, crop_file=observation.cluster.master_crop_file ), outfile=observation.acisI_high_energy_temp_image, clobber=True ) ##########need to change to obs specific rt.dmcopy.punlearn() rt.dmcopy( infile="{fits_file}[EVENTS][bin sky=4][energy=9500:12000]".format( fits_file=observation.acisI_high_energy_temp_image ), outfile=observation.acisI_high_energy_combined_image_file, option="image", clobber=True ) io.delete_if_exists(observation.acisI_high_energy_temp_image) print("Creating high band (9.5-12 keV) background image cropped to combined region.") rt.dmcopy.punlearn() rt.dmcopy( infile="{fits_file}[sky=region({crop_file})]".format( fits_file=observation.back, crop_file=observation.cluster.master_crop_file ), outfile=observation.backI_high_energy_temp_image, clobber=True ) rt.dmcopy.punlearn() rt.dmcopy( infile="{fits_file}[EVENTS][bin sky=4][energy=9500:12000]".format( fits_file=observation.backI_high_energy_temp_image ), outfile=observation.backI_high_energy_combined_image_file, option="image", clobber=True ) io.delete_if_exists(observation.backI_high_energy_temp_image)
def lightcurves_with_exclusion(cluster): for observation in cluster.observations: # data_nosrc_hiEfilter = "{}/acisI_nosrc_fullE.fits".format(obs_analysis_dir) data_nosrc_hiEfilter = "{}/acisI_nosrc_hiEfilter.fits".format(observation.analysis_directory) print("Creating the image with sources removed") data = observation.acis_nosrc_filename image_nosrc = "{}/img_acisI_nosrc_fullE.fits".format(observation.analysis_directory) if io.file_exists(observation.exclude_file): print("Removing sources from event file to be used in lightcurve") infile = "{}[exclude sky=region({})]".format(data_nosrc_hiEfilter, observation.exclude) outfile = "{}/acisI_lcurve.fits".format(observation.analysis_directory) clobber = True rt.dmcopy.punlearn() rt.dmcopy(infile=infile, outfile=outfile, clobber=clobber) data_lcurve = "{}/acisI_lcurve.fits".format(observation.analysis_directory) else: yes_or_no = io.check_yes_no( "Are there sources to be excluded from observation {} while making the lightcurve? ".format(observation.id)) if yes_or_no: # yes_or_no == True print("Create the a region file with the region to be excluded and save it as {}".format(observation.exclude_file)) else: data_lcurve = data_nosrc_hiEfilter backbin = 259.28 echo = True tstart = rt.dmkeypar(infile=data_nosrc_hiEfilter, keyword="TSTART", echo=echo) tstop = rt.dmkeypar(infile=data_nosrc_hiEfilter, keyword="TSTOP", echo=echo) print("Creating lightcurve from the events list with dmextract") infile = "{}[bin time={}:{}:{}]".format(data_lcurve, tstart, tstop, backbin) outfile = "{}/acisI_lcurve.lc".format(observation.analysis_directory) opt = "ltc1" rt.dmextract.punlearn() rt.dmextract(infile=infile, outfile=outfile, opt=opt, clobber=clobber) lcurve = outfile print("Cleaning the lightcurve by removing flares with deflare. Press enter to continue.") rt.deflare.punlearn() infile = lcurve outfile = "{}/acisI_gti.gti".format(observation.analysis_directory) method = "clean" save = "{}/acisI_lcurve".format(observation.analysis_directory) rt.deflare(infile=infile, outfile=outfile, method=method, save=save) gti = outfile print("filtering the event list using GTI info just obtained.") infile = "{}[@{}]".format(data_nosrc_hiEfilter, gti) outfile = observation.clean clobber = True rt.dmcopy(infile=infile, outfile=outfile, clobber=clobber) data_clean = outfile print("Don't forget to check the light curves!")
def ciao_back(cluster, overwrite=False): print("Running ciao_back on {}.".format(cluster.name)) for observation in cluster.observations: pcad_file = make_pcad_lis(cluster, observation.id) backI_lis = [] backS_lis = [] analysis_path = observation.analysis_directory filelist = io.read_contents_of_file(observation.ccd_merge_list).split('\n') pcad = io.read_contents_of_file(pcad_file) for acis_file in filelist: rt.acis_bkgrnd_lookup.punlearn() print("Finding background for {}".format(acis_file)) path_to_background = rt.acis_bkgrnd_lookup(infile=acis_file) print("Found background at {}".format(path_to_background)) acis_id = int(acis_file.split('/')[-1].split('.')[-2][-1]) assert isinstance(acis_id, int), "acis_id = {}".format(acis_id) assert not isinstance(path_to_background, type(None)), "Cannot find background {}".format(acis_file) local_background_path = io.get_path("{}/back_ccd{}.fits".format(analysis_path, acis_id)) try: if io.file_exists(local_background_path) and overwrite: io.delete(local_background_path) io.copy(path_to_background, local_background_path) except OSError: print("Problem copying background file {}. Do you have the right permissions and a full CALDB?".format( path_to_background)) raise acis_gain = rt.dmkeypar(infile=acis_file, keyword="GAINFILE", echo=True) background_gain = rt.dmkeypar(infile=local_background_path, keyword="GAINFILE", echo=True) print("{}/{}/acis_ccd{}.fits gain: {}".format(cluster.name, observation.id, acis_id, acis_gain)) print("{}/{}/back_ccd{}.fits gain: {}".format(cluster.name, observation.id, acis_id, background_gain)) if dates_and_versions_match(acis_gain, background_gain): print("Date/version numbers don't match on the acis data and background. Reprocessing.") local_background_path = reprocess(cluster, observation.id, acis_gain, background_gain, acis_id) print("Reprojecting background") rt.reproject_events.punlearn() infile = local_background_path outfile = io.get_path("{local_path}/back_reproj_ccd{acis_id}.fits".format(local_path=analysis_path, acis_id=acis_id)) match = acis_file print( "Running:\n reproject_events(infile={infile}, outfile={outfile}, aspect={pcad}, match={match})".format( infile=infile, outfile=outfile, pcad=pcad, match=match) ) rt.reproject_events(infile=infile, outfile=outfile, aspect="{pcad_file}".format(pcad_file=pcad), match=match, random=0, clobber=True) back_reproject = outfile datamode = rt.dmkeypar(infile=io.get_filename_matching(io.get_path("{}/acis*evt1*.fits".format(analysis_path))), keyword="DATAMODE") if datamode == "VFAINT": print("VFAINT Mode, resetting setting status bits") rt.dmcopy.punlearn() rt.dmcopy(infile="{}[status=0]".format(back_reproject), outfile=outfile, clobber=True) if acis_id <= 3: backI_lis.append(back_reproject) else: backS_lis.append(back_reproject) merged_back_list = backI_lis + backS_lis print("writing backI.lis and backS.lis") io.write_contents_to_file("\n".join(backI_lis), io.get_path("{}/backI.lis".format(analysis_path)), binary=False) io.write_contents_to_file("\n".join(backS_lis), io.get_path("{}/backS.lis".format(analysis_path)), binary=False) io.write_contents_to_file("\n".join(merged_back_list), observation.merged_back_lis, binary=False) return
T = clstr.temperature_map header = clstr.temperature_map_header n, T = do.make_sizes_match(n, T) P = n*T norm_P = do.normalize_data(P) fits.writeto(clstr.pressure_map_filename, norm_P, header=header, overwrite=True) if __name__ == '__main__': args, parser = get_arguments() if args.cluster_config is not None: if io.file_exists(args.cluster_config): clstr = cluster.read_cluster_data(args.cluster_config) else: clstr_config = cluster.get_cluster_config(args.cluster_config) if io.file_exists(clstr_config): clstr = cluster.read_cluster_data(clstr_config) else: print("Error finding cluster configuration file. Try passing the full path and filename.") exit(1) if args.commands: make_commands_lis(clstr, args.resolution) if args.eff_times_fits: eff_times_to_fits(clstr) elif args.temperature_map: print("Creating temperature map.")