def prepare_for_spec(cluster_obj: cluster.ClusterObj): try: import ciao except ImportError: print("Must be running CIAO before running prepare_for_spec.") raise io.make_directory(cluster_obj.super_comp_dir) #io.make_directory(cluster_obj.sherpa_save_dir) cluster_obj.initialize_best_fits_file() print("Preparing files for spectral analysis and copying to {super_comp_dir} for offloading computation.".format( super_comp_dir=cluster_obj.super_comp_dir )) io.copy(cluster_obj.configuration_filename, cluster_obj.super_comp_cluster_config) for observation in cluster_obj.observations: print("Copying files for {obsid}".format(obsid=observation.id)) io.copy(observation.clean, cluster_obj.acisI_clean_obs(observation.id)) io.copy(observation.back, cluster_obj.backI_clean_obs(observation.id)) io.copy(observation.aux_response_file, observation.arf_sc) io.copy(observation.redistribution_matrix_file, observation.rmf_sc) #io.copy(observation.effdtime, cluster_obj.effdtime_file_obs(observation.id)) #io.copy(observation.effbtime, cluster_obj.effbtime_file_obs(observation.id)) #io.copy(observation.scale_map_region_list, cluster_obj.scalemap_regionlist_file_obs(observation.id)) io.copy(observation.acis_mask, observation.acis_mask_sc) exposure = ciao.get_exposure(observation.clean) io.write_contents_to_file(exposure, observation.exposure_time_file, binary=False)
def initialize_cluster(self): print("Initializing cluster object") self.get_cluster_info_from_user() io.make_directory(self.directory) self.write_cluster_data() io.make_initial_directories(self) print("Initialization complete.") # Next step is to run the following command: ")
def runpipe5(cluster): # This portion of the pypeline combined_dir = cluster.combined_directory io.make_directory(combined_dir) while not io.file_exists(cluster.master_crop_file): print("Master crop file not found") run_ds9_for_master_crop(cluster) for observation in cluster.observations: # clean data infile = "{}[sky=region({})]".format(observation.clean, cluster.master_crop_file) outfile = cluster.temp_acisI_comb clobber = True rt.dmcopy.punlearn() rt.dmcopy(infile=infile, outfile=outfile, clobber=clobber) infile = "{}[bin sky=4][energy=700:8000]".format( cluster.temp_acisI_comb) outfile = observation.acisI_comb_img clobber = True print("ObsID: {}\t- Extracting just 0.7keV - 8keV.".format( observation.id)) rt.dmcopy.punlearn() rt.dmcopy(infile=infile, outfile=outfile, clobber=clobber) # background infile = "{}[sky=region({})]".format(observation.back, cluster.master_crop_file) outfile = cluster.temp_backI_comb clobber = True rt.dmcopy.punlearn() rt.dmcopy(infile=infile, outfile=outfile, clobber=clobber) infile = "{}[bin sky=4][energy=700:8000]".format( cluster.temp_backI_comb) outfile = observation.backI_comb_img clobber = True rt.dmcopy.punlearn() rt.dmcopy(infile=infile, outfile=outfile, clobber=clobber) io.delete(cluster.temp_acisI_comb) io.delete(cluster.temp_backI_comb) make_mask_file(observation) make_cumulative_mask_file(cluster, observation) create_combined_images(cluster) make_nosrc_cropped_xray_sb(cluster)
def runpipe5(cluster): print("runpipe5") from astropy.io import fits # This portion of the pypeline combined_dir = cluster.combined_directory io.make_directory(combined_dir) while not io.file_exists(cluster.master_crop_file): print("Master crop file not found") run_ds9_for_master_crop(cluster) # the contents of this for loop should be refactored/replaced with the make_acisI_and_back function for observation in cluster.observations: infile = "{}[sky=region({})]".format(observation.clean, cluster.master_crop_file) outfile = cluster.temp_acisI_comb clobber = True rt.dmcopy.punlearn() rt.dmcopy(infile=infile, outfile=outfile, clobber=clobber) print("{} shape: {}".format(outfile, fits.open(outfile)[0].shape)) infile = "{}[bin sky=4][energy=700:8000]".format(cluster.temp_acisI_comb) outfile = observation.acisI_comb_img clobber = True print("ObsID: {}\t- Extracting just 0.7keV - 8keV.".format(observation.id)) rt.dmcopy.punlearn() rt.dmcopy(infile=infile, outfile=outfile, clobber=clobber) # background infile = "{}[sky=region({})]".format(observation.back, cluster.master_crop_file) outfile = cluster.temp_backI_comb clobber = True rt.dmcopy.punlearn() rt.dmcopy(infile=infile, outfile=outfile, clobber=clobber) infile = "{}[bin sky=4][energy=700:8000]".format(cluster.temp_backI_comb) outfile = observation.backI_comb_img clobber = True rt.dmcopy.punlearn() rt.dmcopy(infile=infile, outfile=outfile, clobber=clobber) io.delete(cluster.temp_acisI_comb) io.delete(cluster.temp_backI_comb) make_mask_file(observation) make_cumulative_mask_file(cluster, observation) create_combined_images(cluster) make_nosrc_cropped_xray_sb(cluster)
def prepare_to_merge_observations_from(cluster_obj): print("Preparing to merge the observations.") for observation in cluster_obj.observations: print("Preparing observation: {id}".format(id=observation.id)) io.make_directory(observation.analysis_directory) io.make_directory(observation.reprocessing_directory) io.copytree(src=observation.primary_directory, dst=observation.analysis_directory) io.copytree(src=observation.secondary_directory, dst=observation.analysis_directory) return
def initialize_cluster(name="", obsids=[], abundance=0.3, redshift=0.0, nH=0.0): clstr = cluster.ClusterObj(name=name, observation_ids=obsids, abundance=abundance, redshift=redshift, hydrogen_column_density=nH, data_directory=config.data_directory()) print('Making initial cluster directory: {}'.format(clstr.directory)) io.make_directory(clstr.directory) io.make_initial_directories(clstr) clstr.last_step_completed = 1 print("Downloading cluster data.") download_data(clstr) clstr.last_step_completed = 2 print("Merging observations.") merge_observations(clstr) clstr.last_step_completed = 3
def actually_merge_observations_from(cluster): print("Merging observations from {}.".format(cluster.name)) merged_directory = io.get_path('{}/merged_obs_evt2/'.format( cluster.directory)) io.make_directory(merged_directory) os.chdir(merged_directory) merged_observations = [] for observation in cluster.observation_list: evt2_file = "{}/{}/analysis/evt2.fits".format(cluster.directory, observation) merged_observations.append(evt2_file) merged_lis = "{}/merged_obs.lis".format(merged_directory) io.write_contents_to_file("\n".join(merged_observations), merged_lis, binary=False) outroot = io.get_path("{}/{}".format(cluster.directory, cluster.name)) infile = "@{infile}[ccd_id=0:3]".format(infile=merged_lis) # for ACIS-I # infile = "@{infile}".format(infile=merged_lis) # for ACIS-I & ACIS-S xygrid = "1500:6500:4,1500:6500:4" if len(merged_observations) == 1: rt.fluximage.punlearn() rt.fluximage(infile=infile, outroot=outroot, xygrid=xygrid, clobber=True) print("Only single observation, flux image created.") elif len(merged_observations) > 1: rt.merge_obs.punlearn() rt.merge_obs(infiles=infile, outroot=outroot, xygrid=xygrid, clobber=True, parallel=True, nproc=12)
def automated_cluster_init(batch_file): print("Automated cluster initialization using: {batch_file}".format(batch_file=batch_file)) data_directory = config.data_directory() csv_clusters = io.get_cluster_info_from_csv(batch_file) for clstr in csv_clusters: cluster_obj = cluster.ClusterObj(name=clstr['name'], observation_ids=clstr['obsids'], data_directory=data_directory, abundance=clstr['abundance'], redshift=clstr['redshift'], hydrogen_column_density=clstr['hydrogen_column_density'] ) io.make_directory(cluster_obj.directory) cluster_obj.write_cluster_data() io.make_initial_directories(cluster_obj) cluster_obj.last_step_completed = 1 download_data(cluster_obj) cluster_obj.last_step_completed = 2 merge_observations(cluster_obj) cluster_obj.last_step_completed = 3
def make_temperature_map(cluster: cluster.ClusterObj, resolution, average=False): #coordinates = get_pixel_coordinates(cluster) # indeces of this array are the region number minus 1 # that is, region number 1 is coordinate array index 0 # region 100 = coordinates[99] # high_res_offset = 0 # med_res_offset = 1 # low_res_offset = 2 io.make_directory(cluster.output_dir) offset = [None, 2, 1, 0][resolution] mask_fits = fits.open(cluster.combined_mask) mask = mask_fits[0].data scale_map_regions = cluster.scale_map_region_index temps_with_errors = cluster.average_temperature_fits if average else cluster.temperature_fits temperature_map = np.zeros(mask.shape) temperature_error_map = np.zeros(mask.shape) temperature_fractional_error_map = np.zeros(mask.shape) regions = temps_with_errors['region'] temperatures = temps_with_errors['temperature'] temp_error_plus = temps_with_errors['temp_err_plus'] temp_error_minus = temps_with_errors['temp_err_minus'] for i, region in enumerate(regions): if i % 1000 == 0: _update_completed_things(i, len(regions), "regions") coordinates = cluster.coordinates_for_scale_map_region(region, scale_map_regions) x = int(coordinates[0]) y = int(coordinates[1]) low_x = x - offset high_x = x + offset + 1 low_y = y - offset high_y = y + offset + 1 temperature_map[low_x:high_x, low_y:high_y] = temperatures[i] temperature_error_map[low_x:high_x, low_y:high_y] = (np.abs(temp_error_plus[i] - temp_error_minus[i]))/2 temperature_fractional_error_map[low_x:high_x, low_y:high_y] = \ (temperature_error_map[x,y]/temperature_map[x,y])*100 _update_completed_things(i, len(regions), "regions") header = mask_fits[0].header # This header contains all coordinate information needed fits.writeto(cluster.temperature_map_filename, temperature_map, header, overwrite=True) fits.writeto(cluster.temperature_error_map_filename, temperature_error_map, header, overwrite=True) fits.writeto(cluster.temperature_fractional_error_map_filename, temperature_fractional_error_map, header, overwrite=True)
def create_scale_map(cluster): target_sn = cluster.signal_to_noise mask = cluster.combined_mask_data cts_image = np.zeros(mask.shape) back_rescale = np.zeros(mask.shape) for obs in cluster.observations: cts_image += obs.acisI_combined_image t_obs = obs.acisI_combined_image_header['EXPOSURE'] t_back = obs.backI_combined_image_header['EXPOSURE'] back_rescale += (t_obs/t_back)*obs.backI_combined_image signal = cts_image - back_rescale signal[np.where(signal < 0)] = 0 sz = signal.shape max_x = sz[0] max_y = sz[1] # radius_map = np.zeros(sz) scale_map = np.zeros(sz) sn_map = np.zeros(sz) pix_x = np.zeros(sz) pix_y = np.zeros(sz) for j in range(max_y): for i in range(max_x): pix_x[i, j] = float(i) pix_y[i, j] = float(j) max_radius = 100 r = max_radius + 1 # +1 may be a remnant of IDL indexing #dr = 24.0 min_dr = 0.125 num_pix = max_x * max_y ci = 0 counter = 0 start_time = time.time() for cj in range(0, max_y): #print("{} out of {} pixels complete.".format(cj*ci, num_pix)) _update_completed_things(cj*ci, num_pix, "pixels") for ci in range(0, max_x): if mask[ci, cj] == 1: delta_x = ci-pix_x delta_y = cj-pix_y # an array where each value is the distance away from ci,cj radius = np.sqrt(delta_x**2 + delta_y**2) dr = 24.0 hilo = 0 niter = 0 while (dr > min_dr) and (niter < 100): indeces = np.where(radius <= r) cts_map_total = np.sum(cts_image[indeces]) if cts_map_total == 0: counter += 1 _source_free_region(counter, ci*cj, num_pix) sn_val = 0 hilo = -1 else: backmap_tot = np.sum(back_rescale[indeces]) signal_total = cts_map_total - backmap_tot noise_total = np.sqrt(cts_map_total+backmap_tot) sn_val = signal_total/noise_total if float(sn_val) < float(target_sn): if r > max_radius: r = max_radius + 1 niter = 110 # exit by setting niter=110. # (niter=100 means niter hit max niter. # niter=110 means radius hit max radius) sn_map[ci,cj] = 0 scale_map[ci,cj] = 0 else: if hilo == 1: dr *= 0.5 r += dr hilo = -1 else: snmapval = sn_map[ci, cj] if (sn_val < snmapval) or (snmapval == 0.0): sn_map[ci,cj] = sn_val scale_map[ci, cj] = r if hilo == -1: dr *= 0.5 r -= dr hilo = 1 niter += 1 # end while end_time = time.time() print("Time elapsed {:0.2f} seconds.".format(end_time - start_time)) io.make_directory(cluster.acb_dir) header = cluster.observations[0].acisI_combined_image_header io.write_numpy_array_to_fits(scale_map, cluster.scale_map_file, header=header) io.write_numpy_array_to_fits(sn_map, cluster.sn_map, header=header)
def create_scale_map_in_parallel(cluster: cluster.ClusterObj): mask = cluster.combined_mask_data cts_image = np.zeros(mask.shape) back_rescale = np.zeros(mask.shape) for obs in cluster.observations: cts_image += obs.acisI_combined_image t_obs = obs.acisI_combined_image_header['EXPOSURE'] t_back = obs.backI_combined_image_header['EXPOSURE'] back_rescale += (t_obs / t_back) * obs.backI_combined_image signal = cts_image - back_rescale signal[np.where(signal < 0)] = 0 sz = signal.shape max_x = sz[0] max_y = sz[1] io.make_directory(cluster.acb_dir) cluster.initialize_scale_map_csv() pix_x = np.zeros(sz) pix_y = np.zeros(sz) for j in range(max_y): for i in range(max_x): pix_x[i, j] = float(i) pix_y[i, j] = float(j) num_pix = max_x * max_y start_time = time.time() indices = np.vstack(np.where(mask==1)).T num_index_lists = (indices.shape[0] // mp.cpu_count()) index_lists = np.array_split(indices, num_index_lists) num_iterations = len(index_lists) for i, index_list in enumerate(index_lists): if i % 100 == 0: print("{} of {} iterations complete.".format(i, num_iterations)) processes = [mp.Process(target=calculate_radius_at_index, args=(index, cluster, pix_x, pix_y, cts_image, num_pix, back_rescale)) for index in index_list] for process in processes: process.start() for process in processes: process.join() cluster.write_scale_map_csv_to_fits() end_time = time.time() print("Time elapsed {:0.2f} seconds.".format(end_time - start_time))
def create_global_response_file_for(cluster, obsid, region_file): observation = cluster.observation(obsid) #min_counts = 525 obs_analysis_dir = observation.analysis_directory global_response_dir = "{}/globalresponse/".format(obs_analysis_dir) io.make_directory(global_response_dir) clean = observation.clean back = observation.back pbk0 = io.get_filename_matching("{}/acis*pbk0*.fits".format(obs_analysis_dir))[0] bad_pixel_file = io.get_filename_matching("{}/bpix1_new.fits".format(obs_analysis_dir))[0] rt.ardlib.punlearn() rt.acis_set_ardlib(badpixfile=bad_pixel_file) mask_file = io.get_filename_matching("{}/*msk1.fits".format(obs_analysis_dir)) make_pcad_lis(cluster, obsid) infile = "{}[sky=region({})]".format(clean, region_file) outroot = "{}/acisI_region_0".format(global_response_dir) weight = True correct_psf = False pcad = "@{}/pcad_asol1.lis".format(obs_analysis_dir) combine = False bkg_file = "" bkg_resp = False group_type = "NUM_CTS" binspec = 1 clobber = True rt.specextract(infile=infile, outroot=outroot, weight=weight, correctpsf=correct_psf, asp=pcad, combine=combine, mskfile=mask_file, bkgfile=bkg_file, bkgresp=bkg_resp, badpixfile=bad_pixel_file, grouptype=group_type, binspec=binspec, clobber=clobber) infile = "{}[sky=region({})][bin pi]".format(back, region_file) outfile = "{}/acisI_back_region_0.pi".format(global_response_dir) clobber = True rt.dmextract.punlearn() print("Running: dmextract infile={}, outfile={}, clobber={}".format(infile, outfile, clobber)) rt.dmextract(infile=infile, outfile=outfile, clobber=clobber) rt.dmhedit.punlearn() infile = "{}/acisI_region_0.pi".format(global_response_dir) filelist = "" operation = "add" key = "BACKFILE" value = outfile rt.dmhedit(infile=infile, filelist=filelist, operation=operation, key=key, value=value) observation = cluster.observation(obsid) aux_response_file = '{global_response_directory}/acisI_region_0.arf'.format( global_response_directory=observation.global_response_directory) redist_matrix_file = '{global_response_directory}/acisI_region_0.rmf'.format( global_response_directory=observation.global_response_directory) io.copy(aux_response_file, observation.aux_response_file) io.copy(redist_matrix_file, observation.redistribution_matrix_file)
def create_scale_map(cluster): from astropy.io import fits target_sn = cluster.target_sn mask_fits = fits.open(cluster.combined_mask) cts_image = np.zeros(mask_fits[0].data.shape) back_rescale = np.zeros(mask_fits[0].data.shape) t_obs = 0.0 t_back = 0.0 for obs in cluster.observations: obs_cts_image_fits = fits.open(obs.acisI_comb_img) obs_back_image_fits = fits.open(obs.backI_comb_img) cts_image += obs_cts_image_fits[0].data t_obs = obs_cts_image_fits[0].header['EXPOSURE'] # back_rescale += obs_back_image_fits[0].data t_back = obs_back_image_fits[0].header['EXPOSURE'] back_rescale += (t_obs/t_back)*obs_back_image_fits[0].data sigma_back = np.sqrt(back_rescale) sigma_cts = np.sqrt(cts_image) signal = cts_image - back_rescale signal[np.where(signal < 0)] = 0 noise = np.sqrt(sigma_cts**2 + sigma_back**2) scale_file = cluster.scale_map_file sn_file = cluster.sn_map sz = signal.shape nx = sz[0] ny = sz[1] radius_map = np.zeros(sz) scale_map = np.zeros(sz) sn_map = np.zeros(sz) pix_x = np.zeros(sz) pix_y = np.zeros(sz) for j in range(ny): for i in range(nx): pix_x[i, j] = float(i) pix_y[i, j] = float(j) bpix_x = 0 bpix_y = 0 epix_x = nx epix_y = ny max_radius = 100 r = max_radius + 1 # +1 may be a remnant of IDL indexing dr = 24.0 min_dr = 0.125 det = mask_fits[0].data num_pix = nx*ny ci=0 start_time = time.time() # output_queue = mp.Queue() for cj in range(bpix_y,epix_y): #print("{} out of {} pixels complete.".format(cj*ci, num_pix)) _update_completed_things(cj*ci, num_pix, "pixels") for ci in range(bpix_x, epix_x): if det[ci,cj] == 1: delta_x = ci-pix_x delta_y = cj-pix_y # an array where each value is the distance away from ci,cj radius = np.sqrt(delta_x**2 + delta_y**2) dr = 24.0 hilo = 0 niter = 0 while (dr > min_dr) and (niter < 100): indeces = np.where(radius <= r) cts_map_total = np.sum(cts_image[indeces]) if cts_map_total == 0: print("Encountered a source-free region -- recalculating...") sn_val = 0 hilo = -1 else: backmap_tot = np.sum(back_rescale[indeces]) signal_total = cts_map_total - backmap_tot noise_total = np.sqrt(cts_map_total+backmap_tot) sn_val = signal_total/noise_total if sn_val < target_sn: if r > max_radius: r = max_radius + 1 niter = 110 # exit by setting niter=110. # (niter=100 means niter hit max niter. # niter=110 means radius hit max radius) sn_map[ci,cj] = 0 scale_map[ci,cj] = 0 else: if hilo == 1: dr *= 0.5 r += dr hilo = -1 else: snmapval = sn_map[ci,cj] if (sn_val < snmapval) or (snmapval == 0.0): sn_map[ci,cj] = sn_val scale_map[ci,cj] = r if hilo == -1: dr *= 0.5 r -= dr hilo = 1 niter += 1 # end while end_time = time.time() print("Time elapsed {:0.2f} seconds.".format(end_time - start_time)) io.make_directory(cluster.acb_dir) obs_cts_image_fits[0].data = scale_map obs_cts_image_fits.writeto(cluster.scale_map_file, overwrite=True) obs_cts_image_fits[0].data = sn_map obs_cts_image_fits.writeto(cluster.sn_map, overwrite=True)