def make_commands_lis(cluster: cluster.ClusterObj, resolution):
print("Creating {}".format(cluster.command_lis))
offset = [None, 5, 3, 1][resolution]
start_time = time.time()
region_list = cluster.scale_map_regions_to_fit(resolution)
command_string = []
pypeline_dir = io.get_user_input("Enter the directory containing the pix2pix.py portion of the pypeline on the remote machine: ")
pix2pix_path = "{pypeline_dir}/pix2pix.py".format(pypeline_dir=pypeline_dir)
data_dir = io.get_user_input("Enter the directory containing the cluster data on the remote machine:\n"
"For example: /home/user/data/clustername/\n")
for region in region_list:
new_command = "python {pix2pix} {cluster_config} {region}".format(
pix2pix=pix2pix_path,
cluster_config="{data_dir}/{name}_pypeline_config.ini".format(
data_dir=data_dir,
name=cluster.name
),
region=region
)
command_string.append(new_command)
command_lis = "\n".join(command_string)
region_string = '\n'.join([str(x) for x in region_list])
io.write_contents_to_file(command_lis, cluster.command_lis, binary=False)
io.write_contents_to_file(region_string, cluster.filtered_region_list, binary=False)
end_time = time.time()
print("Time elapsed: {time:0.2f} sec".format(time=(end_time-start_time)))
def ccd_sort(cluster):
print("Running ccd_sort on {}.".format(cluster.name))
for observation in cluster.observations:
print("Working on {}/{}".format(cluster.name, observation.id))
analysis_path = observation.analysis_directory
os.chdir(analysis_path)
evt1_filename = io.get_path("{}/{}".format(analysis_path,
io.get_filename_matching("acis*evt1.fits")[0]))
evt2_filename = io.get_path("{}/{}".format(analysis_path,
io.get_filename_matching("evt2.fits")[0]))
detname = rt.dmkeypar(infile=evt1_filename, keyword="DETNAM", echo=True)
print("evt1 : {}\nevt2 : {}\ndetname : {}".format(evt1_filename,
evt2_filename,
detname))
assert not isinstance(detname, type(None)), "detname returned nothing!"
detnums = [int(x) for x in detname.split('-')[-1]]
for acis_id in detnums:
print("{cluster}/{observation}: Making level 2 event file for ACIS Chip id: {acis_id}".format(
cluster=cluster.name,
observation=observation.id,
acis_id=acis_id))
rt.dmcopy(infile="{evt2_file}[ccd_id={acis_id}]".format(evt2_file=evt2_filename,
acis_id=acis_id),
outfile="acis_ccd{acis_id}.fits".format(acis_id=acis_id),
clobber=True)
acisI_list = io.get_filename_matching("acis_ccd[0-3].fits")
for i in range(len(acisI_list)):
acisI_list[i] = io.get_path("{obs_analysis_dir}/{file}".format(obs_analysis_dir=observation.analysis_directory,
file=acisI_list[i]))
io.write_contents_to_file("\n".join(acisI_list), observation.ccd_merge_list, binary=False)
merge_data_and_backgrounds(cluster, acisI_list)
return
def prepare_for_spec(cluster_obj: cluster.ClusterObj):
try:
import ciao
except ImportError:
print("Must be running CIAO before running prepare_for_spec.")
raise
io.make_directory(cluster_obj.super_comp_dir)
#io.make_directory(cluster_obj.sherpa_save_dir)
cluster_obj.initialize_best_fits_file()
print("Preparing files for spectral analysis and copying to {super_comp_dir} for offloading computation.".format(
super_comp_dir=cluster_obj.super_comp_dir
))
io.copy(cluster_obj.configuration_filename, cluster_obj.super_comp_cluster_config)
for observation in cluster_obj.observations:
print("Copying files for {obsid}".format(obsid=observation.id))
io.copy(observation.clean, cluster_obj.acisI_clean_obs(observation.id))
io.copy(observation.back, cluster_obj.backI_clean_obs(observation.id))
io.copy(observation.aux_response_file, observation.arf_sc)
io.copy(observation.redistribution_matrix_file, observation.rmf_sc)
#io.copy(observation.effdtime, cluster_obj.effdtime_file_obs(observation.id))
#io.copy(observation.effbtime, cluster_obj.effbtime_file_obs(observation.id))
#io.copy(observation.scale_map_region_list, cluster_obj.scalemap_regionlist_file_obs(observation.id))
io.copy(observation.acis_mask, observation.acis_mask_sc)
exposure = ciao.get_exposure(observation.clean)
io.write_contents_to_file(exposure, observation.exposure_time_file, binary=False)
def make_pcad_lis(cluster, obsid):
analysis_dir = cluster.obs_analysis_directory(obsid)
search_str = "{}/*asol1.fits".format(analysis_dir)
pcad_files = io.get_filename_matching(search_str)
pcad_list_string = "\n".join(pcad_files)
pcad_filename = "{}/pcad_asol1.lis".format(analysis_dir)
io.write_contents_to_file(pcad_list_string, pcad_filename, binary=False)
return pcad_filename
def run_in_parallel(cluster_obj: cluster.ClusterObj, region_list, start_time):
regions_processed = []
num_regions = len(region_list)
time_of_last = 0
for region in region_list:
pix2pix.pix_to_pix(cluster_obj, region, mp.current_process().name)
regions_processed.append(region)
if len(regions_processed) % 10 == 0:
current_time = time.time()
time_since_last = current_time - time_of_last
time_of_last = time.time()
average_second_per_completion = time_since_last / 10
elapsed = current_time - start_time
elapsed_str = time.strftime("%H Hours %M minutes %S seconds.",
time.gmtime(elapsed))
num_regions_processed = len(regions_processed)
seconds_per_region = elapsed / num_regions_processed
name = mp.current_process().name
completion_seconds = (
(num_regions - num_regions_processed) * seconds_per_region
) # in seconds
completion_time = time.strftime("%H Hours %M minutes %S seconds.",
time.gmtime(completion_seconds))
log_string = "\n{color}*******************************************{reset}\n"\
"\n{name}: Elapsed time: {elapsed}\n"\
"{name}: Seconds per region (total average): {seconds_per_region}\n"\
"Seconds per region (last 10 regions fit): {recent_average}\n"\
"{num_processed} of {total_regions} processed\n" \
"Estimated time for completion: {completion}.\n" \
"\n{color}*******************************************{reset}\n".format(
color=io.Colors.RED,
reset=io.Colors.RESET,
name=name,
elapsed=elapsed_str,
seconds_per_region=seconds_per_region,
recent_average=average_second_per_completion,
num_processed=num_regions_processed,
total_regions=num_regions,
completion=completion_time
)
print(log_string)
log_file = "{super_comp_dir}/{p_name}_time.txt".format(
super_comp_dir=cluster_obj.super_comp_dir,
p_name=mp.current_process().name)
io.write_contents_to_file(log_string, log_file, False)
print("Manual garbage collection.")
gc.collect()
mp.Queue().put('Regions processed: {}'.format(len(regions_processed)))
def make_response_files(cluster):
for obsid in cluster.observation_ids:
print("Making response files for observation {}".format(obsid))
obs_analysis_dir = cluster.obs_analysis_directory(obsid)
region_file = io.get_path("{}/acisI_region_0.reg".format(obs_analysis_dir))
if (not io.file_exists(region_file)) or (io.file_size(region_file) == 0):
print("Region file {} does not exist.".format(region_file))
print("When DS9 opens, draw a small circle that covers a piece of each ACIS-I chip (~20 pixels) and save it as:\n" \
"{}".format(region_file))
print("Opening SAO DS9")
io.write_contents_to_file("", region_file, False)
ds9_arguments = "ds9 -regions system physical -regions shape circle -regions format ciao -zoom 0.5 " \
"-bin factor 4 {}/acisI_clean.fits".format(obs_analysis_dir)
subprocess.run([ds9_arguments], shell=True)
print('Creating global response file.')
create_global_response_file_for(cluster, obsid, region_file)
def actually_merge_observations_from(cluster):
print("Merging observations from {}.".format(cluster.name))
merged_directory = io.get_path('{}/merged_obs_evt2/'.format(
cluster.directory))
io.make_directory(merged_directory)
os.chdir(merged_directory)
merged_observations = []
for observation in cluster.observation_list:
evt2_file = "{}/{}/analysis/evt2.fits".format(cluster.directory,
observation)
merged_observations.append(evt2_file)
merged_lis = "{}/merged_obs.lis".format(merged_directory)
io.write_contents_to_file("\n".join(merged_observations),
merged_lis,
binary=False)
outroot = io.get_path("{}/{}".format(cluster.directory, cluster.name))
infile = "@{infile}[ccd_id=0:3]".format(infile=merged_lis) # for ACIS-I
# infile = "@{infile}".format(infile=merged_lis) # for ACIS-I & ACIS-S
xygrid = "1500:6500:4,1500:6500:4"
if len(merged_observations) == 1:
rt.fluximage.punlearn()
rt.fluximage(infile=infile,
outroot=outroot,
xygrid=xygrid,
clobber=True)
print("Only single observation, flux image created.")
elif len(merged_observations) > 1:
rt.merge_obs.punlearn()
rt.merge_obs(infiles=infile,
outroot=outroot,
xygrid=xygrid,
clobber=True,
parallel=True,
nproc=12)
def ciao_back(cluster, overwrite=False):
print("Running ciao_back on {}.".format(cluster.name))
for observation in cluster.observations:
pcad_file = make_pcad_lis(cluster, observation.id)
backI_lis = []
backS_lis = []
analysis_path = observation.analysis_directory
filelist = io.read_contents_of_file(observation.ccd_merge_list).split('\n')
pcad = io.read_contents_of_file(pcad_file)
for acis_file in filelist:
rt.acis_bkgrnd_lookup.punlearn()
print("Finding background for {}".format(acis_file))
path_to_background = rt.acis_bkgrnd_lookup(infile=acis_file)
print("Found background at {}".format(path_to_background))
acis_id = int(acis_file.split('/')[-1].split('.')[-2][-1])
assert isinstance(acis_id, int), "acis_id = {}".format(acis_id)
assert not isinstance(path_to_background, type(None)), "Cannot find background {}".format(acis_file)
local_background_path = io.get_path("{}/back_ccd{}.fits".format(analysis_path, acis_id))
try:
if io.file_exists(local_background_path) and overwrite:
io.delete(local_background_path)
io.copy(path_to_background, local_background_path)
except OSError:
print("Problem copying background file {}. Do you have the right permissions and a full CALDB?".format(
path_to_background))
raise
acis_gain = rt.dmkeypar(infile=acis_file,
keyword="GAINFILE",
echo=True)
background_gain = rt.dmkeypar(infile=local_background_path,
keyword="GAINFILE",
echo=True)
print("{}/{}/acis_ccd{}.fits gain: {}".format(cluster.name, observation.id, acis_id, acis_gain))
print("{}/{}/back_ccd{}.fits gain: {}".format(cluster.name, observation.id, acis_id, background_gain))
if dates_and_versions_match(acis_gain, background_gain):
print("Date/version numbers don't match on the acis data and background. Reprocessing.")
local_background_path = reprocess(cluster, observation.id, acis_gain, background_gain, acis_id)
print("Reprojecting background")
rt.reproject_events.punlearn()
infile = local_background_path
outfile = io.get_path("{local_path}/back_reproj_ccd{acis_id}.fits".format(local_path=analysis_path,
acis_id=acis_id))
match = acis_file
print(
"Running:\n reproject_events(infile={infile}, outfile={outfile}, aspect={pcad}, match={match})".format(
infile=infile, outfile=outfile, pcad=pcad, match=match)
)
rt.reproject_events(infile=infile,
outfile=outfile,
aspect="{pcad_file}".format(pcad_file=pcad),
match=match,
random=0,
clobber=True)
back_reproject = outfile
datamode = rt.dmkeypar(infile=io.get_filename_matching(io.get_path("{}/acis*evt1*.fits".format(analysis_path))),
keyword="DATAMODE")
if datamode == "VFAINT":
print("VFAINT Mode, resetting setting status bits")
rt.dmcopy.punlearn()
rt.dmcopy(infile="{}[status=0]".format(back_reproject),
outfile=outfile,
clobber=True)
if acis_id <= 3:
backI_lis.append(back_reproject)
else:
backS_lis.append(back_reproject)
merged_back_list = backI_lis + backS_lis
print("writing backI.lis and backS.lis")
io.write_contents_to_file("\n".join(backI_lis), io.get_path("{}/backI.lis".format(analysis_path)),
binary=False)
io.write_contents_to_file("\n".join(backS_lis), io.get_path("{}/backS.lis".format(analysis_path)),
binary=False)
io.write_contents_to_file("\n".join(merged_back_list), observation.merged_back_lis, binary=False)
return
def make_commands_lis(cluster: cluster.ClusterObj, resolution):
from astropy.io import fits
print("Creating {}".format(cluster.command_lis))
offset = [None, 5, 3, 1][resolution]
start_time = time.time()
scalemap = cluster.scale_map
# mask = fits.open(cluster.combined_mask)[0].data
region_list = cluster.scale_map_regions_to_fit(resolution)
#sz = scalemap.shape
#sx = sz[0]
#sy = sz[1]
command_string = []
pypeline_dir = io.get_user_input("Enter the directory containing the pix2pix.py portion of the pypeline on the remote machine: ")
pix2pix_path = "{pypeline_dir}/pix2pix.py".format(pypeline_dir=pypeline_dir)
data_dir = io.get_user_input("Enter the directory containing the cluster data on the remote machine:\n"
"For example: /home/user/data/clustername/\n")
for region in region_list:
new_command = "python {pix2pix} {cluster_config} {region}".format(
pix2pix=pix2pix_path,
cluster_config="{data_dir}/{name}_pypeline_config.ini".format(
data_dir=data_dir,
name=cluster.name
),
region=region
)
command_string.append(new_command)
# for x in range(sx):
# for y in range(sy):
# if mask[i,j] == 1:
# image_radius = scalemap[i,j]
# if 0 < image_radius <= 100:
# if i % offset == 0 and j % offset == 0:
# new_command = "python {pix2pix} {cluster_config} {region}".format(
# pix2pix=pix2pix_path,
# cluster_config="{data_dir}/{name}_pypeline_config.ini".format(
# data_dir=data_dir,
# name=cluster.name
# ),
# region=position
# )
# command_string.append(new_command)
# region_list.append("{}".format(position))
# #end if
# # end if
# position += 1
command_lis = "\n".join(command_string)
region_string = '\n'.join([str(x) for x in region_list])
io.write_contents_to_file(command_lis, cluster.command_lis, binary=False)
io.write_contents_to_file(region_string, cluster.filtered_region_list, binary=False)
end_time = time.time()
print("Time elapsed: {time:0.2f} sec".format(time=(end_time-start_time)))
