def run_sourcelist_comparision(total_list, log_level=logutil.logging.INFO):
""" This subroutine automates execution of drizzlepac/devutils/comparison_tools/compare_sourcelist_flagging.py to
compare HAP-generated filter catalogs with their HLA classic counterparts.
NOTE: In order for this subroutine to run, the following environment variables need to be set:
- HLA_CLASSIC_BASEPATH
- HLA_BUILD_VER
Alternatively, if the HLA classic path is unavailable, The comparison can be run using locally stored HLA classic
files. The relevant HLA classic imagery and sourcelist files must be placed in a subdirectory of the current working
directory called 'hla_classic'.
Parameters
----------
total_list: list
List of TotalProduct objects, one object per instrument/detector combination is
a visit. The TotalProduct objects are comprised of FilterProduct and ExposureProduct
objects.
log_level : int, optional
The desired level of verboseness in the log statements displayed on the screen and written to the .log file.
Default value is 20, or 'info'.
RETURNS
-------
Nothing.
"""
#get HLA classic path details from envroment variables
hla_classic_basepath = os.getenv('HLA_CLASSIC_BASEPATH')
hla_build_ver = os.getenv("HLA_BUILD_VER")
for tot_obj in total_list:
if hla_classic_basepath and hla_build_ver and os.path.exists(
hla_classic_basepath):
hla_cassic_basepath = os.path.join(hla_classic_basepath,
tot_obj.instrument,
hla_build_ver)
hla_classic_path = os.path.join(
hla_cassic_basepath, tot_obj.prop_id, tot_obj.prop_id + "_" +
tot_obj.obset_id) # Generate path to HLA classic products
elif os.path.exists(os.path.join(os.getcwd(),
"hla_classic")): # For local testing
hla_classic_basepath = os.path.join(os.getcwd(), "hla_classic")
hla_classic_path = hla_classic_basepath
else:
return # bail out if HLA classic path can't be found.
for filt_obj in tot_obj.fdp_list:
hap_imgname = filt_obj.drizzle_filename
hla_imgname = glob.glob("{}/{}{}_dr*.fits".format(
hla_classic_path, filt_obj.basename, filt_obj.filters))[0]
if not os.path.exists(hap_imgname) or not os.path.exists(
hla_imgname
): # Skip filter if one or both of the images can't be found
continue
for hap_sourcelist_name in [
filt_obj.point_cat_filename, filt_obj.segment_cat_filename
]:
if hap_sourcelist_name.endswith("point-cat.ecsv"):
hla_classic_cat_type = "dao"
plotfile_prefix = filt_obj.product_basename + "_point"
else:
hla_classic_cat_type = "sex"
plotfile_prefix = filt_obj.product_basename + "_segment"
if hla_classic_basepath and hla_build_ver and os.path.exists(
hla_classic_basepath):
hla_sourcelist_name = "{}/logs/{}{}_{}phot.txt".format(
hla_classic_path, filt_obj.basename, filt_obj.filters,
hla_classic_cat_type)
else:
hla_sourcelist_name = "{}/{}{}_{}phot.txt".format(
hla_classic_path, filt_obj.basename, filt_obj.filters,
hla_classic_cat_type)
if not os.path.exists(
hap_sourcelist_name
) or not os.path.exists(
hla_sourcelist_name
): # Skip catalog type if one or both of the catalogs can't be found
continue
log.info("HAP image: {}".format(
os.path.basename(hap_imgname)))
log.info("HLA Classic image: {}".format(
os.path.basename(hla_imgname)))
log.info("HAP catalog: {}".format(
os.path.basename(hap_sourcelist_name)))
log.info("HLA Classic catalog: {}".format(
os.path.basename(hla_sourcelist_name)))
# once all file exist checks are passed, execute sourcelist comparision
return_status = compare_sourcelists.comparesourcelists(
[hla_sourcelist_name, hap_sourcelist_name],
[hla_imgname, hap_imgname],
plotGen="file",
diffMode="absolute",
plotfile_prefix=plotfile_prefix,
verbose=True,
log_level=log_level,
debugMode=False)
def run_compare_sourcelists(hff_inputs, log_level):
"""locate HLA classic image and sourcelists, convert HLA classic sorucelist X, Y, RA and Dec to HAP ref frame for
apples-to-apples comparision, and run comparision code
Parameters
----------
hff_inputs : dict
dictionary containing all necessary inputs to run hla_flag_filter.run_source_list_flagging().
log_level : int
desired log level
Returns
-------
Nothing.
"""
# get HLA classic path details from environment variables
hla_classic_basepath = os.getenv('HLA_CLASSIC_BASEPATH')
hla_build_ver = os.getenv("HLA_BUILD_VER")
parse_imgname = hff_inputs['drizzled_image'].split("_")
instrument = parse_imgname[3]
prop_id = parse_imgname[1]
obset_id = parse_imgname[2]
filter = parse_imgname[5]
hla_product_basename = "_".join(parse_imgname[0:6])
hap_product_basename = "_".join(parse_imgname[0:7])
if hla_classic_basepath and hla_build_ver and os.path.exists(
hla_classic_basepath):
hla_cassic_basepath = os.path.join(hla_classic_basepath, instrument,
hla_build_ver)
hla_classic_path = os.path.join(
hla_cassic_basepath, prop_id,
prop_id + "_" + obset_id) # Generate path to HLA classic products
elif os.path.exists(os.path.join(os.getcwd(),
"hla_classic")): # For local testing
hla_classic_basepath = os.path.join(os.getcwd(), "hla_classic")
hla_classic_path = hla_classic_basepath
else:
return # bail out if HLA classic path can't be found.
hap_imgname = hff_inputs['drizzled_image']
hla_imgname = glob.glob("{}/{}_dr*.fits".format(hla_classic_path,
hla_product_basename))[0]
if not os.path.exists(hap_imgname) or not os.path.exists(
hla_imgname): # bail if one or both of the images can't be found
sys.exit("Error: one or both of the images can't be found")
hap_sourcelist_name = hff_inputs['catalog_name']
if hap_sourcelist_name.endswith("point-cat.ecsv"):
hla_classic_cat_type = "dao"
plotfile_prefix = hap_product_basename + "_point"
else:
hla_classic_cat_type = "sex"
plotfile_prefix = hap_product_basename + "_segment"
if hla_classic_basepath and hla_build_ver and os.path.exists(
hla_classic_basepath):
hla_sourcelist_name = "{}/logs/{}_{}phot.txt".format(
hla_classic_path, hla_product_basename, hla_classic_cat_type)
else:
hla_sourcelist_name = "{}/{}_{}phot.txt".format(
hla_classic_path, hla_product_basename, hla_classic_cat_type)
plotfile_prefix += "_testing"
if not os.path.exists(hap_sourcelist_name) or not os.path.exists(
hla_sourcelist_name
): # Skip catalog type if one or both of the catalogs can't be found
sys.exit() # TODO: Circle back to this
# convert HLA Classic RA and Dec values to HAP reference frame so the RA and Dec comparisons are correct
updated_hla_sourcelist_name = hapsequencer.correct_hla_classic_ra_dec(
hla_sourcelist_name, hap_imgname, hla_classic_cat_type, log_level)
log.info("HAP image: {}".format(
os.path.basename(hap_imgname)))
log.info("HLA Classic image: {}".format(
os.path.basename(hla_imgname)))
log.info("HAP catalog: {}".format(
os.path.basename(hap_sourcelist_name)))
log.info("HLA Classic catalog: {}".format(
os.path.basename(updated_hla_sourcelist_name)))
# once all file exist checks are passed, execute sourcelist comparision
return_status = compare_sourcelists.comparesourcelists(
[updated_hla_sourcelist_name, hap_sourcelist_name],
[hla_imgname, hap_imgname],
good_flag_sum=255,
plotGen="file",
plotfile_prefix=plotfile_prefix,
verbose=True,
log_level=log_level,
debugMode=True)
mod_sl_name = mod_sl_name.replace(".txt","_corrected.txt")
print("Corrected version of HLA Classic file {} with new X, Y and RA, Dec values written to {}.".format(orig_hla_classic_sl_name,mod_sl_name))
cat.write(mod_sl_name,format="ascii.csv")
return mod_sl_name
if __name__ == "__main__":
hla_sourcelist_name = sys.argv[1]
hla_imgname = sys.argv[2]
hap_sourcelist_name = sys.argv[3]
hap_imgname = sys.argv[4]
# hla_sourcelist_name = "hla_classic/hst_11665_06_wfc3_uvis_f555w_daophot.txt"
# hla_imgname = "hla_classic/hst_11665_06_wfc3_uvis_f555w_drz.fits"
# hap_sourcelist_name = "hst_11665_06_wfc3_uvis_f555w_ib4606_point-cat.ecsv"
# hap_imgname = "hst_11665_06_wfc3_uvis_f555w_ib4606_drc.fits"
# hla_sourcelist_name = "hla_classic/hst_11671_03_wfc3_ir_f153m_daophot.txt"
# hla_imgname = "hla_classic/hst_11671_03_wfc3_ir_f153m_drz.fits"
# hap_sourcelist_name = "hst_11671_03_wfc3_ir_f153m_ib4103_point-cat.ecsv"
# hap_imgname = "hst_11671_03_wfc3_ir_f153m_ib4103_drz.fits"
log_level = 10
log.setLevel(log_level)
hla_cat_type = hla_sourcelist_name.replace("_corrected","")[-11:-8]
updated_hla_sourcelist_name = hapsequencer.correct_hla_classic_ra_dec(hla_sourcelist_name,hap_imgname,hla_cat_type,log_level)
return_status = compare_sourcelists.comparesourcelists([updated_hla_sourcelist_name,hap_sourcelist_name], [hla_imgname, hap_imgname],
plotfile_prefix=hap_sourcelist_name.replace("-cat.ecsv",""), plotGen="file", verbose=True, debugMode = True,log_level=log_level)