def main(): parser = argparse.ArgumentParser() parser.add_argument('--astrom-filename', default=None, help="Give the astrom file directly instead of looking-up " "using the field/ccd naming scheme.") parser.add_argument('--reals', action='store_true', default=False) parser.add_argument('--type', choices=['o', 'p', 's'], help="Which type of image.", default='s') parser.add_argument('--measure3', default='vos:OSSOS/measure3/2013B-L_redo/') parser.add_argument('--dbimages', default=None) parser.add_argument('--dry-run', action='store_true', default=False) parser.add_argument('--force', action='store_true', default=False) args = parser.parse_args() logging.basicConfig(level=logging.INFO) prefix = 'fk' ext = args.reals and 'reals' or 'cands' storage.MEASURE3 = args.measure3 if args.dbimages is not None: storage.DBIMAGES = args.dbimages astrom.DATASET_ROOT = args.dbimages astrom_uri = storage.get_cands_uri(args.field, ccd=args.ccd, version=args.type, prefix=prefix, ext="measure3.{}.astrom".format(ext)) if args.astrom_filename is None: astrom_filename = os.path.basename(astrom_uri) else: astrom_filename = args.astrom_filename if not os.access(astrom_filename, os.F_OK): astrom_filename = os.path.dirname(astrom_uri) + "/" + astrom_filename # Load the list of astrometric observations that will be looked at. fk_candidate_observations = astrom.parse(astrom_filename)
def main(): parser = argparse.ArgumentParser() parser.add_argument('field') parser.add_argument('ccd') parser.add_argument( '--expnum', default=None, help="Which exposure is the lead for this astrom file?") parser.add_argument( '--astrom-filename', default=None, help="Give the astrom file directly instead of looking-up " "using the field/ccd naming scheme.") parser.add_argument('--reals', action='store_true', default=False) parser.add_argument('--type', choices=['o', 'p', 's'], help="Which type of image.", default='s') parser.add_argument('--measure3', default='vos:OSSOS/measure3/2013B-L_redo/') parser.add_argument('--dbimages', default=None) parser.add_argument('--dry-run', action='store_true', default=False) parser.add_argument('--force', action='store_true', default=False) parser.add_argument("--fk", action="store_true", default=False, help="Do fakes?") parser.add_argument('--object-planted', default=OBJECT_PLANTED, help="Name of file contains list of planted objects.") parser.add_argument( '--bright-limit', default=BRIGHT_LIMIT, type=float, help= "Sources brighter than this limit {} are used to diagnose planting issues." .format(BRIGHT_LIMIT)) parser.add_argument( '--minimum-bright-detections', default=MINIMUM_BRIGHT_DETECTIONS, type=int, help= "required number of detections with mag brighter than bright-limit.") parser.add_argument( '--minimum-bright-fraction', default=MINIMUM_BRIGHT_FRACTION, type=float, help= "minimum fraction of objects above bright limit that should be found.") parser.add_argument("--debug", action="store_true") parser.add_argument("--verbose", "-v", action="store_true") cmd_line = " ".join(sys.argv) args = parser.parse_args() util.set_logger(args) logging.info("Starting {}".format(cmd_line)) prefix = (args.fk and "fk") or "" version = args.type ext = 'cands' if args.reals: ext = 'reals' if args.dbimages is not None: storage.DBIMAGES = args.dbimages astrom.DATASET_ROOT = args.dbimages storage.MEASURE3 = args.measure3 astrom_uri = storage.get_cands_uri(args.field, ccd=args.ccd, version=args.type, prefix=prefix, ext="measure3.{}.astrom".format(ext)) if args.astrom_filename is None: astrom_filename = os.path.basename(astrom_uri) else: astrom_filename = args.astrom_filename if not os.access(astrom_filename, os.F_OK): astrom_filename = os.path.dirname(astrom_uri) + "/" + astrom_filename # Load the list of astrometric observations that will be looked at. fk_candidate_observations = astrom.parse(astrom_filename) if args.expnum is None: expnum = fk_candidate_observations.observations[0].expnum else: expnum = args.expnum message = storage.SUCCESS if storage.get_status(task, prefix, expnum, version='', ccd=args.ccd) and not args.force: logging.info("{} completed successfully for {} {} {} {}".format( task, prefix, expnum, '', args.ccd)) return with storage.LoggingManager(task, prefix, expnum, args.ccd, version, args.dry_run): try: match_filename = os.path.splitext( os.path.basename(astrom_filename))[0] + '.match' logging.info(f'Got back:{match_filename}') match_uri = storage.get_cands_uri( args.field, ccd=args.ccd, version=args.type, prefix=prefix, ext="measure3.{}.match".format(ext), block=args.field) try: storage.copy(match_uri, match_filename) except NotFoundException as ex: logging.warning( f'No match file found at {match_uri}, creating one.') logging.debug(f'{ex}') pass logging.info( ("Comparing planted and measured magnitudes " "for sources in {} and {}\n".format(args.object_planted, astrom_filename))) result = match_planted( fk_candidate_observations, match_filename=match_filename, object_planted=args.object_planted, bright_limit=args.bright_limit, minimum_bright_detections=args.minimum_bright_detections, bright_fraction=args.minimum_bright_fraction) if not args.dry_run: storage.copy(match_filename, match_uri) uri = os.path.dirname(astrom_uri) keys = [storage.tag_uri(os.path.basename(astrom_uri))] values = [result] storage.set_tags_on_uri(uri, keys, values) logger.info(message) except Exception as err: import traceback traceback.print_exc() message = str(err) logging.error(message) if not args.dry_run: storage.set_status(task, prefix, expnum, version='', ccd=args.ccd, status=message) return
def main(): parser = argparse.ArgumentParser() parser.add_argument('field') parser.add_argument('ccd') parser.add_argument('--expnum', default=None, help="Which exposure is the lead for this astrom file?") parser.add_argument('--astrom-filename', default=None, help="Give the astrom file directly instead of looking-up " "using the field/ccd naming scheme.") parser.add_argument('--reals', action='store_true', default=False) parser.add_argument('--type', choices=['o', 'p', 's'], help="Which type of image.", default='s') parser.add_argument('--measure3', default='vos:OSSOS/measure3/2013B-L_redo/') parser.add_argument('--dbimages', default=None) parser.add_argument('--dry-run', action='store_true', default=False) parser.add_argument('--force', action='store_true', default=False) parser.add_argument('--object-planted', default=OBJECT_PLANTED, help="Name of file contains list of planted objects.") parser.add_argument('--bright-limit', default=BRIGHT_LIMIT, help="Sources brighter than this limit {} are used to diagnose planting issues.".format( BRIGHT_LIMIT)) parser.add_argument('--minimum-bright-detections', default=MINIMUM_BRIGHT_DETECTIONS, help="required number of detections with mag brighter than bright-limit.") parser.add_argument('--minimum-bright-fraction', default=MINIMUM_BRIGHT_FRACTION, help="minimum fraction of objects above bright limit that should be found.") args = parser.parse_args() logging.basicConfig(level=logging.INFO) prefix = 'fk' ext = args.reals and 'reals' or 'cands' storage.MEASURE3 = args.measure3 if args.dbimages is not None: storage.DBIMAGES = args.dbimages astrom.DATASET_ROOT = args.dbimages astrom_uri = storage.get_cands_uri(args.field, ccd=args.ccd, version=args.type, prefix=prefix, ext="measure3.{}.astrom".format(ext)) if args.astrom_filename is None: astrom_filename = os.path.basename(astrom_uri) else: astrom_filename = args.astrom_filename if not os.access(astrom_filename, os.F_OK): astrom_filename = os.path.dirname(astrom_uri) + "/" + astrom_filename # Load the list of astrometric observations that will be looked at. fk_candidate_observations = astrom.parse(astrom_filename) if args.expnum is None: expnum = fk_candidate_observations.observations[0].expnum else: expnum = args.expnum storage.set_logger(os.path.splitext(os.path.basename(sys.argv[0]))[0], prefix, expnum, "", ext, args.dry_run) match_filename = os.path.splitext(os.path.basename(astrom_filename))[0] + '.match' exit_status = 0 status = storage.SUCCESS try: if (not storage.get_status(expnum, ccd=args.ccd, program='astrom_mag_check', version='')) or args.force: message = match_planted(fk_candidate_observations, match_filename=match_filename, object_planted=args.object_planted, bright_limit=args.bright_limit, minimum_bright_detections=args.minimum_bright_detections, bright_fraction=args.minimum_bright_fraction) match_uri = storage.get_cands_uri(args.field, ccd=args.ccd, version=args.type, prefix=prefix, ext="measure3.{}.match".format(ext)) if not args.dry_run: storage.copy(match_filename, match_uri) uri = os.path.dirname(astrom_uri) keys = [storage.tag_uri(os.path.basename(astrom_uri))] values = [message] storage.set_tags_on_uri(uri, keys, values) except Exception as err: sys.stderr.write(str(err)) status = str(err) exit_status = err.message if not args.dry_run: storage.set_status(expnum, args.ccd, 'astrom_mag_check', version='', status=status) return exit_status
def match_planted(cand_filename, measures): """ """ # Load the planted objects associated with this candidate file. try: cands = astrom.parse(cand_filename) except: sys.stderr("Failed while reading {}".format(cand_filename)) return matches_fptr = open(os.path.basename(cand_filename)+".eff", 'w') objects_planted_uri = cands.observations[0].get_object_planted_uri() planted_object_file = Planted_object_file(objects_planted_uri) planted_objects = planted_object_file.planted_objects matched = {} false_positive_sources = [] false_negative_sources = [] confused_measure = {} for idx in range(len(planted_objects)): planted_object = planted_objects[idx] # look for a matching .cand entry cand_dist = None cand_source = None for source in cands.get_sources(): obs = source.get_readings() dist = math.sqrt((obs[0].x-planted_object.x)**2 + (obs[0].y - planted_object.y)**2) if cand_dist is None or cand_dist > dist: cand_dist = dist cand_source = source # look for a matching .mpc entry measure_dist = None measure_source = None for provisional in measures: x = float(measures[provisional][0].comment.X) y = float(measures[provisional][0].comment.Y) dist = math.sqrt( (x - planted_object.x)**2 + (y-planted_object.y)**2) if measure_dist is None or measure_dist > dist: measure_dist = dist measure_source = measures[provisional] if measure_dist < 6.0: # this gets 'unset' if we match this measure with a cand, in the next step. confused_measure[provisional] = planted_object planted_object.confused += 1 # determine if planted_object was found if cand_dist < 6.0: # In candidate list if measure_dist is not None and measure_dist < 6.0: # accepted. planted_object.recovered = measure_source planted_object.false_negative = None planted_object.confused -= 1 del(confused_measure[measure_source[0].provisional_name]) matched[measure_source[0].provisional_name] = True else: # rejected. planted_object.false_negative = cand_source false_negative_sources.append(cand_source) matches_fptr.write( ("## F: found\n" "## M: multiple matches\n" "## C: other match of confused multiple match\n" "## N: not found\n" "## P: false Positive \n") ) matches_fptr.write("{} {} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s}\n".format( "Key",planted_object_file.header,"x_dao","y_dao", "rate_mes", "ang_mes", "mag1_dao","merr1_dao", "mag2_dao","merr2_dao", "mag3_dao","merr3_dao" )) for planted_object in planted_objects: if planted_object.recovered is not None: confused = "F" if planted_object.confused > 1 : confused = "M" measure = planted_object.recovered start_jd = measure[0].date.jd x = float(measure[0].comment.X) x3 = float(measure[2].comment.X) y = float(measure[0].comment.Y) y3 = float(measure[2].comment.Y) end_jd = measure[2].date.jd rate = math.sqrt((x3-x)**2 + (y3-y)**2)/( 24*(end_jd - start_jd) ) angle = math.degrees(math.atan2(y3 - y,x3 - x)) matches_fptr.write("{:3s} {} {:8.2f} {:8.2f} {:8.2f} {:8.2f} ".format( confused, str(planted_object), x, y, rate, angle)) # record the three discovery magnitudes for idx in range(3): try: mag = float(measure[idx].comment.mag) merr = float(measure[idx].comment.mag_uncertainty) except Exception as e: mag = -1.0 merr = -1.0 logger.warning(str(e)) matches_fptr.write("{:8.2f} {:8.2f} ".format(mag,merr)) matches_fptr.write("\n") elif planted_object.false_negative is not None: source = planted_object.false_negative reading = source.get_reading(0) third = source.get_reading(2) cutout = image_slice_downloader.download_cutout(reading, needs_apcor=True) start_jd = mpc.Time(reading.obs.header['MJD_OBS_CENTER'],format='mpc', scale='utc').jd end_jd = mpc.Time(third.obs.header['MJD_OBS_CENTER'], format='mpc', scale='utc').jd rate = math.sqrt((third.x - reading.x)**2 + (third.y - reading.y)**2)/( 24*(end_jd - start_jd) ) angle = math.degrees(math.atan2(third.y - reading.y,third.x - reading.x)) matches_fptr.write("{:3s} {} {:8.2f} {:8.2f} {:8.2f} {:8.2f} ".format( "N", str(planted_object), reading.x, reading.y, rate, angle)) for idx in range(3): try: (x, y, mag, merr) = cutout.get_observed_magnitude() except TaskError as e: logger.warning(str(e)) mag = -1.0 merr = -1.0 matches_fptr.write("{:8.2f} {:8.2f} ".format(mag,merr)) matches_fptr.write("\n") else: matches_fptr.write("{:3s} {}".format("X",str(planted_object))) matches_fptr.write(10*" {:8.2f}".format(0.0)) matches_fptr.write("\n") matches_fptr.flush() for provisional in measures: if matched.get(provisional,False): continue # this source is a false positive measure = measures[provisional] start_jd = measure[0].date.jd x = float(measure[0].comment.X) x3 = float(measure[2].comment.X) y = float(measure[0].comment.Y) y3 = float(measure[2].comment.Y) end_jd = measure[2].date.jd if provisional in confused_measure: confused = "C" planted_object = confused_measure[provisional] else: confused = "P" planted_object = " {:4d}".format(-1)+6*" {:8.2f}".format(0) # look for the matching cand entry cand_dist = None cand_source = None for source in cands.get_sources(): obs = source.get_readings() dist = math.sqrt((obs[0].x - x)**2 + (obs[0].y - y)**2) if cand_dist is None or cand_dist > dist: cand_dist = dist cand_source = source false_positive_sources.append(cand_source) rate = math.sqrt((x3-x)**2 + (y3-y)**2)/( 24*(end_jd - start_jd)) angle = math.degrees(math.atan2(y3 - y,x3 - x)) # record the three discovery magnitudes matches_fptr.write("{:3s} {} {:8.2f} {:8.2f} {:8.2f} {:8.2f} ".format( confused, planted_object, x, y, rate, angle)) for idx in range(3): try: mag = float(measure[idx].comment.mag) merr = float(measure[idx].comment.mag_uncertainty) except Exception as e: mag = -1.0 merr = -1.0 logger.warning(str(e)) matches_fptr.write("{:8.2f} {:8.2f} ".format(mag,merr)) matches_fptr.write("\n") matches_fptr.close() ## write out the false_positives and false_negatives if not os.access('false_positives',os.R_OK): os.mkdir('false_positives') if not os.access('false_negatives', os.R_OK): os.mkdir('false_negatives') if len(false_positive_sources) > 0 : wh = open('false_positives/'+os.path.basename(cand_filename),'w+') writer = astrom.StreamingAstromWriter(wh,cands.sys_header) #writer.write_headers(cands.observations) for source in false_positive_sources: writer.write_source(source) writer.flush() writer.close() if len(false_negative_sources) > 0 : wh = open('false_negatives/'+os.path.basename(cand_filename),'w+') writer = astrom.StreamingAstromWriter(wh,cands.sys_header) #writer.write_headers(cands.observations) for source in false_negative_sources: writer.write_source(source) writer.flush() writer.close() return matches_fptr.name
def match_planted(astrom_filename, match_filename, false_positive_filename): """ Using the astrom_filename as input get the Object.planted file from VOSpace and match planted sources with found sources. The Object.planted list is pulled from VOSpace based on the standard file-layout and name of the first exposure as read from the .astrom file. :param astrom_filename: name of the fk*reals.astrom file to check against Object.planted :param match_filename: a file that will contain a list of all planted sources and the matched found source :param false_positive_filename: .astrom format output containing input objects that had no match in planted """ image_slice_downloader = ImageCutoutDownloader(slice_rows=100, slice_cols=100) fk_candidate_observations = astrom.parse(astrom_filename) matches_fptr = storage.open_vos_or_local(match_filename,'w') objects_planted_uri = fk_candidate_observations.observations[0].get_object_planted_uri() objects_planted = image_slice_downloader.download_raw(objects_planted_uri, view='data').split('\n') planted_objects = [] for line in objects_planted[1:]: if len(line) == 0 or line[0] == '#': continue planted_objects.append(PlantedObject(line)) false_positives_stream_writer = None matches_fptr.write("#{}\n".format(fk_candidate_observations.observations[0].rawname)) matches_fptr.write("{:1s}{} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s}\n".format( "",objects_planted[0],"x_dao","y_dao","mag_dao","merr_dao", "rate_mes", "ang_mes", "dr_pixels" )) found_idxs = [] for source in fk_candidate_observations.get_sources(): reading = source.get_reading(0) third = source.get_reading(2) cutout = image_slice_downloader.download_cutout(reading, needs_apcor=True) try: (x, y, mag, merr) = cutout.get_observed_magnitude() except TaskError as e: logger.warning(str(e)) mag = 0.0 merr = -1.0 matched = None for idx in range(len(planted_objects)): planted_object = planted_objects[idx] dist = math.sqrt((reading.x-planted_object.x)**2 + (reading.y - planted_object.y)**2) if matched is None or dist < matched: matched = dist matched_object_idx = idx start_jd = Time(reading.obs.header['MJD_OBS_CENTER'],format='mpc', scale='utc').jd end_jd = Time(third.obs.header['MJD_OBS_CENTER'], format='mpc', scale='utc').jd exptime = float(reading.obs.header['EXPTIME']) rate = math.sqrt((third.x - reading.x)**2 + (third.y - reading.y)**2)/( 24*(end_jd - start_jd) ) angle = math.degrees(math.atan2(third.y - reading.y,third.x - reading.x)) if matched > 3*rate*exptime/3600.0 and False : # this is a false positive (candidate not near artificial source) # create a .astrom style line for feeding to validate for checking later if false_positives_ftpr is None or false_positives_stream_writer is None: # create false positive file for storing results false_positives_ftpr = open(false_positive_filename,'w+') false_positives_stream_writer = StreamingAstromWriter( false_positives_ftpr,fk_candidate_observations.sys_header) false_positives_stream_writer.write_source(source) false_positives_ftpr.flush() continue elif matched_object_idx in found_idxs: repeat = '#' else: repeat = ' ' found_idxs.append(matched_object_idx) mags = [] merrs = [] for this_reading in source.get_readings()[1:]: cutout = image_slice_downloader.download_cutout(this_reading, needs_apcor=True) try: (this_x, this_y, this_mag, this_merr) = cutout.get_observed_magnitude() except TaskError as e: logger.warning(str(e)) this_mag = 0.0 this_merr = -1.0 mags.append(this_mag) merrs.append(this_merr) matches_fptr.write("{:1s}{} {:8.2f} {:8.2f} {:8.2f} {:8.2f} {:8.2f} {:8.2f} {:8.2f} ".format( repeat, str(planted_objects[matched_object_idx]), reading.x, reading.y, mag, merr, rate, angle, matched)) for idx in range(len(mags)): matches_fptr.write("{:8.2f} {:8.2f}".format(mags[idx], merrs[idx])) matches_fptr.write("\n") # close the false_positives if false_positives_ftpr is not None: false_positives_ftpr.close() # record the unmatched Object.planted entries, for use in efficiency computing for idx in range(len(planted_objects)): if idx not in found_idxs: planted_object = planted_objects[idx] matches_fptr.write("{:1s}{} {:8.2f} {:8.2f} {:8.2f} {:8.2f} {:8.2f} {:8.2f} {:8.2f}\n".format("",str(planted_object), 0, 0, 0, 0, 0, 0, 0)) matches_fptr.close()
def main(): parser = argparse.ArgumentParser() parser.add_argument('field') parser.add_argument('ccd') parser.add_argument('--expnum', default=None, help="Which exposure is the lead for this astrom file?") parser.add_argument('--astrom-filename', default=None, help="Give the astrom file directly instead of looking-up " "using the field/ccd naming scheme.") parser.add_argument('--reals', action='store_true', default=False) parser.add_argument('--type', choices=['o', 'p', 's'], help="Which type of image.", default='s') parser.add_argument('--measure3', default='vos:OSSOS/measure3/2013B-L_redo/') parser.add_argument('--dbimages', default=None) parser.add_argument('--dry-run', action='store_true', default=False) parser.add_argument('--force', action='store_true', default=False) parser.add_argument('--object-planted', default=OBJECT_PLANTED, help="Name of file contains list of planted objects.") parser.add_argument('--bright-limit', default=BRIGHT_LIMIT, help="Sources brighter than this limit {} are used to diagnose planting issues.".format( BRIGHT_LIMIT)) parser.add_argument('--minimum-bright-detections', default=MINIMUM_BRIGHT_DETECTIONS, help="required number of detections with mag brighter than bright-limit.") parser.add_argument('--minimum-bright-fraction', default=MINIMUM_BRIGHT_FRACTION, help="minimum fraction of objects above bright limit that should be found.") args = parser.parse_args() logging.basicConfig(level=logging.CRITICAL) prefix = 'fk' ext = args.reals and 'reals' or 'cands' task = util.task() storage.MEASURE3 = args.measure3 if args.dbimages is not None: storage.DBIMAGES = args.dbimages astrom.DATASET_ROOT = args.dbimages astrom_uri = storage.get_cands_uri(args.field, ccd=args.ccd, version=args.type, prefix=prefix, ext="measure3.{}.astrom".format(ext)) if args.astrom_filename is None: astrom_filename = os.path.basename(astrom_uri) else: astrom_filename = args.astrom_filename if not os.access(astrom_filename, os.F_OK): astrom_filename = os.path.dirname(astrom_uri) + "/" + astrom_filename # Load the list of astrometric observations that will be looked at. fk_candidate_observations = astrom.parse(astrom_filename) if args.expnum is None: expnum = fk_candidate_observations.observations[0].expnum else: expnum = args.expnum storage.set_logger(os.path.splitext(os.path.basename(sys.argv[0]))[0], prefix, expnum, "", ext, args.dry_run) match_filename = os.path.splitext(os.path.basename(astrom_filename))[0] + '.match' exit_status = 0 status = storage.SUCCESS try: if (not storage.get_status(task, prefix, expnum=expnum, version='', ccd=args.ccd)) or args.force: logging.info(("Comparing planted and measured magnitudes " "for sources in {} and {}\n".format(args.object_planted, astrom_filename))) message = match_planted(fk_candidate_observations, match_filename=match_filename, object_planted=args.object_planted, bright_limit=args.bright_limit, minimum_bright_detections=args.minimum_bright_detections, bright_fraction=args.minimum_bright_fraction) match_uri = storage.get_cands_uri(args.field, ccd=args.ccd, version=args.type, prefix=prefix, ext="measure3.{}.match".format(ext), block=args.field) if not args.dry_run: storage.copy(match_filename, match_uri) uri = os.path.dirname(astrom_uri) keys = [storage.tag_uri(os.path.basename(astrom_uri))] values = [message] storage.set_tags_on_uri(uri, keys, values) except Exception as err: sys.stderr.write(str(err)) status = str(err) exit_status = err.message if not args.dry_run: storage.set_status(task, prefix, expnum, version='', ccd=args.ccd, status=status) return exit_status