def check_tags(my_expnum, ops_set, my_ccds, dry_run=True):
"""
check the tags for the given expnum/ccd set.
@param ops:
@param my_expnum:
@return:
"""
tags = storage.get_tags(my_expnum)
count = 0
outcount = 0
fails = []
for ccd in my_ccds:
success = True
count += 1
for ops in ops_set:
for fake in ops[0]:
for my_program in ops[1]:
for version in ops[2]:
#print my_expnum, fake, my_program, version, ccd
key = storage.get_process_tag(fake + my_program, ccd, version)
uri = storage.tag_uri(key)
if "Failed to get image" in tags.get(uri, "Failed to get image"):
#print tags.get(uri, None)
fails.append(ccd)
success = False
if success:
outcount += 1
sys.stderr.write("{} {} {:5.1f}%\n".format(outcount, count,100* float(outcount)/count))
#print fails
return set(fails)
def check_tags(my_expnum, ops_set, my_ccds, dry_run=True):
"""
check the tags for the given expnum/ccd set.
@param ops:
@param my_expnum:
@return:
"""
tags = storage.get_tags(my_expnum)
count = 0
success = 0
for ops in ops_set:
for fake in ops[0]:
for my_program in ops[1]:
for version in ops[2]:
for ccd in my_ccds:
count += 1
# print my_expnum, fake, my_program, version, ccd
key = storage.get_process_tag(fake + my_program, ccd,
version)
uri = storage.tag_uri(key)
if tags.get(uri, None) != storage.SUCCESS:
print "{} --> {}".format(uri, tags.get(uri, None))
else:
success += 1
return "{} {} {:5.1f}%".format(success, count,
100 * float(success) / count)
def check_tags(my_expnum, ops_set, my_ccds, dry_run=True):
"""
check the tags for the given expnum/ccd set.
@param ops:
@param my_expnum:
@return:
"""
tags = storage.get_tags(my_expnum)
count = 0
outcount = 0
fails = []
for ccd in my_ccds:
success = True
count += 1
for ops in ops_set:
for fake in ops[0]:
for my_program in ops[1]:
for version in ops[2]:
#print my_expnum, fake, my_program, version, ccd
key = storage.get_process_tag(fake + my_program, ccd,
version)
uri = storage.tag_uri(key)
if "Failed to get image" in tags.get(
uri, "Failed to get image"):
#print tags.get(uri, None)
fails.append(ccd)
success = False
if success:
outcount += 1
sys.stderr.write("{} {} {:5.1f}%\n".format(outcount, count,
100 * float(outcount) / count))
#print fails
return set(fails)
def main(expnum):
commands = ['mkpsf', 'step1', 'step2', 'step3', 'combine']
tags = storage.get_tags(expnum)
for ccd in range(36):
for command in commands:
tag = storage.tag_uri(storage.get_process_tag(command, ccd, 'p'))
if tags.get(tag, None) != storage.SUCCESS:
sys.stderr.write("FAILED: {} {} {}\n".format(
expnum, ccd, command))
continue
tag = storage.tag_uri(
storage.get_process_tag('fk' + command, ccd, 's'))
if tags.get(tag, None) != storage.SUCCESS:
sys.stderr.write("FAILED: {} {} {}\n".format(
expnum, ccd, command))
continue
def main(expnum):
commands= ['mkpsf',
'step1',
'step2',
'step3',
'combine']
tags = storage.get_tags(expnum)
for ccd in range(36):
for command in commands:
tag = storage.tag_uri(storage.get_process_tag(command, ccd, 'p'))
if tags.get(tag, None) != storage.SUCCESS:
sys.stderr.write("FAILED: {} {} {}\n".format(expnum, ccd, command))
continue
tag = storage.tag_uri(storage.get_process_tag('fk'+command, ccd, 's'))
if tags.get(tag, None) != storage.SUCCESS:
sys.stderr.write("FAILED: {} {} {}\n".format(expnum, ccd, command))
continue
def main(commands=['update_header', 'mkpsf', 'step1', 'step2', 'step3', 'combine'],
ccds=range(0,36),
launch=False, triplets_dir=TRIPLETS_NODE,
delete=False):
cmd_order={'LEAD': ['update_header', 'mkpsf', 'step1', 'step2', 'step3', 'combine'],
'OTHER': ['update_header', 'mkpsf', 'step1' ] }
existance = {'mkpsf': ['psf.fits', ],
'step1': ['obj.jmp', 'obj.matt'],
'step2': ['unid.jmp', 'unid.matt'],
'step3': ['cands.jmp', 'cands.matt']}
for line in get_triplets(triplets_dir):
for command in commands:
exp_type = 'LEAD'
for expnum in line[0:3]:
tags = storage.get_tags(expnum)
if command not in cmd_order[exp_type]:
continue
for ccd in ccds:
tag = storage.tag_uri(storage.get_process_tag(command, ccd))
print line[3], command, expnum, ccd, tags.get(tag,None)
if tags.get(tag,None) != storage.SUCCESS:
if launch:
submit(command, expnum, ccd)
if delete:
success=True
for ext in existance[command]:
uri = storage.dbimages_uri(expnum, ccd, version='p', ext=ext)
if not storage.exists(uri):
success=False
print expnum, ccd, success
#for next_command in cmd_order[exp_type][commands.index(command)+1:]:
# storage.set_status(expnum, ccd, next_command, command+" FAILED")
if success:
storage.set_status(expnum, ccd, command, version='p', status='success')
exp_type='OTHER'
def main():
parser = argparse.ArgumentParser()
parser.add_argument('field')
parser.add_argument('ccd')
parser.add_argument(
'--expnum',
default=None,
help="Which exposure is the lead for this astrom file?")
parser.add_argument(
'--astrom-filename',
default=None,
help="Give the astrom file directly instead of looking-up "
"using the field/ccd naming scheme.")
parser.add_argument('--reals', action='store_true', default=False)
parser.add_argument('--type',
choices=['o', 'p', 's'],
help="Which type of image.",
default='s')
parser.add_argument('--measure3',
default='vos:OSSOS/measure3/2013B-L_redo/')
parser.add_argument('--dbimages', default=None)
parser.add_argument('--dry-run', action='store_true', default=False)
parser.add_argument('--force', action='store_true', default=False)
parser.add_argument("--fk",
action="store_true",
default=False,
help="Do fakes?")
parser.add_argument('--object-planted',
default=OBJECT_PLANTED,
help="Name of file contains list of planted objects.")
parser.add_argument(
'--bright-limit',
default=BRIGHT_LIMIT,
type=float,
help=
"Sources brighter than this limit {} are used to diagnose planting issues."
.format(BRIGHT_LIMIT))
parser.add_argument(
'--minimum-bright-detections',
default=MINIMUM_BRIGHT_DETECTIONS,
type=int,
help=
"required number of detections with mag brighter than bright-limit.")
parser.add_argument(
'--minimum-bright-fraction',
default=MINIMUM_BRIGHT_FRACTION,
type=float,
help=
"minimum fraction of objects above bright limit that should be found.")
parser.add_argument("--debug", action="store_true")
parser.add_argument("--verbose", "-v", action="store_true")
cmd_line = " ".join(sys.argv)
args = parser.parse_args()
util.set_logger(args)
logging.info("Starting {}".format(cmd_line))
prefix = (args.fk and "fk") or ""
version = args.type
ext = 'cands'
if args.reals:
ext = 'reals'
if args.dbimages is not None:
storage.DBIMAGES = args.dbimages
astrom.DATASET_ROOT = args.dbimages
storage.MEASURE3 = args.measure3
astrom_uri = storage.get_cands_uri(args.field,
ccd=args.ccd,
version=args.type,
prefix=prefix,
ext="measure3.{}.astrom".format(ext))
if args.astrom_filename is None:
astrom_filename = os.path.basename(astrom_uri)
else:
astrom_filename = args.astrom_filename
if not os.access(astrom_filename, os.F_OK):
astrom_filename = os.path.dirname(astrom_uri) + "/" + astrom_filename
# Load the list of astrometric observations that will be looked at.
fk_candidate_observations = astrom.parse(astrom_filename)
if args.expnum is None:
expnum = fk_candidate_observations.observations[0].expnum
else:
expnum = args.expnum
message = storage.SUCCESS
if storage.get_status(task, prefix, expnum, version='',
ccd=args.ccd) and not args.force:
logging.info("{} completed successfully for {} {} {} {}".format(
task, prefix, expnum, '', args.ccd))
return
with storage.LoggingManager(task, prefix, expnum, args.ccd, version,
args.dry_run):
try:
match_filename = os.path.splitext(
os.path.basename(astrom_filename))[0] + '.match'
logging.info(f'Got back:{match_filename}')
match_uri = storage.get_cands_uri(
args.field,
ccd=args.ccd,
version=args.type,
prefix=prefix,
ext="measure3.{}.match".format(ext),
block=args.field)
try:
storage.copy(match_uri, match_filename)
except NotFoundException as ex:
logging.warning(
f'No match file found at {match_uri}, creating one.')
logging.debug(f'{ex}')
pass
logging.info(
("Comparing planted and measured magnitudes "
"for sources in {} and {}\n".format(args.object_planted,
astrom_filename)))
result = match_planted(
fk_candidate_observations,
match_filename=match_filename,
object_planted=args.object_planted,
bright_limit=args.bright_limit,
minimum_bright_detections=args.minimum_bright_detections,
bright_fraction=args.minimum_bright_fraction)
if not args.dry_run:
storage.copy(match_filename, match_uri)
uri = os.path.dirname(astrom_uri)
keys = [storage.tag_uri(os.path.basename(astrom_uri))]
values = [result]
storage.set_tags_on_uri(uri, keys, values)
logger.info(message)
except Exception as err:
import traceback
traceback.print_exc()
message = str(err)
logging.error(message)
if not args.dry_run:
storage.set_status(task,
prefix,
expnum,
version='',
ccd=args.ccd,
status=message)
return
def main():
parser = argparse.ArgumentParser()
parser.add_argument('field')
parser.add_argument('ccd')
parser.add_argument('--expnum', default=None, help="Which exposure is the lead for this astrom file?")
parser.add_argument('--astrom-filename', default=None, help="Give the astrom file directly instead of looking-up "
"using the field/ccd naming scheme.")
parser.add_argument('--reals', action='store_true', default=False)
parser.add_argument('--type', choices=['o', 'p', 's'], help="Which type of image.", default='s')
parser.add_argument('--measure3', default='vos:OSSOS/measure3/2013B-L_redo/')
parser.add_argument('--dbimages', default=None)
parser.add_argument('--dry-run', action='store_true', default=False)
parser.add_argument('--force', action='store_true', default=False)
parser.add_argument('--object-planted', default=OBJECT_PLANTED,
help="Name of file contains list of planted objects.")
parser.add_argument('--bright-limit', default=BRIGHT_LIMIT,
help="Sources brighter than this limit {} are used to diagnose planting issues.".format(
BRIGHT_LIMIT))
parser.add_argument('--minimum-bright-detections', default=MINIMUM_BRIGHT_DETECTIONS,
help="required number of detections with mag brighter than bright-limit.")
parser.add_argument('--minimum-bright-fraction', default=MINIMUM_BRIGHT_FRACTION,
help="minimum fraction of objects above bright limit that should be found.")
args = parser.parse_args()
logging.basicConfig(level=logging.INFO)
prefix = 'fk'
ext = args.reals and 'reals' or 'cands'
storage.MEASURE3 = args.measure3
if args.dbimages is not None:
storage.DBIMAGES = args.dbimages
astrom.DATASET_ROOT = args.dbimages
astrom_uri = storage.get_cands_uri(args.field,
ccd=args.ccd,
version=args.type,
prefix=prefix,
ext="measure3.{}.astrom".format(ext))
if args.astrom_filename is None:
astrom_filename = os.path.basename(astrom_uri)
else:
astrom_filename = args.astrom_filename
if not os.access(astrom_filename, os.F_OK):
astrom_filename = os.path.dirname(astrom_uri) + "/" + astrom_filename
# Load the list of astrometric observations that will be looked at.
fk_candidate_observations = astrom.parse(astrom_filename)
if args.expnum is None:
expnum = fk_candidate_observations.observations[0].expnum
else:
expnum = args.expnum
storage.set_logger(os.path.splitext(os.path.basename(sys.argv[0]))[0], prefix, expnum, "", ext, args.dry_run)
match_filename = os.path.splitext(os.path.basename(astrom_filename))[0] + '.match'
exit_status = 0
status = storage.SUCCESS
try:
if (not storage.get_status(expnum, ccd=args.ccd, program='astrom_mag_check', version='')) or args.force:
message = match_planted(fk_candidate_observations,
match_filename=match_filename,
object_planted=args.object_planted,
bright_limit=args.bright_limit,
minimum_bright_detections=args.minimum_bright_detections,
bright_fraction=args.minimum_bright_fraction)
match_uri = storage.get_cands_uri(args.field,
ccd=args.ccd,
version=args.type,
prefix=prefix,
ext="measure3.{}.match".format(ext))
if not args.dry_run:
storage.copy(match_filename, match_uri)
uri = os.path.dirname(astrom_uri)
keys = [storage.tag_uri(os.path.basename(astrom_uri))]
values = [message]
storage.set_tags_on_uri(uri, keys, values)
except Exception as err:
sys.stderr.write(str(err))
status = str(err)
exit_status = err.message
if not args.dry_run:
storage.set_status(expnum, args.ccd, 'astrom_mag_check', version='', status=status)
return exit_status
def main():
parser = argparse.ArgumentParser()
parser.add_argument('field')
parser.add_argument('ccd')
parser.add_argument('--expnum', default=None, help="Which exposure is the lead for this astrom file?")
parser.add_argument('--astrom-filename', default=None, help="Give the astrom file directly instead of looking-up "
"using the field/ccd naming scheme.")
parser.add_argument('--reals', action='store_true', default=False)
parser.add_argument('--type', choices=['o', 'p', 's'], help="Which type of image.", default='s')
parser.add_argument('--measure3', default='vos:OSSOS/measure3/2013B-L_redo/')
parser.add_argument('--dbimages', default=None)
parser.add_argument('--dry-run', action='store_true', default=False)
parser.add_argument('--force', action='store_true', default=False)
parser.add_argument('--object-planted', default=OBJECT_PLANTED,
help="Name of file contains list of planted objects.")
parser.add_argument('--bright-limit', default=BRIGHT_LIMIT,
help="Sources brighter than this limit {} are used to diagnose planting issues.".format(
BRIGHT_LIMIT))
parser.add_argument('--minimum-bright-detections', default=MINIMUM_BRIGHT_DETECTIONS,
help="required number of detections with mag brighter than bright-limit.")
parser.add_argument('--minimum-bright-fraction', default=MINIMUM_BRIGHT_FRACTION,
help="minimum fraction of objects above bright limit that should be found.")
args = parser.parse_args()
logging.basicConfig(level=logging.CRITICAL)
prefix = 'fk'
ext = args.reals and 'reals' or 'cands'
task = util.task()
storage.MEASURE3 = args.measure3
if args.dbimages is not None:
storage.DBIMAGES = args.dbimages
astrom.DATASET_ROOT = args.dbimages
astrom_uri = storage.get_cands_uri(args.field,
ccd=args.ccd,
version=args.type,
prefix=prefix,
ext="measure3.{}.astrom".format(ext))
if args.astrom_filename is None:
astrom_filename = os.path.basename(astrom_uri)
else:
astrom_filename = args.astrom_filename
if not os.access(astrom_filename, os.F_OK):
astrom_filename = os.path.dirname(astrom_uri) + "/" + astrom_filename
# Load the list of astrometric observations that will be looked at.
fk_candidate_observations = astrom.parse(astrom_filename)
if args.expnum is None:
expnum = fk_candidate_observations.observations[0].expnum
else:
expnum = args.expnum
storage.set_logger(os.path.splitext(os.path.basename(sys.argv[0]))[0], prefix, expnum, "", ext, args.dry_run)
match_filename = os.path.splitext(os.path.basename(astrom_filename))[0] + '.match'
exit_status = 0
status = storage.SUCCESS
try:
if (not storage.get_status(task, prefix, expnum=expnum, version='', ccd=args.ccd)) or args.force:
logging.info(("Comparing planted and measured magnitudes "
"for sources in {} and {}\n".format(args.object_planted, astrom_filename)))
message = match_planted(fk_candidate_observations,
match_filename=match_filename,
object_planted=args.object_planted,
bright_limit=args.bright_limit,
minimum_bright_detections=args.minimum_bright_detections,
bright_fraction=args.minimum_bright_fraction)
match_uri = storage.get_cands_uri(args.field,
ccd=args.ccd,
version=args.type,
prefix=prefix,
ext="measure3.{}.match".format(ext), block=args.field)
if not args.dry_run:
storage.copy(match_filename, match_uri)
uri = os.path.dirname(astrom_uri)
keys = [storage.tag_uri(os.path.basename(astrom_uri))]
values = [message]
storage.set_tags_on_uri(uri, keys, values)
except Exception as err:
sys.stderr.write(str(err))
status = str(err)
exit_status = err.message
if not args.dry_run:
storage.set_status(task, prefix, expnum, version='', ccd=args.ccd, status=status)
return exit_status