Exemple #1
0
def clear_tags(expnum, ops_set, my_ccds, dry_run=True):
    """
    Clear the tags for the given expnum/ccd set.
    @param ops:
    @param my_expnum:
    @return:
    """

    status = None
    for ops in ops_set:
        for prefix in ops[0]:
            for task in ops[1]:
                for version in ops[2]:
                    for ccd in my_ccds:
                        if not dry_run:
                            storage.set_status(task, prefix, expnum, version,
                                               ccd, status)
                        else:
                            sys.stdout.write("{} {} {} {} {} {}\n".format(
                                task, prefix, expnum, version, ccd, status))
Exemple #2
0
def main(commands=['update_header', 'mkpsf', 'step1', 'step2', 'step3', 'combine'], 
         ccds=range(0,36),
         launch=False, triplets_dir=TRIPLETS_NODE,
         delete=False):
   
    cmd_order={'LEAD': ['update_header', 'mkpsf', 'step1', 'step2', 'step3', 'combine'],
               'OTHER': ['update_header', 'mkpsf', 'step1' ] }
 
    existance = {'mkpsf': ['psf.fits', ],
                 'step1': ['obj.jmp', 'obj.matt'],
                 'step2': ['unid.jmp', 'unid.matt'],
                 'step3': ['cands.jmp', 'cands.matt']}

    for line in get_triplets(triplets_dir):
        for command in commands:        
            exp_type = 'LEAD'
            for expnum in line[0:3]:
                tags = storage.get_tags(expnum)
                if command not in cmd_order[exp_type]:
                    continue
                for ccd in ccds:
                    tag = storage.tag_uri(storage.get_process_tag(command, ccd))
                    print line[3], command, expnum, ccd, tags.get(tag,None)
                    if tags.get(tag,None) != storage.SUCCESS:
                        if launch:
                            submit(command, expnum, ccd)
                        if delete:
                            success=True
                            for ext in existance[command]:
                                uri = storage.dbimages_uri(expnum, ccd, version='p', ext=ext)
                                if not storage.exists(uri):
                                    success=False
                            print expnum, ccd, success
                            #for next_command in cmd_order[exp_type][commands.index(command)+1:]:
                            #    storage.set_status(expnum, ccd, next_command, command+" FAILED")
                        if success:
                            storage.set_status(expnum, ccd, command, version='p', status='success')
                exp_type='OTHER'
Exemple #3
0
def main(task='step1'):
    ### Must be running as a script

    parser = argparse.ArgumentParser(
        description='Run step1jmp and step1matt on a given exposure.')

    parser.add_argument("--ccd", "-c",
                        action="store",
                        default=None,
                        type=int,
                        dest="ccd")
    parser.add_argument("--ignore", help="Try to run even in previous step failed.",
                        default=False,
                        action="store_true")
    parser.add_argument("--fk", help="add the fk prefix on processing?",
                        default=False,
                        action='store_true')
    parser.add_argument("--dbimages",
                        action="store",
                        default="vos:OSSOS/dbimages",
                        help='vospace dbimages containerNode')
    parser.add_argument("--sex_thresh",
                        action="store",
                        type=float,
                        default=_SEX_THRESHOLD,
                        help="sExtractor detection threhold")
    parser.add_argument("--wavelet_thresh",
                        type=float,
                        default=_WAVE_THRESHOLD,
                        help="Wavelet detection threhold")
    parser.add_argument("expnum",
                        type=int,
                        nargs='+',
                        help="expnum(s) to process")
    parser.add_argument("--version",
                        action='version',
                        version='%(prog)s 1.0')
    parser.add_argument('--type', default='p',
                        choices=['o', 'p', 's'], help="which type of image")
    parser.add_argument('--log', default=None, help="Write standard out to this file")
    parser.add_argument("--verbose", "-v",
                        action="store_true")
    parser.add_argument("--debug", '-d',
                        action='store_true')
    parser.add_argument("--force", action="store_true")
    parser.add_argument("--dry-run", action="store_true", help="Do a dry run, not changes to vospce, implies --force")


    cmd_line = " ".join(sys.argv)
    args = parser.parse_args()

    util.set_logger(args)
    logging.info("Starting {}".format(cmd_line))

    storage.DBIMAGES = args.dbimages

    prefix = (args.fk and 'fk') or ''
    task = util.task()
    dependency = "mkpsf"
    version = args.type

    exit_code = 0
    for expnum in args.expnum:
        if args.ccd is None:
           if int(expnum) < 1785619:
              # Last exposures with 36 CCD Megaprime
              ccdlist = range(0,36)
           else:
              # First exposrues with 40 CCD Megaprime
              ccdlist = range(0, 40)
        else:
            ccdlist = [args.ccd]
        for ccd in ccdlist:
            try:
                message = storage.SUCCESS
                if not (args.force or args.dry_run) and storage.get_status(task, prefix, expnum, version, ccd):
                    logging.critical("{} completed successfully for {}{}{}{:02d}".format(
                        task, prefix, expnum, version, ccd))
                    continue
                if not storage.get_status(dependency, prefix, expnum, version, ccd) and not args.ignore:
                    raise IOError(35, "Cannot start {} as {} not yet completed for {}{}{}{:02d}".format(
                        task, dependency, prefix, expnum, version, ccd))
                storage.set_logger(task, prefix, expnum, ccd, args.type, args.dry_run)
                step1(expnum, ccd, prefix=prefix, version=version, dry_run=args.dry_run)
            except Exception as ex:
                message = str(ex)
                logging.error(message)

            if not args.dry_run:
                storage.set_status(task, prefix, expnum, version, ccd, status=message)
            logging.info(message)
    return exit_code
Exemple #4
0
            ccdlist = [args.ccd]

        for ccd in ccdlist:
            if storage.get_status(
                    task, prefix, expnum, version=args.type,
                    ccd=ccd) and not args.force:
                logging.info(
                    "{} completed successfully for {} {} {} {}".format(
                        task, prefix, expnum, args.type, ccd))
                continue
            storage.set_logger(task, prefix, expnum, ccd, args.type,
                               args.dry_run)
            message = 'success'
            try:
                if not storage.get_status(
                        dependency, prefix, expnum, "p", ccd=ccd):
                    raise IOError("{} not yet run for {}".format(
                        dependency, expnum))
                main(expnum, ccd)  # , args.type, args.dry_run, prefix=prefix)
            except Exception as e:
                message = str(e)

            logging.error(message)
            if not args.dry_run:
                storage.set_status(task,
                                   prefix,
                                   expnum,
                                   version=args.type,
                                   ccd=ccd,
                                   status=message)
Exemple #5
0
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('field')
    parser.add_argument('ccd')
    parser.add_argument(
        '--expnum',
        default=None,
        help="Which exposure is the lead for this astrom file?")
    parser.add_argument(
        '--astrom-filename',
        default=None,
        help="Give the astrom file directly instead of looking-up "
        "using the field/ccd naming scheme.")
    parser.add_argument('--reals', action='store_true', default=False)
    parser.add_argument('--type',
                        choices=['o', 'p', 's'],
                        help="Which type of image.",
                        default='s')
    parser.add_argument('--measure3',
                        default='vos:OSSOS/measure3/2013B-L_redo/')
    parser.add_argument('--dbimages', default=None)
    parser.add_argument('--dry-run', action='store_true', default=False)
    parser.add_argument('--force', action='store_true', default=False)
    parser.add_argument("--fk",
                        action="store_true",
                        default=False,
                        help="Do fakes?")
    parser.add_argument('--object-planted',
                        default=OBJECT_PLANTED,
                        help="Name of file contains list of planted objects.")
    parser.add_argument(
        '--bright-limit',
        default=BRIGHT_LIMIT,
        type=float,
        help=
        "Sources brighter than this limit {} are used to diagnose planting issues."
        .format(BRIGHT_LIMIT))
    parser.add_argument(
        '--minimum-bright-detections',
        default=MINIMUM_BRIGHT_DETECTIONS,
        type=int,
        help=
        "required number of detections with mag brighter than bright-limit.")
    parser.add_argument(
        '--minimum-bright-fraction',
        default=MINIMUM_BRIGHT_FRACTION,
        type=float,
        help=
        "minimum fraction of objects above bright limit that should be found.")
    parser.add_argument("--debug", action="store_true")
    parser.add_argument("--verbose", "-v", action="store_true")

    cmd_line = " ".join(sys.argv)
    args = parser.parse_args()

    util.set_logger(args)
    logging.info("Starting {}".format(cmd_line))
    prefix = (args.fk and "fk") or ""
    version = args.type
    ext = 'cands'
    if args.reals:
        ext = 'reals'

    if args.dbimages is not None:
        storage.DBIMAGES = args.dbimages
        astrom.DATASET_ROOT = args.dbimages
    storage.MEASURE3 = args.measure3
    astrom_uri = storage.get_cands_uri(args.field,
                                       ccd=args.ccd,
                                       version=args.type,
                                       prefix=prefix,
                                       ext="measure3.{}.astrom".format(ext))

    if args.astrom_filename is None:
        astrom_filename = os.path.basename(astrom_uri)
    else:
        astrom_filename = args.astrom_filename

    if not os.access(astrom_filename, os.F_OK):
        astrom_filename = os.path.dirname(astrom_uri) + "/" + astrom_filename

    # Load the list of astrometric observations that will be looked at.
    fk_candidate_observations = astrom.parse(astrom_filename)
    if args.expnum is None:
        expnum = fk_candidate_observations.observations[0].expnum
    else:
        expnum = args.expnum

    message = storage.SUCCESS

    if storage.get_status(task, prefix, expnum, version='',
                          ccd=args.ccd) and not args.force:
        logging.info("{} completed successfully for {} {} {} {}".format(
            task, prefix, expnum, '', args.ccd))
        return

    with storage.LoggingManager(task, prefix, expnum, args.ccd, version,
                                args.dry_run):
        try:

            match_filename = os.path.splitext(
                os.path.basename(astrom_filename))[0] + '.match'
            logging.info(f'Got back:{match_filename}')
            match_uri = storage.get_cands_uri(
                args.field,
                ccd=args.ccd,
                version=args.type,
                prefix=prefix,
                ext="measure3.{}.match".format(ext),
                block=args.field)

            try:
                storage.copy(match_uri, match_filename)
            except NotFoundException as ex:
                logging.warning(
                    f'No match file found at {match_uri}, creating one.')
                logging.debug(f'{ex}')
                pass

            logging.info(
                ("Comparing planted and measured magnitudes "
                 "for sources in {} and {}\n".format(args.object_planted,
                                                     astrom_filename)))
            result = match_planted(
                fk_candidate_observations,
                match_filename=match_filename,
                object_planted=args.object_planted,
                bright_limit=args.bright_limit,
                minimum_bright_detections=args.minimum_bright_detections,
                bright_fraction=args.minimum_bright_fraction)
            if not args.dry_run:
                storage.copy(match_filename, match_uri)
                uri = os.path.dirname(astrom_uri)
                keys = [storage.tag_uri(os.path.basename(astrom_uri))]
                values = [result]
                storage.set_tags_on_uri(uri, keys, values)
            logger.info(message)
        except Exception as err:
            import traceback
            traceback.print_exc()
            message = str(err)
            logging.error(message)

        if not args.dry_run:
            storage.set_status(task,
                               prefix,
                               expnum,
                               version='',
                               ccd=args.ccd,
                               status=message)

    return
Exemple #6
0
def run(expnums,
        ccd,
        version,
        rate_min,
        rate_max,
        angle,
        width,
        field=None,
        prefix=None,
        dry_run=False,
        force=False):
    """run the actual step2  on the given exp/ccd combo"""

    jmp_args = ['step3jmp']
    matt_args = ['step3matt']

    if storage.get_status(task, prefix, expnums[0], version=version,
                          ccd=ccd) and not force:
        logging.info("{} completed successfully for {}{}{}{:02d}".format(
            task, prefix, expnums[0], version, ccd))
        return

    with storage.LoggingManager(task, prefix, expnums[0], ccd, version,
                                dry_run):
        try:
            if not storage.get_status(
                    dependency, prefix, expnums[0], version=version, ccd=ccd):
                raise IOError(
                    35, "Cannot start {} as {} not yet completed {}{}{}{:02d}".
                    format(task, dependency, prefix, expnums[0], version, ccd))
            # Default message is success, message gets overwritten with failure messages.
            message = storage.SUCCESS

            idx = 0
            cmd_args = []
            for expnum in expnums:
                idx += 1
                for ext in ['unid.jmp', 'unid.matt', 'trans.jmp']:
                    storage.get_file(expnum,
                                     ccd=ccd,
                                     version=version,
                                     ext=ext,
                                     prefix=prefix)
                image = os.path.splitext(
                    os.path.basename(
                        storage.get_uri(expnum,
                                        ccd,
                                        version=version,
                                        prefix=prefix)))[0]
                cmd_args.append('-f%d' % idx)
                cmd_args.append(image)

            cmd_args.extend([
                '-rn',
                str(rate_min), '-rx',
                str(rate_max), '-a',
                str(angle), '-w',
                str(width)
            ])
            jmp_args.extend(cmd_args)
            matt_args.extend(cmd_args)
            logging.info(util.exec_prog(jmp_args))
            logging.info(util.exec_prog(matt_args))

            if dry_run:
                return

            if field is None:
                field = str(expnums[0])

            # Make sure a dbimages destination exists for this file.
            storage.mkdir(
                os.path.dirname(
                    storage.get_uri(field,
                                    ccd=ccd,
                                    version=version,
                                    prefix=prefix)))

            for ext in ['moving.jmp', 'moving.matt']:
                uri = storage.get_uri(field,
                                      ccd=ccd,
                                      version=version,
                                      ext=ext,
                                      prefix=prefix)
                filename = '%s%d%s%s.%s' % (prefix, expnums[0], version,
                                            str(ccd).zfill(2), ext)
                storage.copy(filename, uri)

        except Exception as ex:
            message = str(ex)
            logging.error(message)

        storage.set_status(task,
                           prefix,
                           expnums[0],
                           version=version,
                           ccd=ccd,
                           status=message)

    return
Exemple #7
0
    logging.basicConfig(level=level, format="%(message)s")

    storage._dbimages = args.dbimages

    if args.ccd is None:
        ccdlist = range(0, 36)
    else:
        ccdlist = [args.ccd]

    prefix = (args.fk and 'fk') or ''

    for expnum in args.expnum:
        for ccd in ccdlist:
            try:
                message = storage.SUCCESS
                if not storage.get_status(expnum, ccd, 'mkpsf'):
                    raise IOError(35, "mkpsf hasn't run?")
                if storage.get_status(expnum, ccd,
                                      prefix + 'step1') and not args.force:
                    logging.critical("Already did %s %s, skipping" %
                                     (prefix + str(expnum), str(ccd)))
                    continue
                logging.info("step1 on expnum: %s, ccd: %s" %
                             (prefix + str(expnum), str(ccd)))
                step1(expnum, ccd, prefix=prefix, version=args.type)
            except Exception as e:
                message = str(e)
                logging.error("Error running step1: %s " % (message))

            storage.set_status(expnum, ccd, prefix + 'step1', message)
Exemple #8
0
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('field')
    parser.add_argument('ccd')
    parser.add_argument('--expnum', default=None, help="Which exposure is the lead for this astrom file?")
    parser.add_argument('--astrom-filename', default=None, help="Give the astrom file directly instead of looking-up "
                                                                "using the field/ccd naming scheme.")
    parser.add_argument('--reals', action='store_true', default=False)
    parser.add_argument('--type', choices=['o', 'p', 's'], help="Which type of image.", default='s')
    parser.add_argument('--measure3', default='vos:OSSOS/measure3/2013B-L_redo/')
    parser.add_argument('--dbimages', default=None)
    parser.add_argument('--dry-run', action='store_true', default=False)
    parser.add_argument('--force', action='store_true', default=False)

    parser.add_argument('--object-planted', default=OBJECT_PLANTED,
                        help="Name of file contains list of planted objects.")
    parser.add_argument('--bright-limit', default=BRIGHT_LIMIT,
                        help="Sources brighter than this limit {} are used to diagnose planting issues.".format(
                            BRIGHT_LIMIT))
    parser.add_argument('--minimum-bright-detections', default=MINIMUM_BRIGHT_DETECTIONS,
                        help="required number of detections with mag brighter than bright-limit.")
    parser.add_argument('--minimum-bright-fraction', default=MINIMUM_BRIGHT_FRACTION,
                        help="minimum fraction of objects above bright limit that should be found.")
    args = parser.parse_args()

    logging.basicConfig(level=logging.INFO)

    prefix = 'fk'
    ext = args.reals and 'reals' or 'cands'

    storage.MEASURE3 = args.measure3

    if args.dbimages is not None:
        storage.DBIMAGES = args.dbimages
        astrom.DATASET_ROOT = args.dbimages

    astrom_uri = storage.get_cands_uri(args.field,
                                       ccd=args.ccd,
                                       version=args.type,
                                       prefix=prefix,
                                       ext="measure3.{}.astrom".format(ext))

    if args.astrom_filename is None:
        astrom_filename = os.path.basename(astrom_uri)
    else:
        astrom_filename = args.astrom_filename

    if not os.access(astrom_filename, os.F_OK):
        astrom_filename = os.path.dirname(astrom_uri) + "/" + astrom_filename

    # Load the list of astrometric observations that will be looked at.
    fk_candidate_observations = astrom.parse(astrom_filename)
    if args.expnum is None:
        expnum = fk_candidate_observations.observations[0].expnum
    else:
        expnum = args.expnum

    storage.set_logger(os.path.splitext(os.path.basename(sys.argv[0]))[0], prefix, expnum, "", ext, args.dry_run)

    match_filename = os.path.splitext(os.path.basename(astrom_filename))[0] + '.match'

    exit_status = 0
    status = storage.SUCCESS
    try:
        if (not storage.get_status(expnum, ccd=args.ccd, program='astrom_mag_check', version='')) or args.force:
            message = match_planted(fk_candidate_observations,
                                    match_filename=match_filename,
                                    object_planted=args.object_planted,
                                    bright_limit=args.bright_limit,
                                    minimum_bright_detections=args.minimum_bright_detections,
                                    bright_fraction=args.minimum_bright_fraction)
            match_uri = storage.get_cands_uri(args.field,
                                              ccd=args.ccd,
                                              version=args.type,
                                              prefix=prefix,
                                              ext="measure3.{}.match".format(ext))
            if not args.dry_run:
                storage.copy(match_filename, match_uri)
                uri = os.path.dirname(astrom_uri)
                keys = [storage.tag_uri(os.path.basename(astrom_uri))]
                values = [message]
                storage.set_tags_on_uri(uri, keys, values)
    except Exception as err:
        sys.stderr.write(str(err))
        status = str(err)
        exit_status = err.message

    if not args.dry_run:
        storage.set_status(expnum, args.ccd, 'astrom_mag_check', version='', status=status)

    return exit_status
Exemple #9
0
        message = storage.SUCCESS

        try:
            if not storage.get_status(args.expnums[0], ccd, prefix+'step2', version=args.type):
                raise IOError(35, "did step2 run on %s" % ( str(args.expnums)))
            if storage.get_status(args.expnums[0], ccd, prefix+'step3', version=args.type) and not args.force:
                logging.critical("step3 alread ran on expnum :%s, ccd: %d" % (
                        str(args.expnums), ccd))
                continue
            step3(args.expnums, ccd, version=args.type,
                  rate_min=args.rate_min,
                  rate_max=args.rate_max,
                  angle=args.angle,
                  width=args.width,
                  field=args.field,
                  prefix=prefix)
        except Exception as e:
            message = str(e)

        logging.error(message)

        storage.set_status(args.expnums[0],
                           ccd,
                           prefix+'step3',
                           version=args.type,
                           status=message)
        
            


Exemple #10
0
        args.expnums.sort()

    logging.basicConfig(level=level, format="%(message)s")

    ccds = [args.ccd]
    if args.ccd is None:
        ccds = range(0, 36)

    for ccd in ccds:
        message = storage.SUCCESS
        try:
            if not storage.get_status(args.expnums[0], ccd, 'scramble'):
                raise IOError("scramble not yet run for %s ccd%s" %
                              (str(args.expnums), str(ccd).zfill(2)))
            if storage.get_status(args.expnums[0], ccd,
                                  'plant') and not args.force:
                logging.info("plant done for %s[%s]" % (args.expnums[0], ccd))
                continue
            plant(args.expnums,
                  ccd,
                  args.rmin,
                  args.rmax,
                  args.ang,
                  args.width,
                  version=args.type)
        except Exception as e:
            message = str(e)
            logging.error(str(e))

        storage.set_status(args.expnums[0], ccd, 'plant', message)
Exemple #11
0
    ccdlist = ( args.ccd is None and range(0,36) ) or [args.ccd]

    for ccd in ccdlist:
        message = storage.SUCCESS
        if 1==1:
            if not storage.get_status(args.expnum, ccd, prefix+'step3', version=args.type):
                logging.error(storage.get_status(
                        args.expnum,
                        ccd,
                        'step3',
                        return_message=True))
                raise IOError(35, "need to run step3 first")
            if storage.get_status(args.expnum, ccd,
                                  prefix+'combine',
                                  version=args.type) and not args.force:
                continue
            message = combine(args.expnum, 
                              ccd, 
                              prefix=prefix, 
                              type=args.type, 
                              field=args.field, 
                              measure3=args.measure3
                              )
        #except Exception as e:
        #   message = str(e)
            
        storage.set_status(args.expnum, ccd,
                           prefix+'combine',
                           version=args.type,
                           status=message)
Exemple #12
0
def main():
    ### Must be running as a script

    parser = argparse.ArgumentParser(
        description='Run step3jmp and step3matt on a given triple.')

    parser.add_argument("--ccd",
                        "-c",
                        action="store",
                        default=None,
                        type=int,
                        dest="ccd")
    parser.add_argument("--dbimages",
                        action="store",
                        default="vos:OSSOS/dbimages",
                        help='vospace dbimages containerNode')
    parser.add_argument("expnums",
                        type=int,
                        nargs=3,
                        help="3 expnums to process")
    parser.add_argument("--version", action='version', version='%(prog)s 1.0')
    parser.add_argument('-t',
                        '--type',
                        help='which type of image to process',
                        choices=['s', 'p', 'o'],
                        default='p')
    parser.add_argument('--fk',
                        help='Do fakes?',
                        default=False,
                        action='store_true')
    parser.add_argument(
        '--field',
        help='a string that identifies which field is being searched',
        nargs='?')
    parser.add_argument('--no-sort',
                        help='preserve input exposure order',
                        action='store_true')
    parser.add_argument("--verbose", "-v", action="store_true")
    parser.add_argument("--debug", action="store_true")
    parser.add_argument("--rate_min",
                        default=_RATE_MIN,
                        help='minimum rate to accept',
                        type=float)
    parser.add_argument('--rate_max',
                        default=_RATE_MAX,
                        help='maximum rate to accept',
                        type=float)
    parser.add_argument('--angle',
                        default=_ANGLE_CENTRE,
                        help='angle of x/y motion, West is 0, North 90',
                        type=float)
    parser.add_argument('--width',
                        default=_ANGLE_WIDTH,
                        help='openning angle of search cone',
                        type=float)
    parser.add_argument("--dry_run",
                        action="store_true",
                        help="do not copy to VOSpace, implies --force")
    parser.add_argument("--force", action="store_true")

    args = parser.parse_args()

    if args.dry_run:
        args.force = True

    if args.verbose:
        logging.basicConfig(level=logging.INFO, format="%(message)s")
    if args.debug:
        logging.basicConfig(level=logging.DEBUG, format="%(message)s")

    storage.DBIMAGES = args.dbimages

    if args.ccd is None:
        ccdlist = range(0, 36)
    else:
        ccdlist = [args.ccd]

    if not args.no_sort:
        args.expnums.sort()

    prefix = (args.fk and 'fk') or ''

    exit_status = 0
    for ccd in ccdlist:
        storage.set_logger(
            os.path.splitext(os.path.basename(sys.argv[0]))[0], prefix,
            args.expnums[0], ccd, args.type, args.dry_run)
        message = storage.SUCCESS
        try:
            if not storage.get_status(
                    args.expnums[0], ccd, prefix + 'step2', version=args.type):
                raise IOError(35, "did step2 run on %s" % str(args.expnums))
            if storage.get_status(
                    args.expnums[0], ccd, prefix + 'step3',
                    version=args.type) and not args.force:
                logging.critical("step3 alread ran on expnum :%s, ccd: %d" %
                                 (str(args.expnums), ccd))
                continue
            step3(args.expnums,
                  ccd,
                  version=args.type,
                  rate_min=args.rate_min,
                  rate_max=args.rate_max,
                  angle=args.angle,
                  width=args.width,
                  field=args.field,
                  prefix=prefix,
                  dry_run=args.dry_run)
        except Exception as e:
            message = str(e)
            exit_status = message

        logging.error(message)
        if not args.dry_run:
            storage.set_status(args.expnums[0],
                               ccd,
                               prefix + 'step3',
                               version=args.type,
                               status=message)

    return exit_status
Exemple #13
0
    level = logging.CRITICAL
    if args.debug:
        level = logging.DEBUG
    elif args.verbose:
        level = logging.INFO

    logging.basicConfig(level=level, format="%(message)s")

    if not args.nosort:
        args.expnums.sort()

    ccds = [args.ccd]
    if args.ccd is None:
        ccds = range(0,36)
    for ccd in ccds:
        expnums = args.expnums
        if storage.get_status(expnums[0], ccd,
                              'scramble') and not args.force:
            continue
        message = storage.SUCCESS

        try:
            scramble(expnums=expnums, ccd=ccd, version=args.type)
        except Exception as e:
            logging.error(e)
            message = str(e)

        storage.set_status(expnums[0], ccd,
                           'scramble',
                           message)
Exemple #14
0
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('field')
    parser.add_argument('ccd')
    parser.add_argument('--expnum', default=None, help="Which exposure is the lead for this astrom file?")
    parser.add_argument('--astrom-filename', default=None, help="Give the astrom file directly instead of looking-up "
                                                                "using the field/ccd naming scheme.")
    parser.add_argument('--reals', action='store_true', default=False)
    parser.add_argument('--type', choices=['o', 'p', 's'], help="Which type of image.", default='s')
    parser.add_argument('--measure3', default='vos:OSSOS/measure3/2013B-L_redo/')
    parser.add_argument('--dbimages', default=None)
    parser.add_argument('--dry-run', action='store_true', default=False)
    parser.add_argument('--force', action='store_true', default=False)

    parser.add_argument('--object-planted', default=OBJECT_PLANTED,
                        help="Name of file contains list of planted objects.")
    parser.add_argument('--bright-limit', default=BRIGHT_LIMIT,
                        help="Sources brighter than this limit {} are used to diagnose planting issues.".format(
                            BRIGHT_LIMIT))
    parser.add_argument('--minimum-bright-detections', default=MINIMUM_BRIGHT_DETECTIONS,
                        help="required number of detections with mag brighter than bright-limit.")
    parser.add_argument('--minimum-bright-fraction', default=MINIMUM_BRIGHT_FRACTION,
                        help="minimum fraction of objects above bright limit that should be found.")
    args = parser.parse_args()

    logging.basicConfig(level=logging.CRITICAL)

    prefix = 'fk'
    ext = args.reals and 'reals' or 'cands'
    task = util.task()

    storage.MEASURE3 = args.measure3

    if args.dbimages is not None:
        storage.DBIMAGES = args.dbimages
        astrom.DATASET_ROOT = args.dbimages

    astrom_uri = storage.get_cands_uri(args.field,
                                       ccd=args.ccd,
                                       version=args.type,
                                       prefix=prefix,
                                       ext="measure3.{}.astrom".format(ext))

    if args.astrom_filename is None:
        astrom_filename = os.path.basename(astrom_uri)
    else:
        astrom_filename = args.astrom_filename

    if not os.access(astrom_filename, os.F_OK):
        astrom_filename = os.path.dirname(astrom_uri) + "/" + astrom_filename

    # Load the list of astrometric observations that will be looked at.
    fk_candidate_observations = astrom.parse(astrom_filename)
    if args.expnum is None:
        expnum = fk_candidate_observations.observations[0].expnum
    else:
        expnum = args.expnum

    storage.set_logger(os.path.splitext(os.path.basename(sys.argv[0]))[0], prefix, expnum, "", ext, args.dry_run)
    match_filename = os.path.splitext(os.path.basename(astrom_filename))[0] + '.match'

    exit_status = 0
    status = storage.SUCCESS
    try:
        if (not storage.get_status(task, prefix, expnum=expnum, version='', ccd=args.ccd)) or args.force:
            logging.info(("Comparing planted and measured magnitudes "
                          "for sources in {} and {}\n".format(args.object_planted, astrom_filename)))
            message = match_planted(fk_candidate_observations,
                                    match_filename=match_filename,
                                    object_planted=args.object_planted,
                                    bright_limit=args.bright_limit,
                                    minimum_bright_detections=args.minimum_bright_detections,
                                    bright_fraction=args.minimum_bright_fraction)
            match_uri = storage.get_cands_uri(args.field,
                                              ccd=args.ccd,
                                              version=args.type,
                                              prefix=prefix,
                                              ext="measure3.{}.match".format(ext), block=args.field)
            if not args.dry_run:
                storage.copy(match_filename, match_uri)
                uri = os.path.dirname(astrom_uri)
                keys = [storage.tag_uri(os.path.basename(astrom_uri))]
                values = [message]
                storage.set_tags_on_uri(uri, keys, values)
    except Exception as err:
        sys.stderr.write(str(err))
        status = str(err)
        exit_status = err.message

    if not args.dry_run:
        storage.set_status(task, prefix, expnum, version='', ccd=args.ccd, status=status)

    return exit_status
Exemple #15
0
def main():
    """Run the script."""

    parser = argparse.ArgumentParser()
    parser.add_argument('--ccd',
                        action='store',
                        type=int,
                        default=None,
                        help='which ccd to process')
    parser.add_argument("--dbimages",
                        action="store",
                        default="vos:OSSOS/dbimages",
                        help='vospace dbimages containerNode')
    parser.add_argument("expnums",
                        type=int,
                        nargs=3,
                        help="expnums to scramble")
    parser.add_argument("--nosort",
                        action='store_true',
                        default=False,
                        help="do not sort before processing")
    parser.add_argument("--type",
                        action='store',
                        default='p',
                        choices=['p', 'o'],
                        help='which type of image')
    parser.add_argument("--verbose", "-v",
                        action="store_true")
    parser.add_argument("--debug", '-d',
                        action='store_true')
    parser.add_argument("--dry_run", action="store_true", help="Do not copy back to VOSpace, implies --force")
    parser.add_argument("--force", action='store_true')
    
    args = parser.parse_args()

    if args.dry_run:
        args.force = True

    storage.DBIMAGES = args.dbimages

    ## setup logging
    level = logging.CRITICAL
    if args.debug:
        level = logging.DEBUG
    elif args.verbose:
        level = logging.INFO

    logging.basicConfig(level=level, format="%(message)s")
    logging.getLogger('vos').setLevel(level)
    logging.getLogger('vos').addHandler(logging.StreamHandler())
    
    if not args.nosort:
        args.expnums.sort()

    ccds = [args.ccd]
    exit_status = 0
    if args.ccd is None:
        ccds = range(0, 36)
    for ccd in ccds:
        storage.set_logger(os.path.splitext(os.path.basename(sys.argv[0]))[0],
                           "", args.expnums[0], ccd, args.type, args.dry_run)
        expnums = args.expnums
        if storage.get_status(expnums[0], ccd,
                              'scramble', version='s') and not args.force:
            continue
        message = storage.SUCCESS

        try:
            scramble(expnums=expnums, ccd=ccd, version='p', dry_run=args.dry_run)
        except Exception as e:
            logging.error(e)
            message = str(e)
            exit_status = message
        if not args.dry_run:
            storage.set_status(expnums[0], ccd,
                               'scramble', version='s',
                               status=message)
    return exit_status
Exemple #16
0
def run(expnum, ccd, version='p', prefix='', dry_run=False, force=False):

    message = 'success'

    if storage.get_status(task, prefix, expnum, version=version,
                          ccd=ccd) and not force:
        logging.info("{} completed successfully for {} {} {} {}".format(
            task, prefix, expnum, version, ccd))
        return

    with storage.LoggingManager(task=task,
                                prefix=prefix,
                                expnum=expnum,
                                ccd=ccd,
                                version=version,
                                dry_run=dry_run):
        try:
            if not storage.get_status(dependency, prefix, expnum, "p",
                                      ccd=ccd):
                raise IOError("{} not yet run for {}".format(
                    dependency, expnum))

            header = storage.get_astheader(expnum, ccd)
            datasec = storage.datasec_to_list(
                header.get('DATASEC', '[80:2080,30,4160]'))
            try:
                fwhm = "{:5.2f}".format(storage.get_fwhm(expnum, ccd))
            except:
                fwhm = 'unknown'
            for keyword in del_keyword_list:
                try:
                    del (header[keyword])
                except:
                    pass
            header['FWHM'] = (fwhm, 'FWHM in pixels')
            header['EXTNAME'] = 'header'
            primary_hdu = fits.PrimaryHDU(header=header)
            hdu_list = fits.HDUList([
                primary_hdu,
            ])
            for ext in ['jmp', 'matt']:
                extension = 'obj.' + ext
                name = "{}p{:02d}.{}".format(expnum, ccd, extension)
                try:
                    os.unlink(name)
                    os.unlink(name + ".fits")
                except:
                    pass
                logging.info("Retrieving {}".format(name))
                obj_file = mop_file.Parser(expnum, ccd, extension)
                obj_file.parse()

                t = numpy.all([
                    datasec[0] < obj_file.data['X'],
                    obj_file.data['X'] < datasec[1],
                    datasec[2] < obj_file.data['Y'],
                    obj_file.data['Y'] < datasec[3]
                ],
                              axis=0)
                logging.info(
                    "Source remaining after datasec cut: {} of {}".format(
                        len(obj_file.data[t]['X']), len(t)))
                table_hdu = fits.table_to_hdu(obj_file.data[t])
                table_hdu.header['CATALOG'] = name
                table_hdu.header['EXTNAME'] = ext
                hdu_list.append(table_hdu)
                del table_hdu
                del obj_file
                os.unlink(name)

            name = "{}p{:02d}.{}".format(expnum, ccd, 'obj.fits')
            if os.access(name, os.F_OK):
                os.unlink(name)
            hdu_list.writeto(name)
            uri = storage.dbimages_uri(expnum, ccd, 'p', ext=".obj.fits")
            logging.info(name + " -> " + uri)
            count = 0
            with open(name):
                while True:
                    count += 1
                    logging.info("Copy attempt {}".format(count))
                    try:
                        storage.copy(name, uri)
                        os.unlink(name)
                        break
                    except Exception as ex:
                        if count > 10:
                            raise ex

            logging.info(message)
        except Exception as e:
            message = str(e)
            logging.error(message)

        if not dry_run:
            storage.set_status(task,
                               prefix,
                               expnum,
                               version=version,
                               ccd=ccd,
                               status=message)
Exemple #17
0
def run(expnum,
        ccd,
        prefix='',
        version='p',
        sex_thresh=_SEX_THRESHOLD,
        wave_thresh=_WAVE_THRESHOLD,
        maxcount=_MAX_COUNT,
        dry_run=False, 
        force=True):
    """run the actual step1jmp/matt codes.

    expnum: the CFHT expousre to process
    ccd: which ccd in the mosaic to process
    fwhm: the image quality, FWHM, of the image.  In pixels.
    sex_thresh: the detection threhold to run sExtractor at
    wave_thresh: the detection threshold for wavelet
    maxcount: saturation level

    """
    message = storage.SUCCESS

    if storage.get_status(task, prefix, expnum, version, ccd) and not force:
        logging.info("{} completed successfully for {} {} {} {}".format(task, prefix, expnum, version, ccd))
        return
        
    with storage.LoggingManager(task, prefix, expnum, ccd, version, dry_run):
        try:        
            if not storage.get_status(dependency, prefix, expnum, version, ccd):
                raise IOError(35, "Cannot start {} as {} not yet completed for {}{}{}{:02d}".format(
                    task, dependency, prefix, expnum, version, ccd))
            logging.info("Retrieving imaging and input parameters from VOSpace")    
            storage.get_file(expnum, ccd, prefix=prefix, version=version, ext='mopheader')
            filename = storage.get_image(expnum, ccd, version=version, prefix=prefix)
            fwhm = storage.get_fwhm(expnum, ccd, prefix=prefix, version=version)
            basename = os.path.splitext(filename)[0]

            _get_weight_map(filename, ccd)
            
            logging.info("Launching step1jmp")
            logging.info(util.exec_prog(['step1jmp',
                                         '-f', basename,
                                         '-t', str(wave_thresh),
                                         '-w', str(fwhm),
                                         '-m', str(maxcount)]))

            logging.info(util.exec_prog(['step1matt',
                                 '-f', basename,
                                 '-t', str(sex_thresh),
                                 '-w', str(fwhm),
                                 '-m', str(maxcount)]))

            if os.access('weight.fits', os.R_OK):
                os.unlink('weight.fits')
            
            if not dry_run:
                for ext in ['obj.jmp', 'obj.matt']:
                    obj_uri = storage.get_uri(expnum, ccd, version=version, ext=ext,
                                              prefix=prefix)
                    obj_filename = basename + "." + ext
                    count = 0
                    with open(obj_filename, 'r'):
                      while True:
                        try:
                            count += 1
                            logging.info("Attempt {} to copy {} -> {}".format(count, obj_filename, obj_uri))
                            storage.copy(obj_filename, obj_uri)
                            break
                        except Exception as ex:
                            if count > 10:
                                raise ex
            logging.info(message)
        except Exception as ex:
            message = str(ex)
            logging.error(message)

        if not dry_run:
            storage.set_status(task, prefix, expnum, version, ccd, status=message)
Exemple #18
0
            os.access(args.header, os.W_OK) and args.header ) or (
            storage.get_image(args.header, ext='head')))) or ( 
            storage.get_image(args.expnum, ext='head'))

        image = (os.access(args.expnum,os.W_OK) and args.expnum ) or (
            storage.get_image(args.expnum) )

        logging.info(
            "Swapping header for %s for contents in %s \n" % (
            image, header) )


        expnum = fits.open(image)[0].header['EXPNUM'] or args.expnum

        # skip if already succeeded and not in force mode
        if storage.get_status(expnum, 36, 'update_header_p') and not args.force:
            logging.info("Skipping {} as already done.".format(expnum))
            sys.exit(0)

        run_update_header(image, header)
        if args.replace:
            dest = storage.dbimages_uri(expnum)
            storage.copy(image, dest)
            storage.set_status(expnum, 36, 'update_header_p', message)
    except Exception as e:
        if args.replace:
            message = str(e)
            storage.set_status(expnum, 36, 'update_header_p', message)
        logging.error(str(e))

Exemple #19
0
        level = logging.DEBUG
    elif args.verbose:
        level = logging.INFO

    logging.basicConfig(level=level, format="%(message)s")
    logging.getLogger('vos').setLevel(level)
    logging.getLogger('vos').addHandler(logging.StreamHandler())
    
    if not args.nosort:
        args.expnums.sort()

    ccds = [args.ccd]
    if args.ccd is None:
        ccds = range(0,36)
    for ccd in ccds:
        expnums = args.expnums
        if storage.get_status(expnums[0], ccd,
                              'scramble', version='s') and not args.force:
            continue
        message = storage.SUCCESS

        try:
            scramble(expnums=expnums, ccd=ccd, version='p')
        except Exception as e:
            logging.error(e)
            message = str(e)

        storage.set_status(expnums[0], ccd,
                           'scramble', version='s',
                           status=message)
Exemple #20
0
def main(task='mkpsf'):

    parser = argparse.ArgumentParser(
        description='Run makepsf chunk of the OSSOS pipeline')

    parser.add_argument('--ccd', '-c',
                        action='store',
                        type=int,
                        dest='ccd',
                        default=None,
                        help='which ccd to process, default is all')
    parser.add_argument("--dbimages",
                        action="store",
                        default="vos:OSSOS/dbimages",
                        help='vospace dbimages containerNode')
    parser.add_argument("expnum",
                        type=int,
                        nargs='+',
                        help="expnum(s) to process")
    parser.add_argument("--dry-run",
                        action="store_true",
                        help="DRY RUN, don't copy results to VOSpace, implies --force")

    parser.add_argument("--fk", action="store_true", help="Run fk images")

    parser.add_argument("--type", "-t", choices=['o', 'p', 's'],
                        help="which type of image: o-RAW, p-ELIXIR, s-SCRAMBLE", default='p')
    parser.add_argument("--verbose", "-v",
                        action="store_true")
    parser.add_argument("--force", default=False,
                        action="store_true")
    parser.add_argument("--debug", "-d",
                        action="store_true")

    args = parser.parse_args()

    util.set_logger(args)

    prefix = (args.fk and 'fk') or ''
    task  = util.task()
    dependency = 'mk_mopheader'

    storage.DBIMAGES = args.dbimages

    exit_code = 0
    for expnum in args.expnum:
        if args.ccd is None:
           if int(expnum) < 1785619:
               # Last exposures with 36 CCD Megaprime
               ccdlist = range(0,36)
           else:
               # First exposrues with 40 CCD Megaprime
               ccdlist = range(0, 40)
        else:
           ccdlist = [args.ccd]
        for ccd in ccdlist:
            if storage.get_status(task, prefix, expnum, version=args.type, ccd=ccd) and not args.force:
                logging.info("{} completed successfully for {} {} {} {}".format(task, prefix, expnum, args.type, ccd))
                continue
            storage.set_logger(task,
                               prefix, expnum, ccd, args.type, args.dry_run)
            message = 'success'
            try:
                if not storage.get_status(dependency, prefix, expnum, "p", ccd=ccd):
                    raise IOError("{} not yet run for {}".format(dependency, expnum))
                mkpsf(expnum, ccd, args.type, args.dry_run, prefix=prefix)
                if args.dry_run:
                    continue
                storage.set_status('fwhm', prefix, expnum, version=args.type, ccd=ccd, status=str(storage.get_fwhm(
                    expnum, ccd, version=args.type)))
                storage.set_status('zeropoint', prefix, expnum, version=args.type, ccd=ccd,
                                   status=str(storage.get_zeropoint(
                                       expnum, ccd, version=args.type)))
            except Exception as e:
                message = str(e)

            logging.error(message)
            if not args.dry_run:
                storage.set_status(task, prefix, expnum, version=args.type, ccd=ccd, status=message)
    return exit_code
Exemple #21
0
def main():
    parser = argparse.ArgumentParser(
        description='Run mopheader chunk of the OSSOS pipeline')

    parser.add_argument('--ccd', '-c',
                        action='store',
                        type=int,
                        dest='ccd',
                        default=None,
                        help='which ccd to process, default is all')
    parser.add_argument('--ignore-update-headers', action='store_true', dest='ignore_update_headers')
    parser.add_argument("--dbimages",
                        action="store",
                        default="vos:OSSOS/dbimages",
                        help='vospace dbimages containerNode')
    parser.add_argument("expnum",
                        type=int,
                        nargs='+',
                        help="expnum(s) to process")
    parser.add_argument("--dry-run",
                        action="store_true",
                        help="DRY RUN, don't copy results to VOSpace, implies --force")

    parser.add_argument("--fk", action="store_true", help="Run fk images")

    parser.add_argument("--type", "-t", choices=['o', 'p', 's'],
                        help="which type of image: o-RAW, p-ELIXIR, s-SCRAMBLE", default='p')
    parser.add_argument("--verbose", "-v",
                        action="store_true")
    parser.add_argument("--force", default=False,
                        action="store_true")
    parser.add_argument("--debug", "-d",
                        action="store_true")

    args = parser.parse_args()

    util.set_logger(args)

    
    logging.info("started")
    prefix = (args.fk and 'fk') or ''
    task = util.task()
    dependency = 'update_header'

    storage.DBIMAGES = args.dbimages

    if args.ccd is None:
        ccdlist = range(0, 36)
    else:
        ccdlist = [args.ccd]

    exit_code = 0
    for expnum in args.expnum:
        for ccd in ccdlist:
            logging.info("Attempting to get status on header for {} {}".format(expnum, ccd))
            storage.set_logger(task,
                               prefix, expnum, ccd, args.type, args.dry_run)
            if storage.get_status(task, prefix, expnum, version=args.type, ccd=ccd) and not args.force:
                logging.info("{} completed successfully for {} {} {} {}".format(task, prefix, expnum, args.type, ccd))
                continue
            message = 'success'
            try:
                logging.info("Building a header ")
                if not storage.get_status(dependency, prefix, expnum, "p", 36) and not args.ignore_update_headers:
                    raise IOError("{} not yet run for {}".format(dependency, expnum))
                mk_mopheader(expnum, ccd, args.type, args.dry_run, prefix=prefix)
            except CalledProcessError as cpe:
                message = str(cpe.output)
                exit_code = message
            except Exception as e:
                message = str(e)
            logging.info(message)

            logging.error(message)
            if not args.dry_run:
                storage.set_status(task, prefix, expnum, version=args.type, ccd=ccd, status=message)
    return exit_code
Exemple #22
0
        ccdlist = range(0,36)
    else:
        ccdlist = [args.ccd]

    for expnum in args.expnum:
        for ccd in ccdlist:
            if storage.get_status(expnum, ccd, 'mkpsf') and not args.force:
                logging.info("Already did %s %s, skipping" %( str(expnum),
                                                                  str(ccd)))
                continue
            try:
                message = 'success'
                mkpsf(expnum, ccd)
                storage.set_status(expnum,
                                         ccd,
                                         'fwhm',
                                         str(storage.get_fwhm(
                    expnum, ccd)))
                storage.set_status(expnum,
                                         ccd,
                                         'zeropoint',
                                         str(storage.get_zeropoint(
                    expnum, ccd)))
                                         
            except Exception as e:
                message = str(e)

            logging.error(message)
            storage.set_status( expnum,
                                      ccd,
                                      'mkpsf',
Exemple #23
0
def run(expnum, ccd, prefix=None, version='p', field=None,
        measure3_dir=storage.MEASURE3, dry_run=False, force=False):

    message = storage.SUCCESS

    if storage.get_status(task, prefix, expnum, version, ccd) and not force:
        logging.info(
            "{} completed successfully for {} {} {} {}".format(task, prefix,
                                                               expnum, version,
                                                               ccd))
        return

    with storage.LoggingManager(task, prefix, expnum, ccd, version, dry_run):
        try:
            if not storage.get_status(dependency, prefix, expnum, version, ccd):
                raise IOError(35, "Cannot start {} as {} not yet completed "
                                  "for {}{}{}{:02d}".format(task, dependency,
                                                            prefix, expnum,
                                                            version, ccd))
            if field is None:
                field = str(expnum)

            if prefix is not None and len(prefix) > 0:
                field = "%s_%s" % (prefix, field)
            field += "_%s%s" % (str(version), str(ccd))

            logging.info("Doing combine on field {}".format(field))

            for ext in ['moving.matt', 'moving.jmp']:
                storage.get_file(expnum, ccd=ccd, version=version, ext=ext,
                                 prefix=prefix)

            # Get the list of objects planted into the field if prefix='fk'
            if prefix == 'fk':
                storage.get_file('Object',
                                 version='',
                                 ext='planted',
                                 subdir=str(expnum) + "/ccd%s" % (
                                     str(ccd).zfill(2)))
            else:
                prefix = ''

            cmd_args = ['comb-list',
                        prefix + str(expnum) + version + str(ccd).zfill(2)]
            logging.info(str(cmd_args))
            logging.info(util.exec_prog(cmd_args))

            # things to copy back to VOSpace, if this is an 'fk' image
            # then we have missed and found files too.
            ext_list = ['cands.comb']
            if prefix == 'fk':
                ext_list.extend(['jmp.missed', 'matt.missed',
                                 'jmp.found', 'matt.found',
                                 'comb.missed', 'comb.found'])

            for ext in ext_list:
                uri = storage.get_uri(expnum,
                                      ccd=ccd,
                                      prefix=prefix,
                                      version=version,
                                      ext=ext)
                filename = os.path.basename(uri)
                if not os.access(filename, os.R_OK):
                    logging.critical("No %s file" % filename)
                    continue
                vospace_name = "%s.%s" % (field, ext)
                if not dry_run:
                    logging.info("%s -> %s" % (
                        filename, os.path.join(measure3_dir, vospace_name)))
                    storage.copy(filename, os.path.join(measure3_dir, vospace_name))

            base_name = prefix + str(expnum) + version + str(ccd).zfill(2)
            cands_file = base_name + '.cands.comb'

            if not os.access(cands_file, os.R_OK):
                no_cands_file = (prefix +
                                 str(expnum) +
                                 version +
                                 str(ccd).zfill(2) +
                                 '.no_candidates')
                open(no_cands_file, 'w').close()
                if not dry_run:
                    vospace_name = "%s.no_candidates" % field
                    storage.copy(no_cands_file,
                                 os.path.join(measure3_dir, vospace_name))

            else:
                
                measure3.run(base_name, storage.DBIMAGES)

                if not dry_run:
                    filename = base_name + ".measure3.cands.astrom"
                    vospace_filename = "%s.measure3.cands.astrom" % field
                    storage.copy(filename, os.path.join(measure3_dir, vospace_filename))

        except Exception as ex:
            message = str(ex)
            logging.error(message)

        if not dry_run:
            storage.set_status(task, prefix, expnum, version, ccd,
                               status=message)
Exemple #24
0
def main():
    ### Must be running as a script

    parser = argparse.ArgumentParser(
        description='Run step2jmp and step2matt on a given exposure.')

    parser.add_argument("--ccd", "-c",
                        action="store",
                        default=None,
                        type=int,
                        dest="ccd")
    parser.add_argument("--fk", action="store_true", default=False, help="Do fakes?")
    parser.add_argument("--dbimages",
                        action="store",
                        default="vos:OSSOS/dbimages",
                        help='vospace dbimages containerNode')
    parser.add_argument("expnums",
                        type=int,
                        nargs=3,
                        help="3 expnums to process")
    parser.add_argument("--version",
                        action='version',
                        version='%(prog)s 1.0')
    parser.add_argument('-t', '--type',
                        help='which type of image to process',
                        choices=['s', 'p', 'o'],
                        default='p')
    parser.add_argument('--no-sort',
                        help='preserve input exposure order',
                        action='store_true')
    parser.add_argument("--verbose", "-v",
                        action="store_true")
    parser.add_argument("--default", default="WCS",
                        choices=['WCS', 'JMP'],
                        help="Which shift should be used if they dis-agree?")
    parser.add_argument("--debug",
                        action="store_true")
    parser.add_argument("--dry-run", action="store_true", help="run without pushing back to VOSpace, implies --force")
    parser.add_argument("--force", action="store_true")

    args = parser.parse_args()

    if args.verbose:
        logging.basicConfig(level=logging.INFO, format="%(message)s")
    if args.debug:
        logging.basicConfig(level=logging.DEBUG, format="%(message)s")

    storage.DBIMAGES = args.dbimages

    if args.ccd is None:
        ccd_list = range(0, 36)
    else:
        ccd_list = [args.ccd]

    prefix = (args.fk and "fk") or ""

    if not args.no_sort:
        args.expnums.sort()
    task = util.task()
    dependency = "step1"
    expnums = args.expnums
    version = args.type

    exit_status = 0

    for ccd in ccd_list:
        storage.set_logger(os.path.splitext(os.path.basename(sys.argv[0]))[0],
                           prefix, args.expnums[0], ccd, args.type, args.dry_run)
        try:
            message = storage.SUCCESS

            # Check if dependent task has finished for each of the input exposure ids.
            for expnum in expnums:
                if not storage.get_status(dependency, prefix, expnum, version=version, ccd=ccd):
                    raise IOError(35, "Cannot start {} as {} not yet completed for {}{}{}{:02d}".format(
                        task, dependency, prefix, expnum, args.type, ccd))

            # Check if we need to run this step on the current frame.
            if not (args.force or args.dry_run) and storage.get_status(task, prefix, args.expnums[0], version=version,
                                                                       ccd=ccd):
                logging.info("{} completed successfully for {}{}{}{:02d}".format(
                    task, prefix, expnums[0], version, ccd))
                continue
            logging.info("Executing {} on {} {} {} {}".format(
                        task, prefix, args.expnums, args.type, ccd))
            step2(args.expnums, ccd, version=version, prefix=prefix, dry_run=args.dry_run, default=args.default)
            logging.info(message)
        except CalledProcessError as cpe:
            message = str(cpe)
            logging.error(message)
            exit_status = message
        except Exception as e:
            message = str(e)
            logging.error(message)
            exit_status = message
        if not args.dry_run:
            storage.set_status(task, prefix, args.expnums[0], version=version, ccd=ccd, status=message)
    return exit_status
Exemple #25
0
        ccdlist = [args.ccd]

    prefix = ( args.fk and 'fk') or ''


    for expnum in args.expnum:
        for ccd in ccdlist:
            try:
                message = storage.SUCCESS
                if not storage.get_status(expnum, ccd, 'mkpsf', version=args.type):
                    raise IOError(35, "mkpsf hasn't run?")
                if storage.get_status(expnum, ccd, prefix+'step1', version=args.type) and not args.force:
                    logging.critical(
                        "Step1 completed for %s%s, skipping" %(prefix+str(expnum),
                                                        str(args.type)+str(ccd)))
                    continue
                logging.info("step1_p on expnum: %s, ccd: %s" % (
                    prefix+str(expnum), str(ccd)))
                step1(expnum, ccd, prefix=prefix, version=args.type)
            except Exception as e:
                message = str(e)
                logging.error("Error running step1_p: %s " %(message))

            storage.set_status(expnum,
                               ccd,
                               prefix+'step1',
                               version=args.type,
                               status=message)
        
            
Exemple #26
0
def main():
    """Do the script."""
    parser = argparse.ArgumentParser(
        description='replace image header')
    parser.add_argument('--extname',
                        help='name of extension to in header')
    parser.add_argument('expnum', type=str,
                        help='exposure to update')
    parser.add_argument('-r', '--replace',
                        action='store_true',
                        help='store modified image back to VOSpace?')
    parser.add_argument('-v', '--verbose', action='store_true')
    parser.add_argument('--debug', action='store_true')
    parser.add_argument('--force', action='store_true', help="Re-run even if previous success recorded")
    parser.add_argument('--dbimages', help="VOSpace DATA storage area.", default="vos:OSSOS/dbimages")

    args = parser.parse_args()
    task = util.task()
    dependency = 'preproc'
    prefix = ""

    storage.DBIMAGES = args.dbimages

    level = logging.CRITICAL
    message_format = "%(message)s"
    if args.verbose:
        level = logging.INFO
    if args.debug:
        level = logging.DEBUG
        message_format = "%(module)s %(funcName)s %(lineno)s %(message)s"
    logging.basicConfig(level=level, format=message_format)
    storage.set_logger(task, prefix, args.expnum, None, None, False)

    message = storage.SUCCESS
    expnum = args.expnum

    exit_status = 0
    try:
        # skip if already succeeded and not in force mode
        if storage.get_status(task, prefix, expnum, "p", 36) and not args.force:
            logging.info("Already updated, skipping")
            sys.exit(0)
    
        image_hdulist = storage.get_image(args.expnum, return_file=False)
        ast_hdulist = storage.get_astheader(expnum, ccd=None)

        run_update_header(image_hdulist, ast_hdulist)
        image_filename = os.path.basename(storage.get_uri(expnum))
        image_hdulist.writeto(image_filename)
        if args.replace:
            dest = storage.dbimages_uri(expnum)
            storage.copy(image_filename, dest)
            storage.set_status('update_header', "", expnum, 'p', 36, message)
    except Exception as e:
        message = str(e)
        if args.replace:
            storage.set_status(task, prefix, expnum, 'p', 36, message)
        exit_status = message
        logging.error(message)

    return exit_status
Exemple #27
0
def run(expnum, ccd, version, dry_run=False, prefix="", force=False):
    """Run the OSSOS jmpmakepsf script.

    """

    message = storage.SUCCESS
    if storage.get_status(task, prefix, expnum, version=version, ccd=ccd) and not force:
        logging.info("{} completed successfully for {} {} {} {}".format(task, prefix, expnum, version, ccd))
        return

    with storage.LoggingManager(task, prefix, expnum, ccd, version, dry_run):
        try:
            if not storage.get_status(dependency, prefix, expnum, "p", ccd=ccd):
                raise IOError("{} not yet run for {}".format(dependency, expnum))

            # confirm destination directory exists.
            destdir = os.path.dirname(
                storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext='fits'))
            if not dry_run:
                storage.mkdir(destdir)

            # get image from the vospace storage area
            logging.info("Getting fits image from VOSpace")
            filename = storage.get_image(expnum, ccd, version=version, prefix=prefix)

            # get mopheader from the vospace storage area
            logging.info("Getting mopheader from VOSpace")
            mopheader_filename = storage.get_file(expnum, ccd, version=version, prefix=prefix, ext='mopheader')


            # run mkpsf process
            logging.info("Running mkpsf on %s %d" % (expnum, ccd))
            logging.info(util.exec_prog(['jmpmakepsf.csh',
                                         './',
                                         filename,
                                         'yes', 'yes']))
            
            if dry_run:
                return

            # place the results into VOSpace
            basename = os.path.splitext(filename)[0]

            for ext in ('mopheader', 'psf.fits',
                        'zeropoint.used', 'apcor', 'fwhm', 'phot'):
                dest = storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext=ext)
                source = basename + "." + str(ext)
                count = 0
                with open(source, 'r'):
                  while True:
                    count += 1
                    try:
                        logging.info("Attempt {} to copy {} -> {}".format(count, source, dest))
                        storage.copy(source, dest)
                        break
                    except Exception as ex:
                        if count > 10:
                            raise ex

            # set some data parameters associated with the image, determined in this step.
            storage.set_status('fwhm', prefix, expnum, version=version, ccd=ccd, status=str(storage.get_fwhm(
                expnum, ccd=ccd, prefix=prefix, version=version)))
            storage.set_status('zeropoint', prefix, expnum, version=version, ccd=ccd,
                               status=str(storage.get_zeropoint(
                                   expnum, ccd=ccd, prefix=prefix, version=version)))
            logging.info(message)
        except Exception as e:
            message = str(e)
            logging.error(message)
            
        storage.set_status(task, prefix, expnum, version, ccd=ccd, status=message)

    return
Exemple #28
0
def main():
    parser = argparse.ArgumentParser(
        description='Run step1jmp and step1matt on a given exposure.')

    parser.add_argument("--ccd", "-c",
                        action="store",
                        default=None,
                        type=int,
                        dest="ccd")
    parser.add_argument("--fk", help="add the fk prefix on processing?",
                        default=False,
                        action='store_true')
    parser.add_argument("--dbimages",
                        action="store",
                        default="vos:OSSOS/dbimages",
                        help='vospace dbimages containerNode')
    parser.add_argument("--field",
                        action="store",
                        default=None,
                        help="Name of the field being combined")
    parser.add_argument("expnum",
                        type=int,
                        help="expnum to process")
    parser.add_argument("--measure3",
                        action="store",
                        help="VOSpace location for measure3 files",
                        default=MEASURE3)
    parser.add_argument('--type', default='p', choices=['o', 'p', 's'], help="which type of image")
    parser.add_argument("--verbose", "-v",
                        action="store_true")
    parser.add_argument("--debug", '-d',
                        action='store_true')
    parser.add_argument("--force", '-f', action='store_true')
    parser.add_argument("--dry-run", action="store_true", help="Don't push back to VOSpace, implies --force")

    args = parser.parse_args()

    level = logging.CRITICAL
    if args.debug:
        level = logging.DEBUG
    elif args.verbose:
        level = logging.INFO

    logging.basicConfig(level=level, format="%(message)s")

    storage.DBIMAGES = args.dbimages
    storage.MEASURE3 = args.measure3

    prefix = (args.fk and 'fk') or ''
    task = util.task()
    dependency = 'step3'

    ccd_list = (args.ccd is None and range(0, 36)) or [args.ccd]

    exit_code = 0
    for ccd in ccd_list:
        storage.set_logger(task, prefix, args.expnum, ccd, args.type, args.dry_run)
        try:
            if not storage.get_status(dependency, prefix, args.expnum, version=args.type, ccd=ccd) and not args.dry_run:
               message = storage.get_status(dependency, prefix, args.expnum, "p", ccd, return_message=True)
               raise IOError(35, message)

            if storage.get_status(task, prefix, args.expnum, version=args.type, ccd=ccd) and not args.force:
               continue
            message = combine(args.expnum,
                              ccd,
                              prefix=prefix,
                              file_type=args.type,
                              field=args.field,
                              measure3=args.measure3,
                              dry_run=args.dry_run)
        except CalledProcessError as cpe:
            message = str(cpe)
            exit_code = message
        except Exception as e:
            message = str(e)
            exit_code = message
        logging.info(message)
        if not args.dry_run:
            storage.set_status(task, prefix, args.expnum, version=args.type, ccd=ccd, status=message)
        return exit_code
Exemple #29
0
def main():
    parser = argparse.ArgumentParser(
        description='Run step1jmp and step1matt on a given exposure.')

    parser.add_argument("--ccd",
                        "-c",
                        action="store",
                        default=None,
                        type=int,
                        dest="ccd")
    parser.add_argument("--fk",
                        help="add the fk prefix on processing?",
                        default=False,
                        action='store_true')
    parser.add_argument("--dbimages",
                        action="store",
                        default="vos:OSSOS/dbimages",
                        help='vospace dbimages containerNode')
    parser.add_argument("--field",
                        action="store",
                        default=None,
                        help="Name of the field being combined")
    parser.add_argument("expnum", type=int, help="expnum to process")
    parser.add_argument("--measure3",
                        action="store",
                        help="VOSpace location for measure3 files",
                        default=MEASURE3)
    parser.add_argument('--type',
                        default='p',
                        choices=['o', 'p', 's'],
                        help="which type of image")
    parser.add_argument("--verbose", "-v", action="store_true")
    parser.add_argument("--debug", '-d', action='store_true')
    parser.add_argument("--force", '-f', action='store_true')
    parser.add_argument("--dry_run",
                        action="store_true",
                        help="Don't push back to VOSpace, implies --force")

    args = parser.parse_args()
    if args.dry_run:
        args.force = True

    level = logging.CRITICAL
    if args.debug:
        level = logging.DEBUG
    elif args.verbose:
        level = logging.INFO

    logging.basicConfig(level=level, format="%(message)s")

    storage.DBIMAGES = args.dbimages
    storage.MEASURE3 = args.measure3

    prefix = (args.fk and 'fk') or ''

    ccd_list = (args.ccd is None and range(0, 36)) or [args.ccd]

    exit_code = 0
    for ccd in ccd_list:
        storage.set_logger(
            os.path.splitext(os.path.basename(sys.argv[0]))[0], prefix,
            args.expnum, ccd, args.type, args.dry_run)
        if not storage.get_status(
                args.expnum, ccd, prefix + 'step3',
                version=args.type) and not args.dry_run:
            logging.error(
                storage.get_status(args.expnum,
                                   ccd,
                                   'step3',
                                   return_message=True))
            raise IOError(35, "need to run step3 first")

        if storage.get_status(
                args.expnum, ccd, prefix + 'combine',
                version=args.type) and not args.force:
            continue
        try:
            message = combine(args.expnum,
                              ccd,
                              prefix=prefix,
                              file_type=args.type,
                              field=args.field,
                              measure3=args.measure3,
                              dry_run=args.dry_run)
        except CalledProcessError as cpe:
            message = str(cpe)
            exit_code = message
        except Exception as e:
            message = str(e)
            exit_code = message
        logging.info(message)
        if not args.dry_run:
            storage.set_status(args.expnum,
                               ccd,
                               prefix + 'combine',
                               version=args.type,
                               status=message)
        return exit_code
Exemple #30
0
def main(argv):
    """

    @param argv: an array of arguments to be parsed, normally sys.argv
    @return: None
    """
    parser = argparse.ArgumentParser()
    parser.add_argument('--ccd',
                        action='store',
                        type=int,
                        default=None,
                        help='which ccd to process')
    parser.add_argument("--dbimages",
                        action="store",
                        default="vos:OSSOS/dbimages",
                        help='vospace dbimages containerNode')
    parser.add_argument("expnums",
                        type=int,
                        nargs=3,
                        help="expnum(s) to process")
    parser.add_argument("--type",
                        action='store',
                        default='s',
                        choices=['s', 'p', 'o'],
                        help='which type of image')
    parser.add_argument('--no-sort',
                        action='store_true',
                        default=False,
                        help='do not sort exposure list by expnum before processing')
    parser.add_argument("--verbose", "-v",
                        action="store_true")
    parser.add_argument("--debug", '-d',
                        action='store_true')
    parser.add_argument("--number", "-n", type=int, help="Number of KBOs to plant into images.", default=10)
    parser.add_argument("--mmin", type=float, help="Minimum magnitude value to add source with.", default=21.0)
    parser.add_argument("--mmax", type=float, help="Maximum magnitude value to add source with.", default=25.5)
    parser.add_argument("--rmin", default=0.5,
                        type=float, help="minimum motion rate")
    parser.add_argument("--rmax", default=15,
                        type=float, help="maximum motion rate")
    parser.add_argument("--width", default=30,
                        type=float, help="angle opening")
    parser.add_argument("--ang", default=20,
                        type=float, help="angle of motion, 0 is West")
    parser.add_argument("--force", action="store_true")
    parser.add_argument("--dry-run", action="store_true")
    args = parser.parse_args(argv)

    # # setup logging
    level = logging.CRITICAL
    if args.debug:
        level = logging.DEBUG
    elif args.verbose:
        level = logging.INFO

    if not args.no_sort:
        args.expnums.sort()

    storage.DBIMAGES = args.dbimages

    logging.basicConfig(level=level, format="%(message)s")
    task = util.task()
    prefix = ""
    dependency = 'mkpsf'

    ccds = [args.ccd]
    if args.ccd is None:
        ccds = range(0, 36)

    exit_code = 0
    for ccd in ccds:
        message = storage.SUCCESS
        try:
            storage.set_logger(task, prefix, args.expnums[0], ccd, args.type, args.dry_run)
            if not storage.get_status(dependency, prefix, args.expnums[0], version='s', ccd=ccd):
                raise IOError("{} not yet run for {} {} {}".format(dependency, args.expnums, args.type, ccd))
            if storage.get_status(task, prefix, args.expnums[0], version=args.type, ccd=ccd) and not args.force:
                logging.info("{} previously completed successfully for {}{}{:02d}".format(task,
                                                                                          args.expnums[0],
                                                                                          args.type,
                                                                                          ccd))
                continue
            plant(args.expnums,
                  ccd,
                  args.rmin, args.rmax, args.ang, args.width,
                  number=args.number, mmin=args.mmin, mmax=args.mmax,
                  version=args.type,
                  dry_run=args.dry_run)
        except CalledProcessError as cpe:
            message = str(cpe)
            exit_code = message
        except Exception as e:
            message = str(e)
        logging.critical(message)

        if not args.dry_run:
            storage.set_status(task, prefix, args.expnums[0], version=args.type, ccd=ccd, status=message)
    return exit_code
Exemple #31
0
    dependency = "step1"

    logging.basicConfig(level=level, format="%(message)s")

    ccds = [args.ccd]
    if args.ccd is None:
        ccds = range(0, 36)

    for ccd in ccds:
        message = storage.SUCCESS
        try:
            storage.set_logger(task, prefix, args.expnums[0], ccd, args.type, args.dry_run)
            if not storage.get_status(dependency, prefix, args.expnums[0], version=args.type, ccd=ccd):
                raise IOError("mkpsf not yet run for %s ccd%s, can't run align" % (
                    str(args.expnums), str(ccd).zfill(2)))
            if storage.get_status(task, prefix, args.expnums[0], version=args.type, ccd=ccd) and not args.force:
                logging.info("align done for %s[%s]" % (args.expnums[0], ccd))
                continue
            align(args.expnums, ccd, version=args.type, dry_run=args.dry_run)
        except CalledProcessError as cpe:
            message = str(cpe)
            exit_code = message
	    logging.error("ERROR: {}".format(str(cpe)))
        except Exception as e:
	    logging.error("Exception: {}".format(str(e)))
            message = str(e)
        logging.critical(message)

        if not args.dry_run:
            storage.set_status(task, prefix, args.expnums[0], version=args.type, ccd=ccd, status=message)
Exemple #32
0
    if args.ccd is None:
        ccdlist = range(0, 36)
    else:
        ccdlist = [args.ccd]

    prefix = (args.fk and "fk") or ""

    if not args.no_sort:
        args.expnums.sort()

    for ccd in ccdlist:
        try:
            message = storage.SUCCESS
            for expnum in args.expnums:
                if not storage.get_status(expnum, ccd, prefix + 'step1'):
                    raise IOError(35, "missing step1 for %s" % (expnum))
            if storage.get_status(args.expnums[0], ccd,
                                  prefix + 'step2') and not args.force:
                logging.info("Already did %s %s, skipping" %
                             (str(expnum), str(ccd)))
                continue
            logging.info("step2 on expnums :%s, ccd: %d" %
                         (str(args.expnums), ccd))
            step2(args.expnums, ccd, version=args.type, prefix=prefix)

        except Exception as e:
            message = str(e)
        logging.error(message)
        storage.set_status(args.expnums[0], ccd, prefix + 'step2', message)
Exemple #33
0
    for ccd in ccds:
        message = storage.SUCCESS
        try:
            storage.set_logger(
                os.path.splitext(os.path.basename(sys.argv[0]))[0], "",
                args.expnums[0], ccd, args.type, args.dry_run)
            if not storage.get_status(
                    args.expnums[0], ccd, 'step2', version=args.type):
                raise IOError("scramble not yet run for %s ccd%s" %
                              (str(args.expnums), str(ccd).zfill(2)))
            if storage.get_status(
                    args.expnums[0], ccd, 'align',
                    version=args.type) and not args.force:
                logging.info("align done for %s[%s]" % (args.expnums[0], ccd))
                continue
            align(args.expnums, ccd, version=args.type, dry_run=args.dry_run)
        except CalledProcessError as cpe:
            message = str(cpe)
            exit_code = message
        except Exception as e:
            message = str(e)
        logging.critical(message)

        if not args.dry_run:
            storage.set_status(args.expnums[0],
                               ccd,
                               'align',
                               version=args.type,
                               status=message)
Exemple #34
0
def scramble(expnums, ccd, version='p', dry_run=False, force=False, prefix=''):
    """
    run the plant script on this combination of exposures

    @param expnums: list of exposure numbers to scramble the time on
    @param ccd:  which CCD in (assumes this is a CFHT MegaCam MEF)
    @param version: should we scramble the 'p' or 'o' images?
    @param dry_run: if dry run then don't save back to VOSpace.
    @param force: if true then create scramble set, even if already exists.
    @param prefix: a string that will be pre-pended to the EXPNUM to get the filename, sometimes 'fk'.
    @return: None
    """

    # Get a list of the MJD values and then write a re-ordering of those into files with 's'
    # as their type instead of 'p' or 'o'
    mjds = []
    fobjs = []
    message = storage.SUCCESS
    if not (force or dry_run) and storage.get_status(
            task, prefix, expnums[0], version='s', ccd=ccd):
        logging.info("{} recorded as complete for {} ccd {}".format(
            task, expnums, ccd))
        return

    with storage.LoggingManager(task, prefix, expnums[0], ccd, version):
        try:
            for expnum in expnums:
                filename = storage.get_image(expnum, ccd=ccd, version=version)
                fobjs.append(fits.open(filename))
                # Pull out values to replace in headers.. must pull them
                # as otherwise we get pointers...
                mjds.append(fobjs[-1][0].header['MJD-OBS'])

            order = [0, 2, 1]
            for idx in range(len(fobjs)):
                logging.info(
                    "Flipping %d to %d" %
                    (fobjs[idx][0].header['EXPNUM'], expnums[order[idx]]))
                fobjs[idx][0].header['EXPNUM'] = expnums[order[idx]]
                fobjs[idx][0].header['MJD-OBS'] = mjds[order[idx]]
                uri = storage.get_uri(expnums[order[idx]],
                                      ccd=ccd,
                                      version='s',
                                      ext='fits')
                scramble_file_name = os.path.basename(uri)
                if os.access(scramble_file_name, os.F_OK):
                    os.unlink(scramble_file_name)
                fobjs[idx].writeto(scramble_file_name)
                if not dry_run:
                    storage.copy(scramble_file_name, uri)
            logging.info(message)
        except Exception as ex:
            message = str(ex)
            logging.error(message)

        if not dry_run:
            storage.set_status(task,
                               prefix,
                               expnum,
                               version,
                               ccd,
                               status=message)

    return
Exemple #35
0
        ccdlist = range(0,36)
    else:
        ccdlist = [args.ccd]

    for expnum in args.expnum:
        for ccd in ccdlist:
            if storage.get_status(expnum, ccd, 'mkpsf', version=args.type) and not args.force:
                logging.info("Already did %s %s, skipping" %( str(expnum),
                                                                  str(ccd)))
                continue
            try:
                message = 'success'
                mkpsf(expnum, ccd, args.type)
                storage.set_status(expnum,
                                   ccd,
                                   'fwhm',
                                   version=args.type,
                                   status=str(storage.get_fwhm(
                    expnum, ccd, version=args.type)))
                storage.set_status(expnum,
                                   ccd,
                                   'zeropoint',
                                   version=args.type,
                                   status=str(storage.get_zeropoint(
                    expnum, ccd, version=args.type)))
                                         
            except Exception as e:
                message = str(e)

            logging.error(message)
            storage.set_status( expnum,
                                ccd,
Exemple #36
0
    if args.debug:
        level = logging.DEBUG
    elif args.verbose:
        level = logging.INFO

    if not args.no_sort:
        args.expnums.sort()

    storage.DBIMAGES = args.dbimages

    logging.basicConfig(level=level, format="%(message)s")

    ccds = [args.ccd]
    if args.ccd is None:
        ccds = range(0, 36)

    for ccd in ccds:
        message = storage.SUCCESS
        try:
            if not storage.get_status(args.expnums[0], ccd, "scramble", version=args.type):
                raise IOError("scramble not yet run for %s ccd%s" % (str(args.expnums), str(ccd).zfill(2)))
            if storage.get_status(args.expnums[0], ccd, "plant", version=args.type) and not args.force:
                logging.info("plant done for %s[%s]" % (args.expnums[0], ccd))
                continue
            plant(args.expnums, ccd, args.rmin, args.rmax, args.ang, args.width, version=args.type)
        except Exception as e:
            message = str(e)
            logging.error(str(e))

        storage.set_status(args.expnums[0], ccd, "plant", version=args.type, status=message)
Exemple #37
0
def run(expnums,
        ccd,
        version,
        prefix=None,
        dry_run=False,
        default="WCS",
        force=False):
    """run the actual step2  on the given exp/ccd combo"""

    jmp_trans = ['step2ajmp']
    jmp_args = ['step2bjmp']
    matt_args = ['step2matt_jmp']

    if storage.get_status(task, prefix, expnums[0], version,
                          ccd) and not force:
        logging.info("{} completed successfully for {}{}{}{:02d}".format(
            task, prefix, expnums[0], version, ccd))
        return

    with storage.LoggingManager(task, prefix, expnums[0], ccd, version,
                                dry_run):
        try:
            for expnum in expnums:
                if not storage.get_status(
                        dependency, prefix, expnum, version=version, ccd=ccd):
                    raise IOError(
                        35,
                        "Cannot start {} as {} not yet completed for {}{}{}{:02d}"
                        .format(task, dependency, prefix, expnum, version,
                                ccd))
            message = storage.SUCCESS

            idx = 0
            logging.info("Retrieving catalog files to do matching.")
            for expnum in expnums:
                jmp_args.append(
                    storage.get_file(expnum,
                                     ccd=ccd,
                                     version=version,
                                     ext='obj.jmp',
                                     prefix=prefix)[0:-8])
                jmp_trans.append(
                    storage.get_file(expnum,
                                     ccd=ccd,
                                     version=version,
                                     ext='obj.jmp',
                                     prefix=prefix)[0:-8])
                idx += 1
                matt_args.append('-f%d' % idx)
                matt_args.append(
                    storage.get_file(expnum,
                                     ccd=ccd,
                                     version=version,
                                     ext='obj.matt',
                                     prefix=prefix)[0:-9])

            logging.info(
                "Computing the catalog alignment using sources in catalogs.")
            try:
                logging.info(util.exec_prog(jmp_trans))
                if default == "WCS":
                    logging.info("Comparing computed transform to WCS values")
                    logging.info(
                        compute_trans(expnums,
                                      ccd,
                                      version,
                                      prefix,
                                      default=default))
            except Exception as ex:
                logging.info("JMP Trans failed: {}".format(ex))
                logging.info(
                    compute_trans(expnums, ccd, version, prefix,
                                  default="WCS"))

            logging.info("Using transform to match catalogs for three images.")
            logging.info(util.exec_prog(jmp_args))
            logging.info(util.exec_prog(matt_args))

            # check that the shifts from step2 are rational by matching the bright star lists.
            logging.info(
                "Uisng checktrans to ensure that transforms were reasonable.")
            check_args = ['checktrans']
            if os.access('proc-these-files', os.R_OK):
                os.unlink('proc-these-files')
            ptf = open('proc-these-files', 'w')
            ptf.write(
                "# A dummy file that is created so checktrans could run.\n")
            ptf.write("# Frame FWHM PSF?\n")
            for expnum in expnums:
                filename = os.path.splitext(
                    storage.get_image(expnum,
                                      ccd,
                                      version=version,
                                      prefix=prefix))[0]
                if not os.access(filename + ".bright.psf", os.R_OK):
                    os.link(filename + ".bright.jmp", filename + ".bright.psf")
                if not os.access(filename + ".obj.psf", os.R_OK):
                    os.link(filename + ".obj.jmp", filename + ".obj.psf")
                ptf.write("{:>19s}{:>10.1f}{:>5s}\n".format(
                    filename, _FWHM, "NO"))
            ptf.close()
            if os.access('BAD_TRANS', os.F_OK):
                os.unlink('BAD_TRANS')

            logging.info(util.exec_prog(check_args))

            if os.access('BAD_TRANS', os.F_OK):
                raise OSError(errno.EBADMSG, 'BAD_TRANS')

            if os.access('proc-these-files', os.F_OK):
                os.unlink('proc-these-files')

            if dry_run:
                return

            for expnum in expnums:
                for ext in ['unid.jmp', 'unid.matt', 'trans.jmp']:
                    uri = storage.dbimages_uri(expnum,
                                               ccd=ccd,
                                               version=version,
                                               ext=ext,
                                               prefix=prefix)
                    filename = os.path.basename(uri)
                    storage.copy(filename, uri)

        except Exception as ex:
            message = str(ex)
            logging.error(message)

        storage.set_status(task,
                           prefix,
                           expnums[0],
                           version,
                           ccd,
                           status=message)

    return
Exemple #38
0
    logging.basicConfig(level=level, format="%(message)s")

    ccds = [args.ccd]
    if args.ccd is None:
        ccds = range(0,36)
    
    for ccd in ccds:
        message = storage.SUCCESS
        try:
            if not storage.get_status(args.expnums[0], ccd,
                                      'scramble'):
                raise IOError("scramble not yet run for %s ccd%s" % ( 
                    str(args.expnums), str(ccd).zfill(2)))
            if storage.get_status(args.expnums[0], ccd,
                                  'plant') and not args.force:
                logging.info("plant done for %s[%s]" % ( args.expnums[0], ccd))
                continue
            plant(args.expnums,
                  ccd,
                  args.rmin, args.rmax, args.ang, args.width,
                  version=args.type)
        except Exception as e:
            message = str(e)
            logging.error(str(e))

        storage.set_status(args.expnums[0],
                           ccd,
                           'plant',
                           message)
Exemple #39
0
def run(expnum, ccd, version='p', prefix='', dry_run=False, force=False):

    message = 'success'

    if storage.get_status(task, prefix, expnum, version=version, ccd=ccd) and not force:
        logging.info("{} completed successfully for {} {} {} {}".format(task, prefix, expnum, version, ccd))
        return

    with storage.LoggingManager(task=task, prefix=prefix, expnum=expnum, ccd=ccd, version=version, dry_run=dry_run):
        try:
            if not storage.get_status(dependency, prefix, expnum, "p", ccd=ccd):
                raise IOError("{} not yet run for {}".format(dependency, expnum))


            header = storage.get_astheader(expnum, ccd)
            datasec = storage.datasec_to_list(header.get('DATASEC', '[80:2080,30,4160]'))
            try:
                fwhm = "{:5.2f}".format(storage.get_fwhm(expnum, ccd))
            except:
                fwhm = 'unknown'
            for keyword in del_keyword_list:
                try:
                    del(header[keyword])
                except:
                    pass
            header['FWHM'] = (fwhm, 'FWHM in pixels')
            header['EXTNAME'] = 'header'
            primary_hdu = fits.PrimaryHDU(header=header)
            hdu_list = fits.HDUList([primary_hdu, ])
            for ext in ['jmp', 'matt']:
                extension = 'obj.'+ext
                name = "{}p{:02d}.{}".format(expnum, ccd, extension)
                try:
                    os.unlink(name)
                    os.unlink(name+".fits")
                except:
                    pass
                logging.info("Retrieving {}".format(name))
                obj_file = mop_file.Parser(expnum, ccd, extension)
                obj_file.parse()
        
                t = numpy.all([datasec[0] < obj_file.data['X'], obj_file.data['X'] < datasec[1],
                               datasec[2] < obj_file.data['Y'], obj_file.data['Y'] < datasec[3]], axis=0)
                logging.info("Source remaining after datasec cut: {} of {}".format(len(obj_file.data[t]['X']), len(t)))
                table_hdu = fits.table_to_hdu(obj_file.data[t])
                table_hdu.header['CATALOG'] = name
                table_hdu.header['EXTNAME'] = ext
                hdu_list.append(table_hdu)
                del table_hdu
                del obj_file
                os.unlink(name)

            name = "{}p{:02d}.{}".format(expnum, ccd, 'obj.fits')
            if os.access(name, os.F_OK):
                os.unlink(name)
            hdu_list.writeto(name)
            uri = storage.dbimages_uri(expnum, ccd, 'p', ext=".obj.fits")
            logging.info(name+" -> "+uri)
            count = 0
            with open(name):
              while True:
                count += 1
                logging.info("Copy attempt {}".format(count))
                try:
                    storage.copy(name, uri)
                    os.unlink(name)
                    break
                except Exception as ex:
                    if count > 10:
                        raise ex

            logging.info(message)
        except Exception as e:
            message = str(e)
            logging.error(message)
        
        if not dry_run:
            storage.set_status(task, prefix, expnum, version=version, ccd=ccd, status=message)
Exemple #40
0
def run(expnum,
        ccd,
        prefix='',
        version='p',
        sex_thresh=_SEX_THRESHOLD,
        wave_thresh=_WAVE_THRESHOLD,
        maxcount=_MAX_COUNT,
        dry_run=False,
        force=True, ignore=False):
    """run the actual step1jmp/matt codes.

    expnum: the CFHT expousre to process
    ccd: which ccd in the mosaic to process
    fwhm: the image quality, FWHM, of the image.  In pixels.
    sex_thresh: the detection threhold to run sExtractor at
    wave_thresh: the detection threshold for wavelet
    maxcount: saturation level

    """
    message = storage.SUCCESS

    if storage.get_status(task, prefix, expnum, version, ccd) and not force:
        logging.info("{} completed successfully for {} {} {} {}".format(task, prefix, expnum, version, ccd))
        return

    with storage.LoggingManager(task, prefix, expnum, ccd, version, dry_run):
        try:
            if not storage.get_status(dependency, prefix, expnum, version, ccd) and not ignore:
                raise IOError(35, "Cannot start {} as {} not yet completed for {}{}{}{:02d}".format(
                    task, dependency, prefix, expnum, version, ccd))
            logging.info("Retrieving imaging and input parameters from VOSpace")
            storage.get_file(expnum, ccd, prefix=prefix, version=version, ext='mopheader')
            filename = storage.get_image(expnum, ccd, version=version, prefix=prefix)
            fwhm = storage.get_fwhm(expnum, ccd, prefix=prefix, version=version, default=3.5)
            basename = os.path.splitext(filename)[0]

            _get_weight_map(filename, ccd)

            logging.info("Launching step1jmp")
            logging.info(util.exec_prog(['step1jmp',
                                         '-f', basename,
                                         '-t', str(wave_thresh),
                                         '-w', str(fwhm),
                                         '-m', str(maxcount)]))

            logging.info(util.exec_prog(['step1matt',
                                         '-f', basename,
                                         '-t', str(sex_thresh),
                                         '-w', str(fwhm),
                                         '-m', str(maxcount)]))

            if os.access('weight.fits', os.R_OK):
                os.unlink('weight.fits')

            if not dry_run:
                for ext in ['obj.jmp', 'obj.matt']:
                    obj_uri = storage.get_uri(expnum, ccd, version=version, ext=ext,
                                              prefix=prefix)
                    obj_filename = basename + "." + ext
                    count = 0
                    with open(obj_filename, 'r'):
                        while True:
                            try:
                                count += 1
                                logging.info("Attempt {} to copy {} -> {}".format(count, obj_filename, obj_uri))
                                storage.copy(obj_filename, obj_uri)
                                break
                            except Exception as ex:
                                if count > 10:
                                    raise ex
            logging.info(message)
        except Exception as ex:
            message = str(ex)
            logging.error(message)

        if not dry_run:
            storage.set_status(task, prefix, expnum, version, ccd, status=message)
Exemple #41
0
def plant(expnums,
          ccd,
          rmin,
          rmax,
          ang,
          width,
          number=10,
          mmin=21.0,
          mmax=25.5,
          version='s',
          dry_run=False,
          force=True):
    """Plant artificial sources into the list of images provided.

    @param dry_run: don't push results to VOSpace.
    @param width: The +/- range of angles of motion
    @param ang: The mean angle of motion to add sources
    @param rmax: The maximum rate of motion to add sources at (''/hour)
    @param rmin: The minimum rate of motion to add sources at (''/hour)
    @param expnums: list of MegaPrime exposure numbers to add artificial KBOs to
    @param ccd: which ccd to work on.
    @param mmax: Maximum magnitude to plant sources at
    @param version: Add sources to the 'o', 'p' or 's' images
    @param mmin: Minimum magnitude to plant sources at
    @param number: number of sources to plant.
    @param force: Run, even if we already succeeded at making a fk image.
    """
    message = storage.SUCCESS

    if storage.get_status(task, "", expnums[0], version, ccd) and not force:
        logging.info("{} completed successfully for {}{}{}{:02d}".format(
            task, "", expnums[0], version, ccd))
        return

    with storage.LoggingManager(task, "", expnums[0], ccd, version, dry_run):
        try:
            # Construct a list of artificial KBOs with positions in the image
            # and rates of motion within the bounds given by the caller.
            filename = storage.get_image(expnums[0], ccd=ccd, version=version)
            header = fits.open(filename)[0].header
            bounds = util.get_pixel_bounds_from_datasec_keyword(
                header.get('DATASEC', '[33:2080,1:4612]'))

            # generate a set of artificial KBOs to add to the image.
            kbos = KBOGenerator.get_kbos(n=number,
                                         rate=(rmin, rmax),
                                         angle=(ang - width, ang + width),
                                         mag=(mmin, mmax),
                                         x=(bounds[0][0], bounds[0][1]),
                                         y=(bounds[1][0], bounds[1][1]),
                                         filename='Object.planted')

            for expnum in expnums:
                filename = storage.get_image(expnum, ccd, version)
                psf = storage.get_file(expnum, ccd, version, ext='psf.fits')
                plant_kbos(filename, psf, kbos,
                           get_shifts(expnum, ccd, version), "fk")

            if dry_run:
                return
            uri = storage.get_uri('Object',
                                  ext='planted',
                                  version='',
                                  subdir=f"{expnums[0]}/ccd{int(ccd):02d}")

            storage.copy('Object.planted', uri)
            for expnum in expnums:
                uri = storage.get_uri(expnum,
                                      ccd=ccd,
                                      version=version,
                                      ext='fits',
                                      prefix='fk')
                filename = os.path.basename(uri)
                storage.copy(filename, uri)
        except Exception as ex:
            message = str(ex)
            logging.error(message)

        storage.set_status(task, "", expnums[0], version, ccd, status=message)

    return
Exemple #42
0
    else:
        ccdlist = [args.ccd]

    prefix = ( args.fk and 'fk') or ''


    for expnum in args.expnum:
        for ccd in ccdlist:
            try:
                message = storage.SUCCESS
                if not storage.get_status(expnum, ccd, 'mkpsf'):
                    raise IOError(35, "mkpsf hasn't run?")
                if storage.get_status(expnum, ccd, prefix+'step1') and not args.force:
                    logging.critical(
                        "Already did %s %s, skipping" %(prefix+str(expnum),
                                                        str(ccd)))
                    continue
                logging.info("step1 on expnum: %s, ccd: %s" % (
                    prefix+str(expnum), str(ccd)))
                step1(expnum, ccd, prefix=prefix, version=args.type)
            except Exception as e:
                message = str(e)
                logging.error("Error running step1: %s " %(message))

            storage.set_status(expnum,
                               ccd,
                               prefix+'step1',
                               message)
        
            
Exemple #43
0
        
    storage._dbimages = args.dbimages

    prefix = ( args.fk and 'fk' ) or ''

    ccdlist = ( args.ccd is None and range(0,36) ) or [args.ccd]

    for ccd in ccdlist:
        message = storage.SUCCESS
        try:
            if not storage.get_status(args.expnum, ccd, prefix+'step3'):
                logging.error(storage.get_status(
                        args.expnum,
                        ccd,
                        'step3',
                        return_message=True))
                raise IOError(35, "need to run step3 first")
            if storage.get_status(args.expnum, ccd, prefix+'combine') and not args.force:
                continue
            message = combine(args.expnum, 
                              ccd, 
                              prefix=prefix, 
                              type=args.type, 
                              field=args.field, 
                              measure3=args.measure3
                              )
        except Exception as e:
            message = str(e)

        storage.set_status(args.expnum, ccd, prefix+'combine', message)
Exemple #44
0
def main():
    """Run the script."""

    parser = argparse.ArgumentParser()
    parser.add_argument('--ccd',
                        action='store',
                        type=int,
                        default=None,
                        help='which ccd to process')
    parser.add_argument("--dbimages",
                        action="store",
                        default="vos:OSSOS/dbimages",
                        help='vospace dbimages containerNode')
    parser.add_argument("expnums",
                        type=int,
                        nargs=3,
                        help="expnums to scramble")
    parser.add_argument("--nosort",
                        action='store_true',
                        default=False,
                        help="do not sort before processing")
    parser.add_argument("--type",
                        action='store',
                        default='p',
                        choices=['p', 'o'],
                        help='which type of image')
    parser.add_argument("--verbose", "-v",
                        action="store_true")
    parser.add_argument("--debug", '-d',
                        action='store_true')
    parser.add_argument("--dry-run", action="store_true", help="Do not copy back to VOSpace, implies --force")
    parser.add_argument("--force", action='store_true')
    
    args = parser.parse_args()
    prefix = ""
    task = util.task()
    dependency = "update_header"

    storage.DBIMAGES = args.dbimages

    ## setup logging
    level = logging.CRITICAL
    if args.debug:
        level = logging.DEBUG
    elif args.verbose:
        level = logging.INFO

    logging.basicConfig(level=level, format="%(message)s")
    logging.getLogger('vos').setLevel(level)
    logging.getLogger('vos').addHandler(logging.StreamHandler())
    
    if not args.nosort:
        args.expnums.sort()

    ccds = [args.ccd]
    exit_status = 0
    if args.ccd is None:
        ccds = range(0, 36)
    for ccd in ccds:
        storage.set_logger(task, prefix, args.expnums[0], ccd, args.type, args.dry_run)
        expnums = args.expnums
        if not (args.force or args.dry_run) and storage.get_status(task, prefix, expnums[0], version='s', ccd=ccd):
            continue
        message = storage.SUCCESS

        try:
            scramble(expnums=expnums, ccd=ccd, version='p', dry_run=args.dry_run)
        except Exception as e:
            logging.error(e)
            message = str(e)
            exit_status = message
        if not args.dry_run:
            storage.set_status(task, prefix, expnums[0], version='s', ccd=ccd, status=message)
    return exit_status
Exemple #45
0
    storage._dbimages = args.dbimages

    ## setup logging
    level = logging.CRITICAL
    if args.debug:
        level = logging.DEBUG
    elif args.verbose:
        level = logging.INFO

    logging.basicConfig(level=level, format="%(message)s")

    if not args.nosort:
        args.expnums.sort()

    ccds = [args.ccd]
    if args.ccd is None:
        ccds = range(0, 36)
    for ccd in ccds:
        expnums = args.expnums
        if storage.get_status(expnums[0], ccd, 'scramble') and not args.force:
            continue
        message = storage.SUCCESS

        try:
            scramble(expnums=expnums, ccd=ccd, version=args.type)
        except Exception as e:
            logging.error(e)
            message = str(e)

        storage.set_status(expnums[0], ccd, 'scramble', message)
Exemple #46
0
def align(expnums, ccd, version='s', prefix='', dry_run=False, force=True):
    """Create a 'shifts' file that transforms the space/flux/time scale of all
    images to the first image.

    This function relies on the .fwhm, .trans.jmp, .phot and .zeropoint.used files for inputs.
    The scaling we are computing here is for use in planting sources into the image at the same sky/flux locations
    while accounting for motions of sources with time.

    :param expnums: list of MegaPrime exposure numbers to add artificial KBOs to,
                    the first frame in the list is the reference.
    :param ccd: which ccd to work on.
    :param prefix: put this string in front of expnum when looking for exposure, normally '' or 'fk'
    :param force: When true run task even if this task is recorded as having succeeded
    :param version: Add sources to the 'o', 'p' or 's' images
    :param dry_run: don't push results to VOSpace.
    """
    message = storage.SUCCESS

    if storage.get_status(task, prefix, expnums[0], version,
                          ccd) and not force:
        logging.info("{} completed successfully for {} {} {} {}".format(
            task, prefix, expnums[0], version, ccd))
        return

    # Get the images and supporting files that we need from the VOSpace area
    # get_image and get_file check if the image/file is already on disk.
    # re-computed fluxes from the PSF stars and then recompute x/y/flux scaling.

    # some dictionaries to hold the various scale
    pos = {}
    apcor = {}
    mags = {}
    zmag = {}
    mjdates = {}

    with storage.LoggingManager(task, prefix, expnums[0], ccd, version,
                                dry_run):
        try:
            for expnum in expnums:
                filename = storage.get_image(expnum, ccd=ccd, version=version)
                zmag[expnum] = storage.get_zeropoint(expnum,
                                                     ccd,
                                                     prefix=None,
                                                     version=version)
                mjdates[expnum] = float(
                    fits.open(filename)[0].header.get('MJD-OBS'))
                apcor[expnum] = [
                    float(x) for x in open(
                        storage.get_file(
                            expnum,
                            ccd=ccd,
                            version=version,
                            ext=storage.APCOR_EXT)).read().split()
                ]
                keys = ['crval1', 'cd1_1', 'cd1_2', 'crval2', 'cd2_1', 'cd2_2']
                # load the .trans.jmp values into a 'wcs' like dictionary.
                # .trans.jmp maps current frame to reference frame in pixel coordinates.
                # the reference frame of all the frames supplied must be the same.
                shifts = dict(
                    list(
                        zip(keys, [
                            float(x) for x in open(
                                storage.get_file(
                                    expnum,
                                    ccd=ccd,
                                    version=version,
                                    ext='trans.jmp')).read().split()
                        ])))
                shifts['crpix1'] = 0.0
                shifts['crpix2'] = 0.0
                # now create a wcs object based on those transforms, this wcs links the current frame's
                # pixel coordinates to the reference frame's pixel coordinates.
                w = get_wcs(shifts)

                # get the PHOT file that was produced by the mkpsf routine
                logging.debug("Reading .phot file {}".format(expnum))
                phot = ascii.read(storage.get_file(expnum,
                                                   ccd=ccd,
                                                   version=version,
                                                   ext='phot'),
                                  format='daophot')

                # compute the small-aperture magnitudes of the stars used in the PSF
                logging.debug("Running phot on {}".format(filename))
                mags[expnum] = daophot.phot(filename,
                                            phot['XCENTER'],
                                            phot['YCENTER'],
                                            aperture=apcor[expnum][0],
                                            sky=apcor[expnum][1] + 1,
                                            swidth=apcor[expnum][0],
                                            zmag=zmag[expnum])

                # covert the x/y positions to positions in Frame 1 based on the trans.jmp values.
                logging.debug(
                    "Doing the XY translation to refrence frame: {}".format(w))
                (x, y) = w.wcs_pix2world(mags[expnum]["XCENTER"],
                                         mags[expnum]["YCENTER"], 1)
                pos[expnum] = numpy.transpose([x, y])
                # match this exposures PSF stars position against those in the first image of the set.
                logging.debug("Matching lists")
                idx1, idx2 = util.match_lists(pos[expnums[0]], pos[expnum])

                # compute the magnitdue offset between the current frame and the reference.
                dmags = numpy.ma.array(
                    mags[expnums[0]]["MAG"] - apcor[expnums[0]][2] -
                    (mags[expnum]["MAG"][idx1] - apcor[expnum][2]),
                    mask=idx1.mask)
                dmags.sort()
                # logging.debug("Computed dmags between input and reference: {}".format(dmags))
                error_count = 0

                error_count += 1
                logging.debug("{}".format(error_count))

                # compute the median and determine if that shift is small compared to the scatter.
                try:
                    midx = int(
                        numpy.sum(numpy.any([~dmags.mask], axis=0)) / 2.0)
                    dmag = float(dmags[midx])
                    logging.debug("Computed a mag delta of: {}".format(dmag))
                except Exception as e:
                    logging.error(str(e))
                    logging.error(
                        "Failed to compute mag offset between plant and found using: {}"
                        .format(dmags))
                    dmag = 99.99

                error_count += 1
                logging.debug("{}".format(error_count))

                try:
                    if math.fabs(dmag) > 3 * (dmags.std() + 0.01):
                        logging.warning(
                            "Magnitude shift {} between {} and {} is large: {}"
                            .format(dmag, expnums[0], expnum, shifts))
                except Exception as e:
                    logging.error(str(e))

                error_count += 1
                logging.debug("{}".format(error_count))

                shifts['dmag'] = dmag
                shifts['emag'] = dmags.std()
                shifts['nmag'] = len(dmags.mask) - dmags.mask.sum()
                shifts['dmjd'] = mjdates[expnums[0]] - mjdates[expnum]
                shift_file = os.path.basename(
                    storage.get_uri(expnum, ccd, version, '.shifts'))

                error_count += 1
                logging.debug("{}".format(error_count))

                try:
                    fh = open(shift_file, 'w')
                    fh.write(
                        json.dumps(shifts,
                                   sort_keys=True,
                                   indent=4,
                                   separators=(',', ': '),
                                   cls=NpEncoder))
                    fh.write('\n')
                    fh.close()
                except Exception as e:
                    logging.error(
                        "Creation of SHIFTS file failed while trying to write: {}"
                        .format(shifts))
                    raise e

                error_count += 1
                logging.debug("{}".format(error_count))

                if not dry_run:
                    storage.copy(
                        shift_file,
                        storage.get_uri(expnum, ccd, version, '.shifts'))
            logging.info(message)
        except Exception as ex:
            message = str(ex)
            logging.error(message)

        if not dry_run:
            storage.set_status(task,
                               prefix,
                               expnum,
                               version,
                               ccd,
                               status=message)
Exemple #47
0
def main(task='mkpsf'):

    parser = argparse.ArgumentParser(
        description='Run makepsf chunk of the OSSOS pipeline')

    parser.add_argument('--ccd',
                        '-c',
                        action='store',
                        type=int,
                        dest='ccd',
                        default=None,
                        help='which ccd to process, default is all')
    parser.add_argument('--ignore-update-headers',
                        action='store_true',
                        dest='ignore_update_headers')
    parser.add_argument("--dbimages",
                        action="store",
                        default="vos:OSSOS/dbimages",
                        help='vospace dbimages containerNode')
    parser.add_argument("expnum",
                        type=int,
                        nargs='+',
                        help="expnum(s) to process")
    parser.add_argument(
        "--dry_run",
        action="store_true",
        help="DRY RUN, don't copy results to VOSpace, implies --force")

    parser.add_argument("--fk", action="store_true", help="Run fk images")

    parser.add_argument(
        "--type",
        "-t",
        choices=['o', 'p', 's'],
        help="which type of image: o-RAW, p-ELIXIR, s-SCRAMBLE",
        default='p')
    parser.add_argument("--verbose", "-v", action="store_true")
    parser.add_argument("--force", default=False, action="store_true")
    parser.add_argument("--debug", "-d", action="store_true")

    args = parser.parse_args()

    if args.dry_run:
        args.force = True

    if args.debug:
        logging.basicConfig(level=logging.DEBUG)
    elif args.verbose:
        logging.basicConfig(level=logging.INFO)

    prefix = (args.fk and 'fk') or ''

    storage.DBIMAGES = args.dbimages

    if args.ccd is None:
        ccdlist = range(0, 36)
    else:
        ccdlist = [args.ccd]

    exit_code = 0
    for expnum in args.expnum:
        for ccd in ccdlist:
            storage.set_logger(
                os.path.splitext(os.path.basename(sys.argv[0]))[0], prefix,
                expnum, ccd, args.type, args.dry_run)
            if storage.get_status(
                    expnum, ccd, prefix + task,
                    version=args.type) and not args.force:
                logging.info(
                    "{} completed successfully for {} {} {} {}".format(
                        task, prefix, expnum, args.type, ccd))
                continue
            message = 'success'
            try:
                if not storage.get_status(
                        expnum, 36,
                        'update_header') and not args.ignore_update_headers:
                    raise IOError(
                        "update_header not yet run for {}".format(expnum))
                mkpsf(expnum, ccd, args.type, args.dry_run, prefix=prefix)
                if args.dry_run:
                    continue
                storage.set_status(expnum,
                                   ccd,
                                   prefix + 'fwhm',
                                   version=args.type,
                                   status=str(
                                       storage.get_fwhm(expnum,
                                                        ccd,
                                                        version=args.type)))
                storage.set_status(expnum,
                                   ccd,
                                   prefix + 'zeropoint',
                                   version=args.type,
                                   status=str(
                                       storage.get_zeropoint(
                                           expnum, ccd, version=args.type)))
            except CalledProcessError as cpe:
                message = str(cpe.output)
                exit_code = message
            except Exception as e:
                message = str(e)

            logging.error(message)
            if not args.dry_run:
                storage.set_status(expnum,
                                   ccd,
                                   prefix + 'mkpsf',
                                   version=args.type,
                                   status=message)
    return exit_code
Exemple #48
0
def main():
    """Do the script."""
    parser = argparse.ArgumentParser(description='replace image header')
    parser.add_argument('--extname', help='name of extension to in header')
    parser.add_argument('expnum', type=str, help='exposure to update')
    parser.add_argument('-r',
                        '--replace',
                        action='store_true',
                        help='store modified image back to VOSpace?')
    parser.add_argument('-v', '--verbose', action='store_true')
    parser.add_argument('--debug', action='store_true')
    parser.add_argument('--force',
                        action='store_true',
                        help="Re-run even if previous success recorded")
    parser.add_argument('--dbimages',
                        help="VOSpace DATA storage area.",
                        default="vos:OSSOS/dbimages")

    args = parser.parse_args()
    task = util.task()
    dependency = 'preproc'
    prefix = ""

    storage.DBIMAGES = args.dbimages

    level = logging.CRITICAL
    message_format = "%(message)s"
    if args.verbose:
        level = logging.INFO
    if args.debug:
        level = logging.DEBUG
        message_format = "%(module)s %(funcName)s %(lineno)s %(message)s"
    logging.basicConfig(level=level, format=message_format)
    storage.set_logger(task, prefix, args.expnum, None, None, False)

    message = storage.SUCCESS
    expnum = args.expnum

    exit_status = 0
    try:
        # skip if already succeeded and not in force mode
        if storage.get_status(task, prefix, expnum, "p",
                              36) and not args.force:
            logging.info("Already updated, skipping")
            sys.exit(0)

        image_hdulist = storage.get_image(args.expnum, return_file=False)
        ast_hdulist = storage.get_astheader(expnum, ccd=None)

        run_update_header(image_hdulist, ast_hdulist)
        image_filename = os.path.basename(storage.get_uri(expnum))
        image_hdulist.writeto(image_filename)
        if args.replace:
            dest = storage.dbimages_uri(expnum)
            storage.copy(image_filename, dest)
            storage.set_status('update_header', "", expnum, 'p', 36, message)
    except Exception as e:
        message = str(e)
        if args.replace:
            storage.set_status(task, prefix, expnum, 'p', 36, message)
        exit_status = message
        logging.error(message)

    return exit_status
Exemple #49
0
def run(expnum, ccd, version, dry_run=False, prefix="", force=False):
    """Run the OSSOS jmpmakepsf script.

    """

    message = storage.SUCCESS
    if storage.get_status(task, prefix, expnum, version=version,
                          ccd=ccd) and not force:
        logging.info("{} completed successfully for {} {} {} {}".format(
            task, prefix, expnum, version, ccd))
        return

    with storage.LoggingManager(task, prefix, expnum, ccd, version, dry_run):
        try:
            if not storage.get_status(
                    dependency, prefix, expnum, version, ccd=ccd):
                raise IOError("{} not yet run for {}".format(
                    dependency, expnum))

            # confirm destination directory exists.
            destdir = os.path.dirname(
                storage.dbimages_uri(expnum,
                                     ccd,
                                     prefix=prefix,
                                     version=version,
                                     ext='fits'))
            if not dry_run:
                storage.mkdir(destdir)

            # get image from the vospace storage area
            logging.info("Getting fits image from VOSpace")
            filename = storage.get_image(expnum,
                                         ccd,
                                         version=version,
                                         prefix=prefix)

            # get mopheader from the vospace storage area
            logging.info("Getting mopheader from VOSpace")
            mopheader_filename = storage.get_file(expnum,
                                                  ccd,
                                                  version=version,
                                                  prefix=prefix,
                                                  ext='mopheader')

            # run mkpsf process
            logging.info("Running mkpsf on %s %d" % (expnum, ccd))
            logging.info(
                util.exec_prog(
                    ['jmpmakepsf.csh', './', filename, 'yes', 'yes']))

            if dry_run:
                return

            # place the results into VOSpace
            basename = os.path.splitext(filename)[0]

            for ext in ('mopheader', 'psf.fits', 'zeropoint.used', 'apcor',
                        'fwhm', 'phot'):
                dest = storage.dbimages_uri(expnum,
                                            ccd,
                                            prefix=prefix,
                                            version=version,
                                            ext=ext)
                source = basename + "." + str(ext)
                count = 0
                with open(source, 'r'):
                    while True:
                        count += 1
                        try:
                            logging.info("Attempt {} to copy {} -> {}".format(
                                count, source, dest))
                            storage.copy(source, dest)
                            break
                        except Exception as ex:
                            if count > 10:
                                raise ex

            # set some data parameters associated with the image, determined in this step.
            storage.set_status('fwhm',
                               prefix,
                               expnum,
                               version=version,
                               ccd=ccd,
                               status=str(
                                   storage.get_fwhm(expnum,
                                                    ccd=ccd,
                                                    prefix=prefix,
                                                    version=version)))
            storage.set_status('zeropoint',
                               prefix,
                               expnum,
                               version=version,
                               ccd=ccd,
                               status=str(
                                   storage.get_zeropoint(expnum,
                                                         ccd=ccd,
                                                         prefix=prefix,
                                                         version=version)))
            logging.info(message)
        except Exception as e:
            message = str(e)
            logging.error(message)

        storage.set_status(task,
                           prefix,
                           expnum,
                           version,
                           ccd=ccd,
                           status=message)

    return
Exemple #50
0
        level = logging.DEBUG
    elif args.verbose:
        level = logging.INFO

    logging.basicConfig(level=level, format="%(message)s")

    storage._dbimages = args.dbimages

    prefix = (args.fk and 'fk') or ''

    ccdlist = (args.ccd is None and range(0, 36)) or [args.ccd]

    for ccd in ccdlist:
        message = storage.SUCCESS
        try:
            if not storage.get_status(args.expnum, ccd, prefix + 'step3'):
                logging.error(
                    storage.get_status(args.expnum,
                                       ccd,
                                       'step3',
                                       return_message=True))
                raise IOError(35, "need to run step3 first")
            if storage.get_status(args.expnum, ccd,
                                  prefix + 'combine') and not args.force:
                continue
            message = combine(args.expnum, ccd, prefix=prefix, type=args.type)
        except Exception as e:
            message = str(e)

        storage.set_status(args.expnum, ccd, prefix + 'combine', message)
Exemple #51
0
    for ccd in ccdlist:
        message = storage.SUCCESS

        try:
            if not storage.get_status(args.expnums[0], ccd, prefix+'step2'):
                raise IOError(35, "did step2 run on %s" % ( str(args.expnums)))
            if storage.get_status(args.expnums[0], ccd, prefix+'step3') and not args.force:
                logging.critical("step3 alread ran on expnum :%s, ccd: %d" % (
                        str(args.expnums), ccd))
                continue
            step3(args.expnums, ccd, version=args.type,
                  rate_min=args.rate_min,
                  rate_max=args.rate_max,
                  angle=args.angle,
                  width=args.width,
                  field=args.field,
                  prefix=prefix)
        except Exception as e:
            message = str(e)

        logging.error(message)

        storage.set_status(args.expnums[0],
                           ccd,
                           prefix+'step3',
                           message)
        
            


Exemple #52
0
        logging.basicConfig(level=logging.INFO, format="%(message)s")

    message = storage.SUCCESS
    try:

        image = (os.access(args.expnum, os.W_OK)
                 and args.expnum) or (storage.get_image(args.expnum))

        header = (args.header is not None and
                  ((os.access(args.header, os.W_OK) and args.header) or
                   (storage.get_image(args.header, ext='head')))) or (
                       storage.get_image(args.expnum, ext='head'))

        logging.info("Swapping header for %s for contents in %s \n" %
                     (image, header))

        run_update_header(image, header)

        if args.replace:
            expnum = args.expnum or fits.open(image)[0].header['EXPNUM']
            dest = storage.dbimages_uri(expnum)
            storage.copy(image, dest)
        storage.set_status(args.expnum, 36, 'update_header', message)
        sys.exit(0)
    except Exception as e:
        logging.error("Error replacing header for %s" % (args.expnum))
        logging.error(str(e))
        message = str(e)
        storage.set_status(args.expnum, 36, 'update_header', message)
        sys.exit(2)
Exemple #53
0
    message = storage.SUCCESS
    try:
        
        image = (os.access(args.expnum,os.W_OK) and args.expnum ) or (
            storage.get_image(args.expnum) )
    
        header = (args.header is not None and ((
            os.access(args.header, os.W_OK) and args.header ) or (
            storage.get_image(args.header, ext='head')))) or ( 
            storage.get_image(args.expnum, ext='head'))

        logging.info(
            "Swapping header for %s for contents in %s \n" % (
            image, header) )

        run_update_header(image, header)
    
        if args.replace:
            expnum = args.expnum or fits.open(image)[0].header['EXPNUM']
            dest = storage.dbimages_uri(expnum)
            storage.copy(image, dest)
        storage.set_status(args.expnum, 36, 'update_header', message)
        sys.exit(0)
    except Exception as e:
        logging.error("Error replacing header for %s" % ( args.expnum))
        logging.error(str(e))
        message = str(e)
        storage.set_status(args.expnum, 36, 'update_header', message)
        sys.exit(2)
    
Exemple #54
0
def main():
    ### Must be running as a script

    parser = argparse.ArgumentParser(
        description='Run step2jmp and step2matt on a given exposure.')

    parser.add_argument("--ccd",
                        "-c",
                        action="store",
                        default=None,
                        type=int,
                        dest="ccd")
    parser.add_argument("--fk",
                        action="store_true",
                        default=False,
                        help="Do fakes?")
    parser.add_argument("--dbimages",
                        action="store",
                        default="vos:OSSOS/dbimages",
                        help='vospace dbimages containerNode')
    parser.add_argument("expnums",
                        type=int,
                        nargs=3,
                        help="3 expnums to process")
    parser.add_argument("--version", action='version', version='%(prog)s 1.0')
    parser.add_argument('-t',
                        '--type',
                        help='which type of image to process',
                        choices=['s', 'p', 'o'],
                        default='p')
    parser.add_argument('--no-sort',
                        help='preserve input exposure order',
                        action='store_true')
    parser.add_argument("--verbose", "-v", action="store_true")
    parser.add_argument("--default",
                        default="WCS",
                        choices=['WCS', 'JMP'],
                        help="Which shift should be used if they dis-agree?")
    parser.add_argument("--debug", action="store_true")
    parser.add_argument(
        "--dry_run",
        action="store_true",
        help="run without pushing back to VOSpace, implies --force")
    parser.add_argument("--force", action="store_true")

    args = parser.parse_args()

    if args.dry_run:
        args.force = True

    if args.verbose:
        logging.basicConfig(level=logging.INFO, format="%(message)s")
    if args.debug:
        logging.basicConfig(level=logging.DEBUG, format="%(message)s")

    storage.DBIMAGES = args.dbimages

    if args.ccd is None:
        ccd_list = range(0, 36)
    else:
        ccd_list = [args.ccd]

    prefix = (args.fk and "fk") or ""

    if not args.no_sort:
        args.expnums.sort()

    exit_status = 0
    for ccd in ccd_list:
        storage.set_logger(
            os.path.splitext(os.path.basename(sys.argv[0]))[0], prefix,
            args.expnums[0], ccd, args.type, args.dry_run)
        try:
            message = storage.SUCCESS
            for expnum in args.expnums:
                if not storage.get_status(
                        expnum, ccd, prefix + 'step1', version=args.type):
                    raise IOError(35, "missing step1 for %s" % expnum)
            if storage.get_status(
                    args.expnums[0], ccd, prefix + 'step2',
                    version=args.type) and not args.force:
                logging.info("Already did %s %s, skipping" %
                             (str(args.expnums[0]), str(ccd)))
                continue
            logging.info("step2 on expnums :%s, ccd: %d" %
                         (str(args.expnums), ccd))
            step2(args.expnums,
                  ccd,
                  version=args.type,
                  prefix=prefix,
                  dry_run=args.dry_run,
                  default=args.default)
            logging.info(message)
        except CalledProcessError as cpe:
            message = str(cpe)
            logging.error(message)
            exit_status = message
        except Exception as e:
            message = str(e)
            logging.error(message)
            exit_status = message
        if not args.dry_run:
            storage.set_status(args.expnums[0],
                               ccd,
                               prefix + 'step2',
                               version=args.type,
                               status=message)
    return exit_status