Exemple #1
0
def main(args):
    import os.path
    import sys
    import yaml
    import numpy as np
    import pdb
    import matplotlib
    matplotlib.use('agg')

    import desispec.io
    from desiutil.log import get_logger

    from desisim.spec_qa import redshifts as dsqa_z
    from desiutil.io import yamlify
    import desiutil.depend

    log = get_logger()

    # Initialize
    if args.qaprod_dir is not None:
        qaprod_dir = args.qaprod_dir
    else:
        qaprod_dir = desispec.io.meta.qaprod_root()


    if args.load_simz_table is not None:
        from astropy.table import Table
        log.info("Loading simz info from {:s}".format(args.load_simz_table))
        simz_tab = Table.read(args.load_simz_table)
    else:
        # Grab list of fibermap files
        fibermap_files = []
        zbest_files = []
        nights = desispec.io.get_nights()
        for night in nights:
            for exposure in desispec.io.get_exposures(night, raw=True, rawdata_dir=args.rawdir):
                # Ignore exposures with no fibermap, assuming they are calibration data.
                fibermap_path = desispec.io.findfile(filetype='fibermap', night=night,
                                                     expid=exposure, rawdata_dir=args.rawdir)
                if not os.path.exists(fibermap_path):
                    log.debug('Skipping exposure %08d with no fibermap.' % exposure)
                    continue
                # Load data
                fibermap_data = desispec.io.read_fibermap(fibermap_path)
                # Skip calib
                objtype = fibermap_data['OBJTYPE'][0]
                if objtype in ['FLAT', 'FLT', 'ARC', 'BIAS', 'BIA']:
                    continue
                elif objtype in ['SKY','TGT','BAD']:
                    pass
                else:
                    err_message = 'Unrecognized OBJTYPE {}'.format(objtype)
                    log.critical(err_message)
                    raise ValueError(err_message)

                # Append fibermap file
                fibermap_files.append(fibermap_path)
                # Slurp the zbest_files
                zbest_files += dsqa_z.find_zbest_files(fibermap_data)

        # Cut down zbest_files to unique ones
        zbest_files = list(set(zbest_files))

        if len(zbest_files) == 0:
            log.fatal('No zbest files found')
            sys.exit(1)

        # Write? Table
        simz_tab, zbtab = dsqa_z.load_z(fibermap_files, zbest_files=zbest_files)
        dsqa_z.match_truth_z(simz_tab, zbtab)
        if args.write_simz_table is not None:
            simz_tab.write(args.write_simz_table, overwrite=True)

    # Meta data
    meta = dict(
        DESISIM = desiutil.depend.getdep(simz_tab.meta, 'desisim'),
        SPECPROD = os.getenv('SPECPROD', 'unknown'),
        PIXPROD = os.getenv('PIXPROD', 'unknown'),
        )
    # Include specter version if it was used to generate input files
    # (it isn't used for specsim inputs so that dependency may not exist)
    try:
        meta['SPECTER'] = desiutil.depend.getdep(simz_tab.meta, 'specter')
    except KeyError:
        pass
    
    # Run stats
    log.info("Running stats..")
    summ_dict = dsqa_z.summ_stats(simz_tab)
    if args.yaml_file is not None:
        log.info("Generating yaml file of stats: {:s}".format(args.yaml_file))
        # yamlify
        # Write yaml
        desispec.io.util.makepath(args.yaml_file)
        with open(args.yaml_file, 'w') as outfile:
            outfile.write(yaml.dump(yamlify(meta), default_flow_style=False))
            outfile.write(yaml.dump(yamlify(summ_dict), default_flow_style=False))

    log.info("Generating QA files")
    # Summary for dz of all types
    outfile = qaprod_dir+'/QA_dzsumm.png'
    desispec.io.util.makepath(outfile)
    dsqa_z.dz_summ(simz_tab, outfile=outfile)
    # Summary of individual types
    #outfile = args.qafig_root+'_summ_fig.png'
    #dsqa_z.summ_fig(simz_tab, summ_dict, meta, outfile=outfile)
    for objtype in ['BGS', 'MWS', 'ELG','LRG', 'QSO_T', 'QSO_L']:
        outfile = qaprod_dir+'/QA_zfind_{:s}.png'.format(objtype)
        desispec.io.util.makepath(outfile)
        dsqa_z.obj_fig(simz_tab, objtype, summ_dict, outfile=outfile)
Exemple #2
0
def main(args):
    import os.path
    import sys
    import yaml
    import numpy as np
    import pdb
    import matplotlib
    matplotlib.use('agg')

    import desispec.io
    from desiutil.log import get_logger

    from desisim.spec_qa import redshifts as dsqa_z
    from desiutil.io import yamlify
    import desiutil.depend
    from desimodel.footprint import radec2pix

    log = get_logger()

    if args.load_simz_table is not None:
        from astropy.table import Table
        log.info("Loading simz info from {:s}".format(args.load_simz_table))
        simz_tab = Table.read(args.load_simz_table)
    else:
        # Grab list of fibermap files
        fibermap_files = []
        zbest_files = []
        nights = desispec.io.get_nights()
        for night in nights:
            for exposure in desispec.io.get_exposures(night,
                                                      raw=True,
                                                      rawdata_dir=args.rawdir):
                # Ignore exposures with no fibermap, assuming they are calibration data.
                fibermap_path = desispec.io.findfile(filetype='fibermap',
                                                     night=night,
                                                     expid=exposure,
                                                     rawdata_dir=args.rawdir)
                if not os.path.exists(fibermap_path):
                    log.debug('Skipping exposure %08d with no fibermap.' %
                              exposure)
                    continue
                '''
                # Search for zbest files
                fibermap_data = desispec.io.read_fibermap(fibermap_path)
                flavor = fibermap_data.meta['FLAVOR']
                if flavor.lower() in ('arc', 'flat', 'bias'):
                    log.debug('Skipping calibration {} exposure {:08d}'.format(flavor, exposure))
                    continue

                brick_names = set(fibermap_data['BRICKNAME'])
                import pdb; pdb.set_trace()
                for brick in brick_names:
                    zbest_path=desispec.io.findfile('zbest', groupname=brick, specprod_dir=args.reduxdir)
                    if os.path.exists(zbest_path):
                        log.debug('Found {}'.format(os.path.basename(zbest_path)))
                        zbest_files.append(zbest_path)
                    else:
                        log.warn('Missing {}'.format(os.path.basename(zbest_path)))
                        #pdb.set_trace()
                '''
                # Load data
                fibermap_data = desispec.io.read_fibermap(fibermap_path)
                # Skip calib
                if fibermap_data['OBJTYPE'][0] in ['FLAT', 'ARC', 'BIAS']:
                    continue
                elif fibermap_data['OBJTYPE'][0] in ['SKY', 'STD', 'SCIENCE']:
                    pass
                else:
                    pdb.set_trace()
                # Append fibermap file
                fibermap_files.append(fibermap_path)
                # Search for zbest files with healpy
                ra_targ = fibermap_data['RA_TARGET'].data
                dec_targ = fibermap_data['DEC_TARGET'].data
                # Getting some NAN in RA/DEC
                good = np.isfinite(ra_targ) & np.isfinite(dec_targ)
                pixels = radec2pix(64, ra_targ[good], dec_targ[good])
                uni_pixels = np.unique(pixels)
                for uni_pix in uni_pixels:
                    zbest_files.append(
                        desispec.io.findfile('zbest',
                                             groupname=uni_pix,
                                             nside=64))

        # Cut down zbest_files to unique ones
        zbest_files = list(set(zbest_files))

        if len(zbest_files) == 0:
            log.fatal('No zbest files found')
            sys.exit(1)

        # Write? Table
        simz_tab = dsqa_z.load_z(fibermap_files, zbest_files)
        if args.write_simz_table is not None:
            simz_tab.write(args.write_simz_table, overwrite=True)

    # Meta data
    meta = dict(
        DESISIM=desiutil.depend.getdep(simz_tab.meta, 'desisim'),
        SPECTER=desiutil.depend.getdep(simz_tab.meta, 'specter'),
        SPECPROD=os.getenv('SPECPROD', 'unknown'),
        PIXPROD=os.getenv('PIXPROD', 'unknown'),
    )

    # Run stats
    log.info("Running stats..")
    summ_dict = dsqa_z.summ_stats(simz_tab)
    if args.qafile is not None:
        log.info("Generating yaml file: {:s}".format(args.qafile))
        # yamlify
        # Write yaml
        with open(args.qafile, 'w') as outfile:
            outfile.write(yaml.dump(yamlify(meta), default_flow_style=False))
            outfile.write(
                yaml.dump(yamlify(summ_dict), default_flow_style=False))

    if args.qafig_root is not None:
        log.info("Generating QA files")
        # Summary for dz of all types
        outfile = args.qafig_root + '_dzsumm.png'
        #dsqa_z.dz_summ(simz_tab, outfile=outfile)
        # Summary of individual types
        #outfile = args.qafig_root+'_summ_fig.png'
        #dsqa_z.summ_fig(simz_tab, summ_dict, meta, outfile=outfile)
        for objtype in ['BGS', 'MWS', 'ELG', 'LRG', 'QSO_T', 'QSO_L']:
            outfile = args.qafig_root + '_zfind_{:s}.png'.format(objtype)
            dsqa_z.obj_fig(simz_tab, objtype, summ_dict, outfile=outfile)
Exemple #3
0
def main(args):
    import os.path
    import sys
    import yaml
    import numpy as np
    import pdb
    import matplotlib
    matplotlib.use('agg')

    import desispec.io
    from desiutil.log import get_logger

    from desisim.spec_qa import redshifts as dsqa_z
    from desiutil.io import yamlify
    import desiutil.depend

    log = get_logger()

    # Initialize
    if args.qaprod_dir is not None:
        qaprod_dir = args.qaprod_dir
    else:
        qaprod_dir = desispec.io.meta.qaprod_root()

    if args.load_simz_table is not None:
        from astropy.table import Table
        log.info("Loading simz info from {:s}".format(args.load_simz_table))
        simz_tab = Table.read(args.load_simz_table)
    else:
        # Grab list of fibermap files
        fibermap_files = []
        zbest_files = []
        nights = desispec.io.get_nights()
        for night in nights:
            for exposure in desispec.io.get_exposures(night,
                                                      raw=True,
                                                      rawdata_dir=args.rawdir):
                # Ignore exposures with no fibermap, assuming they are calibration data.
                fibermap_path = desispec.io.findfile(filetype='fibermap',
                                                     night=night,
                                                     expid=exposure,
                                                     rawdata_dir=args.rawdir)
                if not os.path.exists(fibermap_path):
                    log.debug('Skipping exposure %08d with no fibermap.' %
                              exposure)
                    continue
                # Load data
                fibermap_data = desispec.io.read_fibermap(fibermap_path)
                # Skip calib
                objtype = fibermap_data['OBJTYPE'][0]
                if objtype in ['FLAT', 'FLT', 'ARC', 'BIAS', 'BIA']:
                    continue
                elif objtype in ['SKY', 'TGT', 'BAD']:
                    pass
                else:
                    err_message = 'Unrecognized OBJTYPE {}'.format(objtype)
                    log.critical(err_message)
                    raise ValueError(err_message)

                # Append fibermap file
                fibermap_files.append(fibermap_path)
                # Slurp the zbest_files
                zbest_files += dsqa_z.find_zbest_files(fibermap_data)

        # Cut down zbest_files to unique ones
        zbest_files = list(set(zbest_files))

        if len(zbest_files) == 0:
            log.fatal('No zbest files found')
            sys.exit(1)

        # Write? Table
        simz_tab, zbtab = dsqa_z.load_z(fibermap_files,
                                        zbest_files=zbest_files)
        dsqa_z.match_truth_z(simz_tab, zbtab)
        if args.write_simz_table is not None:
            simz_tab.write(args.write_simz_table, overwrite=True)

    # Meta data
    meta = dict(
        DESISIM=desiutil.depend.getdep(simz_tab.meta, 'desisim'),
        SPECPROD=os.getenv('SPECPROD', 'unknown'),
        PIXPROD=os.getenv('PIXPROD', 'unknown'),
    )
    # Include specter version if it was used to generate input files
    # (it isn't used for specsim inputs so that dependency may not exist)
    try:
        meta['SPECTER'] = desiutil.depend.getdep(simz_tab.meta, 'specter')
    except KeyError:
        pass

    # Run stats
    log.info("Running stats..")
    summ_dict = dsqa_z.summ_stats(simz_tab)
    if args.yaml_file is not None:
        log.info("Generating yaml file of stats: {:s}".format(args.yaml_file))
        # yamlify
        # Write yaml
        desispec.io.util.makepath(args.yaml_file)
        with open(args.yaml_file, 'w') as outfile:
            outfile.write(yaml.dump(yamlify(meta), default_flow_style=False))
            outfile.write(
                yaml.dump(yamlify(summ_dict), default_flow_style=False))

    log.info("Generating QA files")
    # Summary for dz of all types
    outfile = qaprod_dir + '/QA_dzsumm.png'
    desispec.io.util.makepath(outfile)
    dsqa_z.dz_summ(simz_tab, outfile=outfile)
    # Summary of individual types
    #outfile = args.qafig_root+'_summ_fig.png'
    #dsqa_z.summ_fig(simz_tab, summ_dict, meta, outfile=outfile)
    for objtype in ['BGS', 'MWS', 'ELG', 'LRG', 'QSO_T', 'QSO_L']:
        outfile = qaprod_dir + '/QA_zfind_{:s}.png'.format(objtype)
        desispec.io.util.makepath(outfile)
        dsqa_z.obj_fig(simz_tab, objtype, summ_dict, outfile=outfile)
Exemple #4
0
def main():
    '''Runs the process
    '''
    # Check environmental variables are set
    assert 'DESI_SPECTRO_DATA' in os.environ, 'Missing $DESI_SPECTRO_DATA environment variable'
    assert 'PRODNAME' in os.environ, 'Missing $PRODNAME environment variable'
    assert 'DESI_SPECTRO_REDUX' in os.environ, 'Missing $DESI_SPECTRO_REDUX environment variable'

    # Grab nights in data redux
    nights = glob.glob(dio_meta.specprod_root() + '/exposures/*')
    if len(nights) == 0:
        raise ValueError('No nights in exposures!')

    # Build up list of fibermap files
    fibermap_files = []
    for night in nights:
        onight = night[night.rfind('/'):]
        files = glob.glob(dio_meta.rawdata_root() + '/' + onight +
                          '/fibermap*')
        #
        fibermap_files += files

    # Get list of zbest files
    zbest_files = glob.glob(dio_meta.specprod_root() + '/bricks/*/zbest*')
    if len(zbest_files) == 0:
        raise ValueError('No redshifts?!')

    # Meta
    meta = get_meta()

    # Load+write table
    simtab_fil = dio_meta.specprod_root() + '/QA/sim_z_table.fits'
    dio_util.makepath(simtab_fil)
    simz_tab = dsqa_z.load_z(fibermap_files, zbest_files)
    simz_tab.meta = meta
    simz_tab.write(simtab_fil, overwrite=True)

    # Summary stats
    summ_file = dio_meta.specprod_root() + '/QA/sim_z_summ.yaml'
    dio_util.makepath(summ_file)
    summ_dict = dsqa_z.summ_stats(simz_tab)
    # Write
    with open(summ_file, 'w') as outfile:
        outfile.write(yaml.dump(meta))  #, default_flow_style=True) )
        outfile.write(yaml.dump(summ_dict, default_flow_style=False))
    #import pdb
    #pdb.set_trace()
    '''
    # Reorder + cut
    summ_tab=full_summ_tab['OBJTYPE', 'NTARG', 'N_SURVEY', 'EFF', 'MED_DZ', 'CAT_RATE', 'REQ_FINAL']
    # Write
    summ_tab.meta = meta
    summ_tab.write(summ_file,format='ascii.ecsv', 
        formats=dict(MED_DZ='%8.6f',EFF='%5.3f',CAT_RATE='%6.4f'))#,clobber=True)
    '''

    # QA Figures
    fig_file = dio_meta.specprod_root() + '/QA/sim_z.pdf'
    dio_util.makepath(fig_file)
    pp = PdfPages(fig_file)
    # Summ
    dsqa_z.summ_fig(simz_tab, summ_dict, meta, pp=pp)
    for objtype in ['ELG', 'LRG', 'QSO_T', 'QSO_L']:
        dsqa_z.obj_fig(simz_tab, objtype, summ_dict, pp=pp)
    # All done
    pp.close()
Exemple #5
0
def main():
    '''Runs the process
    '''
    # Check environmental variables are set
    assert 'DESI_SPECTRO_DATA' in os.environ, 'Missing $DESI_SPECTRO_DATA environment variable'
    assert 'SPECPROD' in os.environ, 'Missing $SPECPROD environment variable'
    assert 'DESI_SPECTRO_REDUX' in os.environ, 'Missing $DESI_SPECTRO_REDUX environment variable'

    # Grab nights in data redux
    nights = glob.glob(dio_meta.specprod_root()+'/exposures/*')
    if len(nights) == 0:
        raise ValueError('No nights in exposures!')

    # Build up list of fibermap files
    fibermap_files = []
    for night in nights:
        onight = night[night.rfind('/'):]
        files = glob.glob(dio_meta.rawdata_root()+'/'+onight+'/fibermap*')
        #
        fibermap_files += files

    # Get list of zbest files
    zbest_files = glob.glob(dio_meta.specprod_root()+'/bricks/*/zbest*')
    if len(zbest_files) == 0:
        raise ValueError('No redshifts?!')

    # Meta
    meta = get_meta()

    # Load+write table
    simtab_fil = dio_meta.specprod_root()+'/QA/sim_z_table.fits'
    dio_util.makepath(simtab_fil)
    simz_tab = dsqa_z.load_z(fibermap_files, zbest_files)
    simz_tab.meta = meta
    simz_tab.write(simtab_fil,overwrite=True)

    # Summary stats
    summ_file = dio_meta.specprod_root()+'/QA/sim_z_summ.yaml'
    dio_util.makepath(summ_file)
    summ_dict = dsqa_z.summ_stats(simz_tab)
    # Write
    with open(summ_file, 'w') as outfile:
        outfile.write( yaml.dump(meta))#, default_flow_style=True) )
        outfile.write( yaml.dump(summ_dict, default_flow_style=False) )
    #import pdb
    #pdb.set_trace()


    '''
    # Reorder + cut
    summ_tab=full_summ_tab['OBJTYPE', 'NTARG', 'N_SURVEY', 'EFF', 'MED_DZ', 'CAT_RATE', 'REQ_FINAL']
    # Write
    summ_tab.meta = meta
    summ_tab.write(summ_file,format='ascii.ecsv',
        formats=dict(MED_DZ='%8.6f',EFF='%5.3f',CAT_RATE='%6.4f'))#,overwrite=True)
    '''

    # QA Figures
    fig_file = dio_meta.specprod_root()+'/QA/sim_z.pdf'
    dio_util.makepath(fig_file)
    pp = PdfPages(fig_file)
    # Summ
    dsqa_z.summ_fig(simz_tab, summ_dict, meta, pp=pp)
    for objtype in ['ELG','LRG', 'QSO_T', 'QSO_L']:
        dsqa_z.obj_fig(simz_tab, objtype, summ_dict, pp=pp)
    # All done
    pp.close()
Exemple #6
0
def main():
    """Runs the process
    """
    # Check environmental variables are set
    assert "DESI_SPECTRO_DATA" in os.environ, "Missing $DESI_SPECTRO_DATA environment variable"
    assert "PRODNAME" in os.environ, "Missing $PRODNAME environment variable"
    assert "DESI_SPECTRO_REDUX" in os.environ, "Missing $DESI_SPECTRO_REDUX environment variable"

    # Grab nights in data redux
    nights = glob.glob(dio_meta.specprod_root() + "/exposures/*")
    if len(nights) == 0:
        raise ValueError("No nights in exposures!")

    # Build up list of fibermap files
    fibermap_files = []
    for night in nights:
        onight = night[night.rfind("/") :]
        files = glob.glob(dio_meta.data_root() + "/" + onight + "/fibermap*")
        #
        fibermap_files += files

    # Get list of zbest files
    zbest_files = glob.glob(dio_meta.specprod_root() + "/bricks/*/zbest*")
    if len(zbest_files) == 0:
        raise ValueError("No redshifts?!")

    # Meta
    meta = get_meta()

    # Load+write table
    simtab_fil = dio_meta.specprod_root() + "/QA/sim_z_table.fits"
    dio_util.makepath(simtab_fil)
    simz_tab = dsqa_z.load_z(fibermap_files, zbest_files)
    simz_tab.meta = meta
    simz_tab.write(simtab_fil, overwrite=True)

    # Summary stats
    summ_file = dio_meta.specprod_root() + "/QA/sim_z_summ.yaml"
    dio_util.makepath(summ_file)
    summ_dict = dsqa_z.summ_stats(simz_tab)
    # Write
    with open(summ_file, "w") as outfile:
        outfile.write(yaml.dump(meta))  # , default_flow_style=True) )
        outfile.write(yaml.dump(summ_dict, default_flow_style=False))
    # import pdb
    # pdb.set_trace()

    """
    # Reorder + cut
    summ_tab=full_summ_tab['OBJTYPE', 'NTARG', 'N_SURVEY', 'EFF', 'MED_DZ', 'CAT_RATE', 'REQ_FINAL']
    # Write
    summ_tab.meta = meta
    summ_tab.write(summ_file,format='ascii.ecsv', 
        formats=dict(MED_DZ='%8.6f',EFF='%5.3f',CAT_RATE='%6.4f'))#,clobber=True)
    """

    # QA Figures
    fig_file = dio_meta.specprod_root() + "/QA/sim_z.pdf"
    dio_util.makepath(fig_file)
    pp = PdfPages(fig_file)
    # Summ
    dsqa_z.summ_fig(simz_tab, summ_dict, meta, pp=pp)
    for objtype in ["ELG", "LRG", "QSO_T", "QSO_L"]:
        dsqa_z.obj_fig(simz_tab, objtype, summ_dict, pp=pp)
    # All done
    pp.close()