Esempio n. 1
0
def preview_namelists(dryrun=False, case=None, casedir=None):
    if (case is None):
        if (casedir is None):
            case = Case()
        else:
            case = Case(case_root=casedir)

    # Get data from XML
    exeroot = case.get_value("EXEROOT")
    libroot = case.get_value("LIBROOT")
    incroot = case.get_value("INCROOT")
    rundir = case.get_value("RUNDIR")
    caseroot = case.get_value("CASEROOT")
    cimeroot = case.get_value("CIMEROOT")
    casebuild = case.get_value("CASEBUILD")
    testcase = case.get_value("TESTCASE")
    compiler = case.get_value("COMPILER")
    mach = case.get_value("MACH")
    debug = case.get_value("DEBUG")
    mpilib = case.get_value("MPILIB")

    logging.debug("LID is: '%s'" % os.getenv("LID", ""))
    logging.debug("caseroot is: '%s'" % caseroot)

    dryrun = True if (testcase == "SBN") else dryrun

    models = ["atm", "lnd", "ice", "ocn", "glc", "wav", "rof", "cpl"]
    docdir = os.path.join(caseroot, "CaseDocs")

    if (dryrun):
        # Only create rundir
        try:
            os.makedirs(rundir)
        except OSError:
            logging.warning("Not able to create $RUNDIR, trying a subdirectory of $CASEROOT")
            rundir = os.path.join(caseroot, rundir)
            try:
                os.makedirs(rundir)
                logging.info("Success! Setting RUNDIR=%s" % rundir)
                case.set_value("RUNDIR", rundir)
            except OSError:
                expect(False, "Could not create rundir")

    else:

        # Load modules
        env_module = EnvModule(mach, compiler, cimeroot, caseroot, mpilib, debug)
        env_module.load_env_for_case()

        # Make necessary directories
        dirs_to_make = [os.path.join(exeroot, model, "obj") for model in models]
        dirs_to_make.extend([exeroot, libroot, incroot, rundir, docdir])

        for dir_to_make in dirs_to_make:
            if (not os.path.isdir(dir_to_make)):
                try:
                    logging.debug("Making dir '%s'" % dir_to_make)
                    os.makedirs(dir_to_make)
                except OSError as e:
                    expect(False, "Could not make directory '%s', error: %s" % (dir_to_make, e))

    # Create namelists
    for model in models:
        model_str = "drv" if model == "cpl" else model
        config_file = case.get_value("CONFIG_%s_FILE" % model_str.upper())
        config_dir = os.path.dirname(config_file)
        cmd = os.path.join(config_dir, "buildnml")

        if (logging.getLogger().level == logging.DEBUG):
            run_cmd("PREVIEW_NML=1 %s %s" % (cmd, caseroot))
        else:
            run_cmd("%s %s" % (cmd, caseroot))

    # Save namelists to docdir
    if (not os.path.isdir(docdir)):
        os.makedirs(docdir)
        try:
            with open(os.path.join(docdir, "README"), "w") as fd:
                fd.write(" CESM Resolved Namelist Files\n   For documentation only DO NOT MODIFY\n")
        except (OSError, IOError) as e:
            expect(False, "Failed to write %s/README: %s" % (docdir, e))


    for cpglob in ["*_in_[0-9]*", "*modelio*", "*_in",
                   "*streams*txt*", "*stxt", "*maps.rc", "*cism.config*"]:
        for file_to_copy in glob.glob(os.path.join(rundir, cpglob)):
            logging.debug("Copy file from '%s' to '%s'" % (file_to_copy, docdir))
            shutil.copy2(file_to_copy, docdir)

    # Copy over chemistry mechanism docs if they exist
    if (os.path.isdir(os.path.join(casebuild, "camconf"))):
        for file_to_copy in glob.glob(os.path.join(casebuild, "camconf", "*chem_mech*")):
            shutil.copy2(file_to_copy, docdir)
Esempio n. 2
0
def case_build(caseroot, case=None, testmode=False, sharedlib_only=False, model_only=False):
###############################################################################
    t1 = time.time()

    expect(not (sharedlib_only and model_only),
           "Contradiction: both sharedlib_only and model_only")

    logger.info("sharedlib_only is %s" % sharedlib_only)
    logger.info("model_only is %s" % model_only)

    expect(os.path.isdir(caseroot), "'%s' is not a valid directory" % caseroot)
    os.chdir(caseroot)

    expect(os.path.exists("case.run"),
           "ERROR: must invoke case.setup script before calling build script ")

    case = Case() if case is None else case
    testcase = case.get_value("TESTCASE")
    cimeroot = case.get_value("CIMEROOT")
    expect(not (testcase is not None and
                os.path.exists("%s/scripts/Testing/Testcases/%s_build.csh" %
                               (cimeroot, testcase)) and not testmode),
           "%s build must be invoked via case.testbuild script" % testcase)

    if not sharedlib_only:
        check_all_input_data(case)

    run_cmd("./Tools/check_lockedfiles --caseroot %s" % caseroot)

    # Retrieve relevant case data
    build_threaded      = case.get_value("BUILD_THREADED")
    casetools           = case.get_value("CASETOOLS")
    exeroot             = case.get_value("EXEROOT")
    incroot             = case.get_value("INCROOT")
    libroot             = case.get_value("LIBROOT")
    sharedlibroot       = case.get_value("SHAREDLIBROOT")
    comp_atm            = case.get_value("COMP_ATM")
    comp_lnd            = case.get_value("COMP_LND")
    comp_ice            = case.get_value("COMP_ICE")
    comp_ocn            = case.get_value("COMP_OCN")
    comp_glc            = case.get_value("COMP_GLC")
    comp_wav            = case.get_value("COMP_WAV")
    comp_rof            = case.get_value("COMP_ROF")
    compiler            = case.get_value("COMPILER")
    comp_interface      = case.get_value("COMP_INTERFACE")
    mpilib              = case.get_value("MPILIB")
    use_esmf_lib        = case.get_value("USE_ESMF_LIB")
    debug               = case.get_value("DEBUG")
    ninst_build         = case.get_value("NINST_BUILD")
    smp_value           = case.get_value("SMP_VALUE")
    clm_use_petsc       = case.get_value("CLM_USE_PETSC")
    cism_use_trilinos   = case.get_value("CISM_USE_TRILINOS")
    mpasli_use_albany   = case.get_value("MPASLI_USE_ALBANY")
    clm_config_opts     = case.get_value("CLM_CONFIG_OPTS")
    cam_config_opts     = case.get_value("CAM_CONFIG_OPTS")
    pio_config_opts     = case.get_value("PIO_CONFIG_OPTS")
    ninst_value         = case.get_value("NINST_VALUE")
    mach                = case.get_value("MACH")
    os_                 = case.get_value("OS")
    comp_cpl            = case.get_value("COMP_CPL")
    machines_file       = case.get_value("MACHINES_SPEC_FILE")
    ocn_submodel        = case.get_value("OCN_SUBMODEL")
    profile_papi_enable = case.get_value("PROFILE_PAPI_ENABLE")
    nthrds_cpl          = int(case.get_value("NTHRDS_CPL"))
    nthrds_atm          = int(case.get_value("NTHRDS_ATM"))
    nthrds_lnd          = int(case.get_value("NTHRDS_LND"))
    nthrds_ice          = int(case.get_value("NTHRDS_ICE"))
    nthrds_ocn          = int(case.get_value("NTHRDS_OCN"))
    nthrds_glc          = int(case.get_value("NTHRDS_GLC"))
    nthrds_wav          = int(case.get_value("NTHRDS_WAV"))
    nthrds_rof          = int(case.get_value("NTHRDS_ROF"))

    # Load some params into env
    os.environ["CIMEROOT"]             = cimeroot
    os.environ["CASETOOLS"]            = casetools
    os.environ["EXEROOT"]              = exeroot
    os.environ["INCROOT"]              = incroot
    os.environ["LIBROOT"]              = libroot
    os.environ["SHAREDLIBROOT"]        = sharedlibroot
    os.environ["CASEROOT"]             = caseroot
    os.environ["COMPILER"]             = compiler
    os.environ["COMP_INTERFACE"]       = comp_interface
    os.environ["NINST_VALUE"]          = ninst_value
    os.environ["BUILD_THREADED"]       = build_threaded
    os.environ["MACH"]                 = mach
    os.environ["USE_ESMF_LIB"]         = use_esmf_lib
    os.environ["MPILIB"]               = mpilib
    os.environ["DEBUG"]                = debug
    os.environ["OS"]                   = os_
    os.environ["COMP_CPL"]             = comp_cpl
    os.environ["COMP_ATM"]             = comp_atm
    os.environ["COMP_LND"]             = comp_lnd
    os.environ["COMP_ICE"]             = comp_ice
    os.environ["COMP_OCN"]             = comp_ocn
    os.environ["COMP_GLC"]             = comp_glc
    os.environ["COMP_WAV"]             = comp_wav
    os.environ["COMP_ROF"]             = comp_rof
    os.environ["CLM_CONFIG_OPTS"]      = clm_config_opts     if clm_config_opts     is not None else ""
    os.environ["CAM_CONFIG_OPTS"]      = cam_config_opts     if cam_config_opts     is not None else ""
    os.environ["PIO_CONFIG_OPTS"]      = pio_config_opts     if pio_config_opts     is not None else ""
    os.environ["OCN_SUBMODEL"]         = ocn_submodel        if ocn_submodel        is not None else ""
    os.environ["PROFILE_PAPI_ENABLE"]  = profile_papi_enable if profile_papi_enable is not None else ""
    os.environ["CLM_USE_PETSC"]        = clm_use_petsc       if clm_use_petsc       is not None else ""
    os.environ["CISM_USE_TRILINOS"]    = cism_use_trilinos   if cism_use_trilinos   is not None else ""
    os.environ["MPASLI_USE_ALBANY"]    = mpasli_use_albany   if mpasli_use_albany   is not None else ""

    # This is a timestamp for the build , not the same as the testid,
    # and this case may not be a test anyway. For a production
    # experiment there may be many builds of the same case.
    lid               = run_cmd("date +%y%m%d-%H%M%S")
    os.environ["LID"] = lid

    # Set the overall USE_PETSC variable to TRUE if any of the
    # XXX_USE_PETSC variables are TRUE.
    # For now, there is just the one CLM_USE_PETSC variable, but in
    # the future there may be others -- so USE_PETSC will be true if
    # ANY of those are true.

    use_petsc = "TRUE" if clm_use_petsc == "TRUE" else "FALSE"
    case.set_value("USE_PETSC", use_petsc)
    os.environ["USE_PETSC"] = use_petsc

    # Set the overall USE_TRILINOS variable to TRUE if any of the
    # XXX_USE_TRILINOS variables are TRUE.
    # For now, there is just the one CISM_USE_TRILINOS variable, but in
    # the future there may be others -- so USE_TRILINOS will be true if
    # ANY of those are true.

    use_trilinos = "TRUE" if cism_use_trilinos == "TRUE" else "FALSE"
    case.set_value("USE_TRILINOS", use_trilinos)
    os.environ["USE_TRILINOS"] = use_trilinos

    # Set the overall USE_ALBANY variable to TRUE if any of the
    # XXX_USE_ALBANY variables are TRUE.
    # For now, there is just the one MPASLI_USE_ALBANY variable, but in
    # the future there may be others -- so USE_ALBANY will be true if
    # ANY of those are true.

    use_albany = "TRUE" if mpasli_use_albany == "TRUE" else "FALSE"
    case.set_value("USE_ALBANY", use_albany)
    os.environ["USE_ALBANY"] = use_albany

    # Load modules
    env_module = EnvModule(mach, compiler, cimeroot, caseroot, mpilib, debug)
    env_module.load_env_for_case()

    # Need to flush case xml to disk before calling preview_namelists
    case.flush()

    if not sharedlib_only:
        preview_namelists(case=case)

    build_checks(case, build_threaded, comp_interface, use_esmf_lib, debug, compiler, mpilib, 
                 sharedlibroot, nthrds_cpl, nthrds_atm, nthrds_lnd, nthrds_ice, nthrds_ocn, 
                 nthrds_glc, nthrds_wav, nthrds_rof, ninst_build, smp_value)

    t2 = time.time()
    logs = []

    if not model_only:
        logs = build_libraries(exeroot, caseroot, cimeroot, libroot, mpilib, lid, machines_file)
        if sharedlib_only and comp_lnd == "clm" and not "clm4_0" in clm_config_opts:
            logging.info("         - Building clm4_5/clm5_0 Library ")
            esmfdir = "esmf" if use_esmf_lib == "TRUE" else "noesmf"
            sharedpath = os.environ["SHAREDPATH"]
            bldroot = os.path.join(sharedpath, comp_interface, esmfdir)
            objdir = os.path.join(bldroot, "lnd", "obj")
            libdir = os.path.join(bldroot, "lib")
            file_build = os.path.join(exeroot, "lnd.bldlog.%s" %  lid)
            config_lnd_dir = os.path.dirname(case.get_value("CONFIG_LND_FILE"))
            results = []
            for ndir in [bldroot, objdir, libdir]:
                if(not os.path.isdir(ndir)):
                    os.makedirs(ndir)

            _build_model_thread(config_lnd_dir, caseroot, bldroot, "clm", file_build,
                                exeroot, "lnd", "clm", objdir, incroot, results)


    if not sharedlib_only:
        logs.extend(build_model(case, build_threaded, exeroot, clm_config_opts, incroot,
                                comp_atm,   comp_lnd,   comp_ice,   comp_ocn,   comp_glc,   comp_wav,   comp_rof,
                                nthrds_atm, nthrds_lnd, nthrds_ice, nthrds_ocn, nthrds_glc, nthrds_wav, nthrds_rof,
                                lid, caseroot, cimeroot, use_esmf_lib, comp_interface))

    if not sharedlib_only:
        post_build(case, logs)

    t3 = time.time()

    logger.info("Time spent not building: %f sec" % (t2 - t1))
    logger.info("Time spent building: %f sec" % (t3 - t2))