Пример #1
0
def makeGrid(batchPath,
             settingName=None,
             settings=None,
             readOnly=False,
             interactive=False):
    batchPath = os.path.abspath(batchPath) + os.sep

    # 0: chains, 1: importance sampling, 2: best-fit, 3: best-fit and Hessian
    cosmomcAction = 0

    if not settings:
        if not settingName:
            if not pathIsGrid(batchPath):
                raise Exception(
                    'Need to give name of setting file if batchPath/config does not exist'
                )
            readOnly = True
            sys.path.insert(0, batchPath + 'config')
            sys.modules['batchJob'] = batchjob  # old name
            settings = __import__(
                IniFile(batchPath +
                        'config/config.ini').params['setting_file'].replace(
                            '.py', ''))
        else:
            settings = __import__(settingName, fromlist=['dummy'])

    batch = batchjob.batchJob(batchPath, settings.ini_dir)

    if hasattr(settings, 'skip'): batch.skip = settings.skip
    batch.makeItems(settings, messages=not readOnly)
    if readOnly:
        for jobItem in [b for b in batch.jobItems]:
            if not jobItem.chainExists():
                batch.jobItems.remove(jobItem)
        batch.save()
        print('OK, configured grid with %u existing chains' %
              (len(batch.jobItems)))
        return batch
    else:
        batch.makeDirectories(settings.__file__)
        batch.save()

    # priors and widths for parameters which are varied
    start_at_bestfit = getattr(settings, 'start_at_bestfit', False)
    params = getattr(settings, 'params', default_params)
    param_extra = getattr(settings, 'param_extra_opts',
                          default_param_extra_opts)

    for jobItem in batch.items(wantSubItems=False):

        jobItem.makeChainPath()
        ini = IniFile()

        for param in jobItem.param_set:
            ini.params['param[' + param + ']'] = params[param]
            if param_extra is not None and param in param_extra:
                ini.params.update(param_extra[param])

        if hasattr(settings, 'extra_opts'):
            ini.params.update(settings.extra_opts)

        ini.params['file_root'] = jobItem.chainRoot

        cov_dir_name = getattr(settings, 'cov_dir', 'planck_covmats')
        covdir = os.path.join(batch.basePath, cov_dir_name)
        covmat = os.path.join(covdir, jobItem.name + '.covmat')
        if not os.path.exists(covmat):
            covNameMappings = getattr(settings, 'covNameMappings', None)
            mapped_name_norm = jobItem.makeNormedName(covNameMappings)[0]
            covmat_normed = os.path.join(covdir, mapped_name_norm + '.covmat')
            covmat = covmat_normed
            if not os.path.exists(covmat) and hasattr(jobItem.data_set,
                                                      'covmat'):
                covmat = batch.basePath + jobItem.data_set.covmat
            if not os.path.exists(covmat) and hasattr(settings, 'covmat'):
                covmat = batch.basePath + settings.covmat
        else:
            covNameMappings = None
        if os.path.exists(covmat):
            ini.params['propose_matrix'] = covmat
            if getattr(settings, 'newCovmats', True):
                ini.params['MPI_Max_R_ProposeUpdate'] = 20
        else:
            hasCov = False
            ini.params['MPI_Max_R_ProposeUpdate'] = 20
            covmat_try = []
            if 'covRenamer' in dir(settings):
                covmat_try += settings.covRenamer(jobItem.name)
                covmat_try += settings.covRenamer(mapped_name_norm)
            if hasattr(settings, 'covrenames'):
                for aname in [jobItem.name, mapped_name_norm]:
                    covmat_try += [
                        aname.replace(old, new, 1)
                        for old, new in settings.covrenames if old in aname
                    ]
                    for new1, old1 in settings.covrenames:
                        if old1 in aname:
                            name = aname.replace(old1, new1, 1)
                            covmat_try += [
                                name.replace(old, new, 1)
                                for old, new in settings.covrenames
                                if old in name
                            ]
            if 'covWithoutNameOrder' in dir(settings):
                if covNameMappings:
                    removes = copy.deepcopy(covNameMappings)
                else:
                    removes = dict()
                for name in settings.covWithoutNameOrder:
                    if name in jobItem.data_set.names:
                        removes[name] = ''
                        covmat_try += [jobItem.makeNormedName(removes)[0]]
            covdir2 = os.path.join(
                batch.basePath,
                getattr(settings, 'cov_dir_fallback', cov_dir_name))
            for name in covmat_try:
                covmat = os.path.join(batch.basePath, covdir2,
                                      name + '.covmat')
                if os.path.exists(covmat):
                    ini.params['propose_matrix'] = covmat
                    print('covmat ' + jobItem.name + ' -> ' + name)
                    hasCov = True
                    break
            if not hasCov:
                print('WARNING: no matching specific covmat for ' +
                      jobItem.name)

        ini.params['start_at_bestfit'] = start_at_bestfit
        updateIniParams(ini, jobItem.data_set.params, batch.commonPath)
        for deffile in settings.defaults:
            ini.defaults.append(batch.commonPath + deffile)
        if hasattr(settings, 'override_defaults'):
            ini.defaults = [
                batch.commonPath + deffile
                for deffile in settings.override_defaults
            ] + ini.defaults

        ini.params['action'] = cosmomcAction
        ini.saveFile(jobItem.iniFile())
        if not start_at_bestfit:
            setMinimize(jobItem, ini)
            variant = '_minimize'
            ini.saveFile(jobItem.iniFile(variant))

            # add ini files for importance sampling runs
        for imp in jobItem.importanceJobs():
            if not getattr(jobItem, 'importanceFilter', None): continue
            if batch.hasName(imp.name.replace('_post', '')):
                raise Exception(
                    'importance sampling something you already have?')
            for minimize in (False, True):
                if minimize and not getattr(imp, 'want_minimize', True):
                    continue
                ini = IniFile()
                updateIniParams(ini, imp.importanceSettings, batch.commonPath)
                if cosmomcAction == 0 and not minimize:
                    for deffile in settings.importanceDefaults:
                        ini.defaults.append(batch.commonPath + deffile)
                    ini.params['redo_outroot'] = imp.chainRoot
                    ini.params['action'] = 1
                else:
                    ini.params['file_root'] = imp.chainRoot
                if minimize:
                    setMinimize(jobItem, ini)
                    variant = '_minimize'
                else:
                    variant = ''
                ini.defaults.append(jobItem.iniFile())
                ini.saveFile(imp.iniFile(variant))
                if cosmomcAction != 0: break

    if not interactive: return batch
    print('Done... to run do: python python/runbatch.py ' + batchPath)
    if not start_at_bestfit:
        print('....... for best fits: python python/runbatch.py ' + batchPath +
              ' --minimize')
    print('')
    print('for importance sampled: python python/runbatch.py ' + batchPath +
          ' --importance')
    print('for best-fit for importance sampled: python python/runbatch.py ' +
          batchPath + ' --importance_minimize')
Пример #2
0
def makeGrid(batchPath, settingName=None, settings=None, readOnly=False, interactive=False):
    batchPath = os.path.abspath(batchPath) + os.sep

    # 0: chains, 1: importance sampling, 2: best-fit, 3: best-fit and Hessian
    cosmomcAction = 0

    if not settings:
        if not settingName:
            if not pathIsGrid(batchPath):
                raise Exception('Need to give name of setting file if batchPath/config does not exist')
            readOnly = True
            sys.path.insert(0, batchPath + 'config')
            sys.modules['batchJob'] = batchjob  # old name
            settings = __import__(IniFile(batchPath + 'config/config.ini').params['setting_file'].replace('.py', ''))
        else:
            settings = __import__(settingName, fromlist=['dummy'])

    batch = batchjob.batchJob(batchPath, settings.ini_dir)

    if hasattr(settings, 'skip'): batch.skip = settings.skip
    batch.makeItems(settings, messages=not readOnly)
    if readOnly:
        for jobItem in [b for b in batch.jobItems]:
            if not jobItem.chainExists():
                batch.jobItems.remove(jobItem)
        batch.save()
        print('OK, configured grid with %u existing chains' % (len(batch.jobItems)))
        return batch
    else:
        batch.makeDirectories(settings.__file__)
        batch.save()

    # priors and widths for parameters which are varied
    start_at_bestfit = getattr(settings, 'start_at_bestfit', False)
    params = getattr(settings, 'params', default_params)
    param_extra = getattr(settings, 'param_extra_opts', default_param_extra_opts)

    for jobItem in batch.items(wantSubItems=False):

        jobItem.makeChainPath()
        ini = IniFile()

        for param in jobItem.param_set:
            ini.params['param[' + param + ']'] = params[param]
            if param_extra is not None and param in param_extra:
                ini.params.update(param_extra[param])

        if hasattr(settings, 'extra_opts'):
            ini.params.update(settings.extra_opts)

        ini.params['file_root'] = jobItem.chainRoot

        cov_dir_name = getattr(settings, 'cov_dir', 'planck_covmats')
        covdir = os.path.join(batch.basePath, cov_dir_name)
        covmat = os.path.join(covdir, jobItem.name + '.covmat')
        if not os.path.exists(covmat):
            covNameMappings = getattr(settings, 'covNameMappings', None)
            mapped_name_norm = jobItem.makeNormedName(covNameMappings)[0]
            covmat_normed = os.path.join(covdir, mapped_name_norm + '.covmat')
            covmat = covmat_normed
            if not os.path.exists(covmat) and hasattr(jobItem.data_set,
                                                      'covmat'): covmat = batch.basePath + jobItem.data_set.covmat
            if not os.path.exists(covmat) and hasattr(settings, 'covmat'): covmat = batch.basePath + settings.covmat
        else:
            covNameMappings = None
        if os.path.exists(covmat):
            ini.params['propose_matrix'] = covmat
            if getattr(settings, 'newCovmats', True): ini.params['MPI_Max_R_ProposeUpdate'] = 20
        else:
            hasCov = False
            ini.params['MPI_Max_R_ProposeUpdate'] = 20
            covmat_try = []
            if 'covRenamer' in dir(settings):
                covmat_try += settings.covRenamer(jobItem.name)
                covmat_try += settings.covRenamer(mapped_name_norm)
            if hasattr(settings, 'covrenames'):
                for aname in [jobItem.name, mapped_name_norm]:
                    covmat_try += [aname.replace(old, new, 1) for old, new in settings.covrenames if old in aname]
                    for new1, old1 in settings.covrenames:
                        if old1 in aname:
                            name = aname.replace(old1, new1, 1)
                            covmat_try += [name.replace(old, new, 1) for old, new in settings.covrenames if old in name]
            if 'covWithoutNameOrder' in dir(settings):
                if covNameMappings:
                    removes = copy.deepcopy(covNameMappings)
                else:
                    removes = dict()
                for name in settings.covWithoutNameOrder:
                    if name in jobItem.data_set.names:
                        removes[name] = ''
                        covmat_try += [jobItem.makeNormedName(removes)[0]]
            covdir2 = os.path.join(batch.basePath, getattr(settings, 'cov_dir_fallback', cov_dir_name))
            for name in covmat_try:
                covmat = os.path.join(batch.basePath, covdir2, name + '.covmat')
                if os.path.exists(covmat):
                    ini.params['propose_matrix'] = covmat
                    print('covmat ' + jobItem.name + ' -> ' + name)
                    hasCov = True
                    break
            if not hasCov: print('WARNING: no matching specific covmat for ' + jobItem.name)

        ini.params['start_at_bestfit'] = start_at_bestfit
        updateIniParams(ini, jobItem.data_set.params, batch.commonPath)
        for deffile in settings.defaults:
            ini.defaults.append(batch.commonPath + deffile)
        if hasattr(settings, 'override_defaults'):
            ini.defaults = [batch.commonPath + deffile for deffile in settings.override_defaults] + ini.defaults

        ini.params['action'] = cosmomcAction
        ini.saveFile(jobItem.iniFile())
        if not start_at_bestfit:
            setMinimize(jobItem, ini)
            variant = '_minimize'
            ini.saveFile(jobItem.iniFile(variant))


            # add ini files for importance sampling runs
        for imp in jobItem.importanceJobs():
            if batch.hasName(imp.name.replace('_post', '')): raise Exception(
                'importance sampling something you already have?')
            for minimize in (False, True):
                if minimize and not getattr(imp, 'want_minimize', True): continue
                ini = IniFile()
                updateIniParams(ini, imp.importanceSettings, batch.commonPath)
                if cosmomcAction == 0 and not minimize:
                    for deffile in settings.importanceDefaults:
                        ini.defaults.append(batch.commonPath + deffile)
                    ini.params['redo_outroot'] = imp.chainRoot
                    ini.params['action'] = 1
                else:
                    ini.params['file_root'] = imp.chainRoot
                if minimize:
                    setMinimize(jobItem, ini)
                    variant = '_minimize'
                else:
                    variant = ''
                ini.defaults.append(jobItem.iniFile())
                ini.saveFile(imp.iniFile(variant))
                if cosmomcAction != 0: break

    if not interactive: return batch
    print('Done... to run do: python python/runbatch.py ' + batchPath)
    if not start_at_bestfit:
        print('....... for best fits: python python/runbatch.py ' + batchPath + ' --minimize')
    print('')
    print('for importance sampled: python python/runbatch.py ' + batchPath + ' --importance')
    print('for best-fit for importance sampled: python python/runbatch.py ' + batchPath + ' --importance_minimize')