def create_docker_image(filenames, MPI_version=None): log.info("Creating Docker image...") if not MPI_version: MPI_version = "3.2" # log.warning("You have not specified an MPICH version. " # "It is strongly encouraged to request the one installed in your cluster," # " using '--mpi-version X.Y'. Defaulting to MPICH v%s.", MPI_version) dc = get_docker_client() modules = yaml_dump(get_modules(*[load_input(f) for f in filenames])).strip() echos_reqs = "RUN " + " && \\ \n ".join([ r'echo "%s" >> %s' % (block, requirements_file_path) for block in modules.split("\n") ]) echos_help = "RUN " + " && \\ \n ".join([ r'echo "%s" >> %s' % (line, help_file_path) for line in image_help("docker").split("\n") ]) recipe = r""" FROM cobaya/base_mpich_%s:latest %s RUN cobaya-install %s --%s %s --just-code --force ### NEEDS PYTHON UPDATE! --no-progress-bars %s CMD ["cat", "%s"] """ % (MPI_version, echos_reqs, requirements_file_path, _modules_path_arg, _modules_path, echos_help, help_file_path) image_name = "cobaya:" + uuid.uuid4().hex[:6] with StringIO(recipe) as stream: dc.images.build(fileobj=stream, tag=image_name) log.info( "Docker image '%s' created! " "Do 'docker save %s | gzip > some_name.tar.gz'" "to save it to the current folder.", image_name, image_name)
def citation(*infos): print(make_header("This framework", "")) print("[Paper in preparation]\n") for kind, modules in get_modules(*infos).items(): for module in modules: print(make_header(kind, module)) print(get_citation_info(module, kind))
def citation(*infos): blocks_text = odict([["Cobaya", "[Paper in preparation]"]]) for kind, modules in get_modules(*infos).items(): for module in modules: blocks_text["%s:%s" % (kind, module)] = get_citation_info( module, kind) return blocks_text
def create_singularity_image(filenames, MPI_version=None): log.info("Creating Singularity image...") if not MPI_version: MPI_version = "2.1.1" # log.warning("You have not specified an OpenMPI version. " # "It is strongly encouraged to request the one installed in your cluster," # " using '--mpi-version X.Y.Z'. Defaulting to OpenMPI v%s.", MPI_version) modules = yaml_dump(get_modules(*[load_input(f) for f in filenames])).strip() echos_reqs = "\n " + "\n ".join([""] + [ 'echo "%s" >> %s' % (block, requirements_file_path) for block in modules.split("\n") ]) recipe = (dedent(""" Bootstrap: docker From: cobaya/base_openmpi_%s:latest\n %%post\n""" % MPI_version) + dedent(echos_reqs) + dedent(""" export CONTAINED=TRUE cobaya-install %s --%s %s --just-code --force ### --no-progress-bars mkdir %s %%help %s """ % (requirements_file_path, _modules_path, os.path.join(_modules_path_arg, _modules_path, _data), "\n ".join(image_help("singularity").split("\n")[1:])))) with NamedTemporaryFile(delete=False) as recipe_file: recipe_file.write(recipe) recipe_file_name = recipe_file.name image_name = "cobaya_" + uuid.uuid4().hex[:6] + ".simg" process_build = Popen( ["singularity", "build", image_name, recipe_file_name], stdout=PIPE, stderr=PIPE) out, err = process_build.communicate() if process_build.returncode: log.info(out) log.info(err) log.error("Image creation failed! See error message above.") raise HandledException log.info("Singularity image '%s' created!", image_name)
def install(*infos, **kwargs): if not log.root.handlers: logger_setup() path = kwargs.get("path", ".") if not path: # See if we can get one (and only one) from infos paths = set( [p for p in [info.get(_path_install) for info in infos] if p]) if len(paths) == 1: path = paths[0] else: print("logging?") log.error( "No 'path' argument given and could not extract one (and only one) " "from the infos.") raise HandledException abspath = os.path.abspath(path) log.info("Installing modules at '%s'\n", abspath) kwargs_install = { "force": kwargs.get("force", False), "no_progress_bars": kwargs.get("no_progress_bars") } for what in (_code, _data): kwargs_install[what] = kwargs.get(what, True) spath = os.path.join(abspath, what) if kwargs_install[what] and not os.path.exists(spath): try: os.makedirs(spath) except OSError: log.error( "Could not create the desired installation folder '%s'", spath) raise HandledException failed_modules = [] for kind, modules in get_modules(*infos).items(): for module in modules: print(make_header(kind, module)) module_folder = get_folder(module, kind, sep=".", absolute=False) try: imported_module = import_module(module_folder, package=_package) except ImportError: if kind == _likelihood: info = (next(info for info in infos if module in info.get( _likelihood, {}))[_likelihood][module]) or {} if isinstance(info, string_types) or _external in info: log.warning( "Module '%s' is a custom likelihood. " "Nothing to do.\n", module) flag = False else: log.error("Module '%s' not recognized.\n" % module) failed_modules += ["%s:%s" % (kind, module)] continue is_installed = getattr(imported_module, "is_installed", None) if is_installed is None: log.info("Built-in module: nothing to do.\n") continue if is_installed(path=abspath, **kwargs_install): log.info("External module already installed.\n") if kwargs_install["force"]: log.info("Forcing re-installation, as requested.") else: log.info("Doing nothing.\n") continue try: success = imported_module.install(path=abspath, **kwargs_install) except: traceback.print_exception(*sys.exc_info(), file=sys.stdout) log.error( "An unknown error occurred. Delete the modules folder and try " "again. Notify the developers if this error persists.") success = False if success: log.info("Successfully installed!\n") else: log.error( "Installation failed! Look at the error messages above. " "Solve them and try again, or, if you are unable to solve, " "install this module manually.") failed_modules += ["%s:%s" % (kind, module)] continue # test installation if not is_installed(path=abspath, **kwargs_install): log.error( "Installation apparently worked, " "but the subsequent installation test failed! " "Look at the error messages above. " "Solve them and try again, or, if you are unable to solve, " "install this module manually.") failed_modules += ["%s:%s" % (kind, module)] if failed_modules: bullet = "\n - " log.error( "The installation (or installation test) of some module(s) has failed: " "%s\nCheck output of the installer of each module above " "for precise error info.\n", bullet + bullet.join(failed_modules)) raise HandledException
def makeGrid(batchPath, settingName=None, settings=None, read_only=False, interactive=False, install_reqs_at=None, install_reqs_force=None): batchPath = os.path.abspath(batchPath) + os.sep # # 0: chains, 1: importance sampling, 2: best-fit, 3: best-fit and Hessian # cosmomcAction = 0 if not settings: if not settingName: raise NotImplementedError( "Re-using previous batch is work in progress...") # if not pathIsGrid(batchPath): # raise Exception('Need to give name of setting file if batchPath/config ' # 'does not exist') # read_only = True # sys.path.insert(0, batchPath + 'config') # sys.modules['batchJob'] = batchjob # old name # settings = __import__(IniFile(batchPath + 'config/config.ini').params['setting_file'].replace('.py', '')) elif os.path.splitext(settingName)[-1].lower() in (".yml", ".yaml"): settings = yaml_load_file(settingName) else: # ACTUALLY, in the scripted case a DICT or a YAML FILE NAME should be passed raise NotImplementedError( "Using a python script is work in progress...") # settings = __import__(settingName, fromlist=['dummy']) from cobaya.grid_tools import batchjob batch = batchjob.batchJob(batchPath, settings.get("yaml_dir", None)) ### batch.skip = settings.get("skip", False) if "skip" in settings: raise NotImplementedError("Skipping not implemented yet.") batch.makeItems(settings, messages=not read_only) if read_only: for jobItem in [b for b in batch.jobItems]: if not jobItem.chainExists(): batch.jobItems.remove(jobItem) batch.save() print('OK, configured grid with %u existing chains' % (len(batch.jobItems))) return batch else: # WAS batch.makeDirectories(settings.__file__) # WHY THE DIR OF settings AND NOT THE GRID DIR GIVEN??? batch.makeDirectories(setting_file=None) batch.save() # NOT IMPLEMENTED YET: start at best fit!!! # start_at_bestfit = getattr(settings, 'start_at_bestfit', False) defaults = copy.deepcopy(settings) modules_used = {} grid_definition = defaults.pop("grid") models_definitions = grid_definition["models"] datasets_definitions = grid_definition["datasets"] for jobItem in batch.items(wantSubItems=False): jobItem.makeChainPath() base_info = copy.deepcopy(defaults) try: model_info = models_definitions[jobItem.param_set] or {} except KeyError: raise ValueError("Model '%s' must be defined." % jobItem.param_set) # COVMATS NOT IMPLEMENTED YET!!! # cov_dir_name = getattr(settings, 'cov_dir', 'planck_covmats') # covdir = os.path.join(batch.basePath, cov_dir_name) # covmat = os.path.join(covdir, jobItem.name + '.covmat') # if not os.path.exists(covmat): # covNameMappings = getattr(settings, 'covNameMappings', None) # mapped_name_norm = jobItem.makeNormedName(covNameMappings)[0] # covmat_normed = os.path.join(covdir, mapped_name_norm + '.covmat') # covmat = covmat_normed # if not os.path.exists(covmat) and hasattr(jobItem.data_set, # 'covmat'): covmat = batch.basePath + jobItem.data_set.covmat # if not os.path.exists(covmat) and hasattr(settings, 'covmat'): covmat = batch.basePath + settings.covmat # else: # covNameMappings = None # if os.path.exists(covmat): # ini.params['propose_matrix'] = covmat # if getattr(settings, 'newCovmats', True): ini.params['MPI_Max_R_ProposeUpdate'] = 20 # else: # hasCov = False # ini.params['MPI_Max_R_ProposeUpdate'] = 20 # covmat_try = [] # if 'covRenamer' in dir(settings): # covmat_try += settings.covRenamer(jobItem.name) # covmat_try += settings.covRenamer(mapped_name_norm) # if hasattr(settings, 'covrenames'): # for aname in [jobItem.name, mapped_name_norm]: # covmat_try += [aname.replace(old, new, 1) for old, new in settings.covrenames if old in aname] # for new1, old1 in settings.covrenames: # if old1 in aname: # name = aname.replace(old1, new1, 1) # covmat_try += [name.replace(old, new, 1) for old, new in settings.covrenames if old in name] # if 'covWithoutNameOrder' in dir(settings): # if covNameMappings: # removes = copy.deepcopy(covNameMappings) # else: # removes = dict() # for name in settings.covWithoutNameOrder: # if name in jobItem.data_set.names: # removes[name] = '' # covmat_try += [jobItem.makeNormedName(removes)[0]] # covdir2 = os.path.join(batch.basePath, getattr(settings, 'cov_dir_fallback', cov_dir_name)) # for name in covmat_try: # covmat = os.path.join(batch.basePath, covdir2, name + '.covmat') # if os.path.exists(covmat): # ini.params['propose_matrix'] = covmat # print('covmat ' + jobItem.name + ' -> ' + name) # hasCov = True # break # if not hasCov: print('WARNING: no matching specific covmat for ' + jobItem.name) ## NOT IMPLEMENTED: start at best fit ## ini.params['start_at_bestfit'] = start_at_bestfit try: dataset_info = datasets_definitions[jobItem.data_set.tag] except KeyError: raise ValueError("Data set '%s' must be defined." % jobItem.data_set.tag) combined_info = merge_info(base_info, model_info, dataset_info) combined_info[_output_prefix] = jobItem.chainRoot # ??? # for deffile in settings.defaults: # ini.defaults.append(batch.commonPath + deffile) # if hasattr(settings, 'override_defaults'): # ini.defaults = [batch.commonPath + deffile for deffile in settings.override_defaults] + ini.defaults # requisites modules_used = get_modules(modules_used, combined_info) if install_reqs_at: combined_info[_path_install] = os.path.abspath(install_reqs_at) # Write the info for this job yaml_dump_file(combined_info, jobItem.iniFile()) # if not start_at_bestfit: # setMinimize(jobItem, ini) # variant = '_minimize' # ini.saveFile(jobItem.iniFile(variant)) # # add ini files for importance sampling runs # for imp in jobItem.importanceJobs(): # if getattr(imp, 'importanceFilter', None): continue # if batch.hasName(imp.name.replace('_post', '')): # raise Exception('importance sampling something you already have?') # for minimize in (False, True): # if minimize and not getattr(imp, 'want_minimize', True): continue # ini = IniFile() # updateIniParams(ini, imp.importanceSettings, batch.commonPath) # if cosmomcAction == 0 and not minimize: # for deffile in settings.importanceDefaults: # ini.defaults.append(batch.commonPath + deffile) # ini.params['redo_outroot'] = imp.chainRoot # ini.params['action'] = 1 # else: # ini.params['file_root'] = imp.chainRoot # if minimize: # setMinimize(jobItem, ini) # variant = '_minimize' # else: # variant = '' # ini.defaults.append(jobItem.iniFile()) # ini.saveFile(imp.iniFile(variant)) # if cosmomcAction != 0: break # Installing requisites print("Installing required code and data for the grid.") if install_reqs_at: install_reqs(modules_used, path=install_reqs_at, force=install_reqs_force) if not interactive: return batch print('Done... to run do: cobaya-grid-run %s' % batchPath)
def makeGrid(batchPath, settingName=None, settings=None, read_only=False, interactive=False, install_reqs_at=None, install_reqs_force=None): print("Generating grid...") batchPath = os.path.abspath(batchPath) + os.sep if not settings: if not settingName: raise NotImplementedError( "Re-using previous batch is work in progress...") # if not pathIsGrid(batchPath): # raise Exception('Need to give name of setting file if batchPath/config ' # 'does not exist') # read_only = True # sys.path.insert(0, batchPath + 'config') # sys.modules['batchJob'] = batchjob # old name # settings = __import__(IniFile(batchPath + 'config/config.ini').params['setting_file'].replace('.py', '')) elif os.path.splitext(settingName)[-1].lower() in _yaml_extensions: settings = yaml_load_file(settingName) else: raise NotImplementedError( "Using a python script is work in progress...") # In this case, info-as-dict would be passed # settings = __import__(settingName, fromlist=['dummy']) batch = batchjob.batchJob(batchPath, settings.get("yaml_dir", None)) # batch.skip = settings.get("skip", False) batch.makeItems(settings, messages=not read_only) if read_only: for jobItem in [b for b in batch.jobItems]: if not jobItem.chainExists(): batch.jobItems.remove(jobItem) batch.save() print('OK, configured grid with %u existing chains' % (len(batch.jobItems))) return batch else: batch.makeDirectories(setting_file=None) batch.save() infos = {} modules_used = {} # Default info defaults = copy.deepcopy(settings) grid_definition = defaults.pop("grid") models_definitions = grid_definition["models"] datasets_definitions = grid_definition["datasets"] for jobItem in batch.items(wantSubItems=False): # Model info jobItem.makeChainPath() try: model_info = copy.deepcopy(models_definitions[jobItem.param_set] or {}) except KeyError: raise ValueError("Model '%s' must be defined." % jobItem.param_set) model_info = merge_info(defaults, model_info) # Dataset info try: dataset_info = copy.deepcopy( datasets_definitions[jobItem.data_set.tag]) except KeyError: raise ValueError("Data set '%s' must be defined." % jobItem.data_set.tag) # Combined info combined_info = merge_info(defaults, model_info, dataset_info) if "preset" in combined_info: preset = combined_info.pop("preset") combined_info = merge_info(create_input(**preset), combined_info) combined_info[_output_prefix] = jobItem.chainRoot # Requisites modules_used = get_modules(modules_used, combined_info) if install_reqs_at: combined_info[_path_install] = os.path.abspath(install_reqs_at) # Save the info (we will write it after installation: # we need to install to add auto covmats if jobItem.param_set not in infos: infos[jobItem.param_set] = {} infos[jobItem.param_set][jobItem.data_set.tag] = combined_info # Installing requisites if install_reqs_at: print("Installing required code and data for the grid.") from cobaya.log import logger_setup logger_setup() install_reqs(modules_used, path=install_reqs_at, force=install_reqs_force) print("Adding covmats (if necessary) and writing input files") for jobItem in batch.items(wantSubItems=False): info = infos[jobItem.param_set][jobItem.data_set.tag] # Covariance matrices # We try to find them now, instead of at run time, to check if correctly selected try: sampler = list(info[_sampler])[0] except KeyError: raise ValueError("No sampler has been chosen") if sampler == "mcmc" and info[_sampler][sampler].get("covmat", "auto"): modules_path = install_reqs_at or info.get(_path_install, None) if not modules_path: raise ValueError( "Cannot assign automatic covariance matrices because no " "modules path has been defined.") # Need full info for covmats: includes renames full_info = get_full_info(info) # Ideally, we use slow+sampled parameters to look for the covariance matrix # but since for that we'd need to initialise a model, we approximate that set # as theory+sampled from itertools import chain like_params = set( chain(*[ list(like[_params]) for like in full_info[_likelihood].values() ])) params_info = { p: v for p, v in full_info[_params].items() if is_sampled_param(v) and p not in like_params } best_covmat = get_best_covmat(os.path.abspath(modules_path), params_info, full_info[_likelihood]) info[_sampler][sampler]["covmat"] = os.path.join( best_covmat["folder"], best_covmat["name"]) # Write the info for this job try: yaml_dump_file(jobItem.iniFile(), info, error_if_exists=True) except IOError: raise IOError( "Can't write chain input file. Maybe the chain configuration " "files already exists?") # Non-translated old code # if not start_at_bestfit: # setMinimize(jobItem, ini) # variant = '_minimize' # ini.saveFile(jobItem.iniFile(variant)) ## NOT IMPLEMENTED: start at best fit ## ini.params['start_at_bestfit'] = start_at_bestfit # --- # for deffile in settings.defaults: # ini.defaults.append(batch.commonPath + deffile) # if hasattr(settings, 'override_defaults'): # ini.defaults = [batch.commonPath + deffile for deffile in settings.override_defaults] + ini.defaults # --- # # add ini files for importance sampling runs # for imp in jobItem.importanceJobs(): # if getattr(imp, 'importanceFilter', None): continue # if batch.hasName(imp.name.replace('_post', '')): # raise Exception('importance sampling something you already have?') # for minimize in (False, True): # if minimize and not getattr(imp, 'want_minimize', True): continue # ini = IniFile() # updateIniParams(ini, imp.importanceSettings, batch.commonPath) # if cosmomcAction == 0 and not minimize: # for deffile in settings.importanceDefaults: # ini.defaults.append(batch.commonPath + deffile) # ini.params['redo_outroot'] = imp.chainRoot # ini.params['action'] = 1 # else: # ini.params['file_root'] = imp.chainRoot # if minimize: # setMinimize(jobItem, ini) # variant = '_minimize' # else: # variant = '' # ini.defaults.append(jobItem.iniFile()) # ini.saveFile(imp.iniFile(variant)) # if cosmomcAction != 0: break if not interactive: return batch print('Done... to run do: cobaya-grid-run %s' % batchPath)
def install(*infos, **kwargs): path = kwargs.get("path", ".") abspath = os.path.abspath(path) kwargs_install = { "force": kwargs.get("force", False), "no_progress_bars": kwargs.get("no_progress_bars") } for what in (_code, _data): kwargs_install[what] = kwargs.get(what, True) spath = os.path.join(abspath, what) if kwargs_install[what] and not os.path.exists(spath): try: os.makedirs(spath) except OSError: log.error( "Could not create the desired installation folder '%s'", spath) raise HandledException failed_modules = [] for kind, modules in get_modules(*infos).items(): for module in modules: print(make_header(kind, module)) module_folder = get_folder(module, kind, sep=".", absolute=False) try: imported_module = import_module(module_folder, package=package) except ImportError: if kind == _likelihood: log.warn( "Module '%s' not recognised. Assuming it's a custom " "likelihood. Nothing to do.\n", module) else: log.error("Module '%s' not recognised.\n" % module) failed_modules += ["%s:%s" % (kind, module)] continue is_installed = getattr(imported_module, "is_installed", None) if is_installed is None: log.info("Built-in module: nothing to do.\n") continue if is_installed(path=abspath, **kwargs_install): log.info("External module already installed.\n") if kwargs_install["force"]: log.info("Forcing re-installation, as requested.") else: log.info("Doing nothing.\n") continue try: success = imported_module.install(path=abspath, **kwargs_install) except: traceback.print_exception(*sys.exc_info(), file=sys.stdout) log.error( "An unknown error occurred. Delete the modules folder and try " "again. Notify the developers if this error persists.") success = False if success: log.info("Successfully installed!\n") else: log.error( "Installation failed! Look at the error messages above. " "Solve them and try again, or, if you are unable to solve, " "install this module manually.") failed_modules += ["%s:%s" % (kind, module)] continue # test installation if not is_installed(path=abspath, **kwargs_install): log.error( "Installation apparently worked, " "but the subsequent installation test failed! " "Look at the error messages above. " "Solve them and try again, or, if you are unable to solve, " "install this module manually.") failed_modules += ["%s:%s" % (kind, module)] if failed_modules: log.error( "The instalation (or installation test) of some module(s) has failed: " "%r. Check output above.\n", failed_modules) raise HandledException