def __init__(self, file_manager: HistFactoryFileManager, verbose: IntBool = True) -> None: """ """ # Declaration of needed types for attributes self.likelihoods_dict: Dict[int, Dict[str, Any]] = {} self.regions: Dict[str, Path] self.log: LogPredDict self.name: str # Initialization of parent class super().__init__(verbose) # Initialization of verbosity mode verbose, verbose_sub = self.set_verbosity(self.verbose) # Initialization of object timestamp = utils.generate_timestamp() print(header_string, "\nInitialize Histfactory object.\n", show=verbose) self.file_manager = file_manager self.predictions = HistfactoryPredictions(self.file_manager.obj_name, verbose=verbose_sub) self.figures = Figures(verbose=verbose_sub) self.regions_folders_base_name = self.file_manager.regions_folders_base_name self.bkg_files_base_name = self.file_manager.bkg_files_base_name self.patch_files_base_name = self.file_manager.patch_files_base_name if self.file_manager.input_file is None: self.log = {timestamp: {"action": "created"}} self.name = self.file_manager.name.name subfolders = [ self.file_manager.workspace_folder.joinpath(f) for f in self.file_manager.workspace_folder.iterdir() if self.file_manager.workspace_folder.joinpath(f).is_dir() ] regions = [ str(f).replace(str(self.regions_folders_base_name), "") for f in self.file_manager.workspace_folder.iterdir() if self.file_manager.workspace_folder.joinpath(f).is_dir() ] self.regions = dict(zip(regions, subfolders)) self.likelihoods_dict = {} self.__build_likelihoods_dict( verbose=verbose_sub) # fills self.likelihoods_dict dictionary self.save(overwrite=False, verbose=verbose_sub) else: self.__load(verbose=verbose_sub) self.save_log(overwrite=True, verbose=verbose_sub)
def __load(self, obj: "Histfactory", verbose: Optional[IntBool] = None) -> None: verbose, _ = self.set_verbosity(verbose) start = timer() obj_dict = dd.io.load(self.input_h5_file) self.regions_folders_base_name = obj_dict["regions_folders_base_name"] self.bkg_files_base_name = obj_dict["bkg_files_base_name"] self.patch_files_base_name = obj_dict["patch_files_base_name"] with self.input_log_file.open() as json_file: obj.log = json.load(json_file) obj.__dict__.update(obj_dict) end = timer() timestamp = utils.generate_timestamp() obj.log[timestamp] = { "action": "loaded", "files names": [self.input_h5_file.name, self.input_log_file.name] } print(header_string, "\nHistfactory object loaded in", str(end - start), ".\n", show=verbose)
def __check_workspace_folder(self, verbose: Optional[Union[int, bool]] = None ) -> None: verbose, _ = self.set_verbosity(verbose) self.workspace_folder = self.output_folder.joinpath( "histfactory_workspace") if self.workspace_folder.exists(): print(header_string, "\nHistfactory Workspace folder\n\t", str(self.workspace_folder), "\nalready present in the output folder.\n", show=verbose) elif self._workspace_folder is not None: shutil.copytree(self._workspace_folder, self.workspace_folder) print(header_string, "\nHistfactory Workspace folder\n\t", str(self._workspace_folder), "\ncopied into the folder\n\t", str(self.workspace_folder), ".\n", show=verbose) else: raise Exception("No workspace folder specified.")
def get_likelihood_object( self, lik_number: Optional[int] = None, output_folder: Optional[StrPath] = None, verbose: Optional[Union[int, bool]] = None) -> Lik: """ Generates a :class:`Lik <DNNLikelihood.Lik>` object (see its documentation for details) containing all properties needed for further processing. If the ``lik_number`` argument corresponds to a likelihood that has not jet been imported with the :meth:`Histfactory.import_likelihoods <DNNLikelihood.Histfactory.import_likelihoods>` method, then this method is called and the likelihood is automatically imported. - **Arguments** - **lik_number** Number of the likelihood for which the :class:`Lik <DNNLikelihood.Lik>` object is created. - **type**: ``int`` - **default**: ``0`` - **output_folder** Folder where the output :class:`Lik <DNNLikelihood.Lik>` object is saved. If specified is passed as input to the :class:`Lik <DNNLikelihood.Lik>` object, otherwise the :attr:`Histfactory.output_folder <DNNLikelihood.Histfactory.output_folder>` is passed and the :class:`Lik <DNNLikelihood.Lik>` object is saved in the same folder as the :class:`Histfactory <DNNLikelihood.Histfactory>` object. - **type**: ``str`` - **default**: ``None`` - **verbose** See :argument:`verbose <common_methods_arguments.verbose>`. - **Returns** :class:`Lik <DNNLikelihood.Lik>` object. - **Creates files** - :attr:`Lik.output_h5_file <DNNLikelihood.Lik.output_h5_file>` - :attr:`Lik.output_json_file <DNNLikelihood.Lik.output_json_file>` - :attr:`Lik.output_log_file <DNNLikelihood.Lik.output_log_file>` - **Updates file** - :attr:`Histfactory.output_log_file <DNNLikelihood.Histfactory.output_log_file>` """ verbose, verbose_sub = self.set_verbosity(verbose) print(header_string, "\nCreating 'Lik' object\n", show=verbose) start = timer() if lik_number is None: raise Exception( "Please specify the ``lik_number`` argument to create a Likelihood object." ) if output_folder is None: output_folder = self.file_manager.output_folder lik = dict(self.likelihoods_dict[lik_number]) if not lik["model_loaded"]: print(header_string, "\nModel for likelihood", lik_number, "not loaded. Attempting to load it.\n") self.import_likelihoods(lik_list=[lik_number], verbose=True) lik = dict(self.likelihoods_dict[lik_number]) lik_file_manager = LikFileManager(name=lik["name"].replace( "_histfactory", ""), input_file=None, output_folder=output_folder, verbose=self.verbose) lik_logpdf = LogPDF(logpdf=lik["model"].logpdf, logpdf_args=[lik["obs_data"]], logpdf_kwargs=None) lik_pars_manager = LikParsManager(pars_central=lik["pars_central"], pars_pos_poi=lik["pars_pos_poi"], pars_pos_nuis=lik["pars_pos_nuis"], pars_labels=lik["pars_labels"], pars_bounds=lik["pars_bounds"], logpdf=lik_logpdf, verbose=self.verbose) lik_obj = Lik(file_manager=lik_file_manager, logpdf=lik_logpdf, parameters=lik_pars_manager, verbose=self.verbose) end = timer() timestamp = utils.generate_timestamp() self.log[timestamp] = { "action": "saved likelihood object", "likelihood number": lik_number, "file names": [ lik_obj.file_manager.output_h5_file.name, lik_obj.file_manager.output_log_file.name ] } print(header_string, "\nLik object for likelihood", lik_number, "created and saved in", str(end - start), "s.\n", show=verbose) self.save_log(overwrite=True, verbose=verbose_sub) return lik_obj
def import_likelihoods(self, lik_list: Optional[List] = None, progressbar: bool = True, verbose: Optional[Union[int, bool]] = None) -> None: """ """ verbose, verbose_sub = self.set_verbosity(verbose) timestamp = utils.generate_timestamp() start = timer() iterator = 0 overall_progress = None if lik_list is None: lik_list = list(self.likelihoods_dict.keys()) else: lik_list = list(lik_list) if len(lik_list) == 1: progressbar = False if progressbar: try: import ipywidgets as widgets # type: ignore overall_progress = widgets.FloatProgress( value=0.0, min=0.0, max=1.0, layout={ "width": "500px", "height": "14px", "padding": "0px", "margin": "-5px 0px -20px 0px" }) display(overall_progress) except: progressbar = False print( header_string, "\nIf you want to show a progress bar please install the ipywidgets package.\n", show=verbose) for n in lik_list: patch_file = self.file_manager.workspace_folder.joinpath( self.regions[self.likelihoods_dict[n]["signal_region"]], self.likelihoods_dict[n]["patch_file"]) bg_only_file = self.file_manager.workspace_folder.joinpath( self.regions[self.likelihoods_dict[n]["signal_region"]], self.likelihoods_dict[n]["bg_only_file"]) if self.likelihoods_dict[n]["model_loaded"]: print(header_string, "\nFile\n\t", patch_file, "\nis already loaded.\n", show=verbose) else: print(header_string, "\nLoading\n\t", patch_file, "\npatch file.\n", show=verbose) start_patch = timer() [bgonly, patch] = self.file_manager.__import_patch( bg_only_file=bg_only_file, patch_file=patch_file) spec = jsonpatch.apply_patch(bgonly, patch) ws = pyhf.Workspace(spec) model = ws.model() self.likelihoods_dict[n]["model"] = model pars_mapping: Dict[int, str] = {} pars_settings: Dict[int, str] = {} ii = 0 for k, v in model.config.par_map.items(): pars_int = range(model.config.npars) for j in list(pars_int[v["slice"]]): pars_mapping = { **pars_mapping, **{ ii: k + "_" + str(j) } } pars_settings = { **pars_settings, **{ ii: model.config.suggested_bounds()[ii] } } ii = ii + 1 obs_data = pyhf.tensorlib.astensor(ws.data(model)) self.likelihoods_dict[n]["obs_data"] = obs_data self.likelihoods_dict[n]["pars_central"] = np.array( model.config.suggested_init()) self.likelihoods_dict[n]["pars_bounds"] = np.array( list(pars_settings.values())) self.likelihoods_dict[n]["pars_labels"] = list( pars_mapping.values()) self.likelihoods_dict[n]["pars_pos_poi"] = np.array( [model.config.poi_index]).flatten() self.likelihoods_dict[n]["pars_pos_nuis"] = np.array([ i for i in range( len(self.likelihoods_dict[n]["pars_central"])) if i not in np.array([model.config.poi_index ]).flatten().tolist() ]) self.likelihoods_dict[n]["model_loaded"] = True schema = requests.get( "https://scikit-hep.org/pyhf/schemas/1.0.0/workspace.json" ).json() jsonschema.validate(instance=spec, schema=schema) end_patch = timer() print(header_string, "\nFile\n\t", patch_file, "\nprocessed in", str(end_patch - start_patch), "s.\n", show=verbose) if progressbar and overall_progress is not None: iterator = iterator + 1 overall_progress.value = float(iterator) / (len(lik_list)) end = timer() self.log[timestamp] = { "action": "imported likelihoods", "likelihoods numbers": lik_list } self.save_log(overwrite=True, verbose=verbose_sub) print(header_string, "\nImported", len(lik_list), "likelihoods in ", str(end - start), "s.\n", show=verbose)
def __build_likelihoods_dict(self, verbose: Optional[IntBool] = None) -> None: """ Private method used by the :meth:`Histfactory.__init__ <DNNLikelihood.Histfactory.__init__>` one to import an histfactory workspace. The method scans through the regions folders in the :attr:`Histfactory.workspace_folder <DNNLikelihood.Histfactory.workspace_folder>`, determines all background and signal (patch) files and generates the dictionary attribute :attr:`Histfactory.likelihoods_dict <DNNLikelihood.Histfactory.likelihoods_dict>`. Upon creation, all items in this dictionary, corresponding to all available likelihoods, have the flag ``load_model = False``. - **Arguments** - **verbose** See :argument:`verbose <common_methods_arguments.verbose>`. """ verbose, _ = self.set_verbosity(verbose) timestamp = utils.generate_timestamp() for region in self.regions.keys(): region_path = self.regions[region] regionfiles = [ f for f in region_path.iterdir() if region_path.joinpath(f).is_file() ] bgonly_files = [ x for x in regionfiles if self.bkg_files_base_name in str(x) ] signal_patch_files = [ x for x in regionfiles if self.patch_files_base_name in str(x) ] if len(bgonly_files) > 0 and len(signal_patch_files) > 0: bgonly_file = bgonly_files[0] range_region = list( range( len(self.likelihoods_dict.keys()), len(self.likelihoods_dict.keys()) + len(signal_patch_files))) dict_region = dict( zip(range_region, [{ "signal_region": region, "bg_only_file": bgonly_file, "patch_file": signal_patch_file, "name": "region_" + region + "_patch_" + signal_patch_file.stem.replace( self.patch_files_base_name + ".", "").split(".")[0], "model_loaded": False } for signal_patch_file in signal_patch_files])) self.likelihoods_dict.update(dict_region) else: print( header_string, "\nLikelihoods import from folder", self.regions[region], " failed. Please check background and patch files base name.\n", show=verbose) for n in list(self.likelihoods_dict.keys()): self.likelihoods_dict[n]["name"] = self.name + "_" + str( n) + "_" + self.likelihoods_dict[n]["name"] + "_likelihood" self.log[timestamp] = { "action": "imported histfactory workspace", "folder": self.file_manager.workspace_folder.name } print(header_string, "\nSuccessfully imported", len(list(self.likelihoods_dict.keys())), "likelihoods from", len(list(self.regions.keys())), "regions.\n", show=verbose)