class DiffuseModelManager(object): """ Small helper class to keep track of diffuse component templates This keeps track of the 'diffuse component infomation' dictionary This keyed by: key = {source_name}_{source_ver} Where: {source_name} is something like 'loopI' {source_ver} is somthinng like v00 The dictioary is diffuse_comp_info_dict[key] - > `model_component.ModelComponentInfo` Note that some components ( those that represent moving sources or are selection depedent ) will have a sub-dictionary of diffuse_comp_info_dict object for each sub-component The compoents are defined in a file called config/diffuse_components.yaml """ def __init__(self, **kwargs): """ C'tor Keyword arguments ----------------- name_policy : str Name of yaml file contain file naming policy definitions basedir : str Top level directory for finding files """ self._name_factory = NameFactory(basedir=kwargs.get('basedir')) self._diffuse_comp_info_dict = {} @staticmethod def read_diffuse_component_yaml(yamlfile): """ Read the yaml file for the diffuse components """ diffuse_components = yaml.safe_load(open(yamlfile)) return diffuse_components def sourcekeys(self): """Return the list of source keys""" return sorted(self._diffuse_comp_info_dict.keys()) def diffuse_comp_info(self, sourcekey): """Return the Component info associated to a particular key """ return self._diffuse_comp_info_dict[sourcekey] def make_template_name(self, model_type, sourcekey): """ Make the name of a template file for particular component Parameters ---------- model_type : str Type of model to use for this component sourcekey : str Key to identify this component Returns filename or None if component does not require a template file """ format_dict = self.__dict__.copy() format_dict['sourcekey'] = sourcekey if model_type == 'IsoSource': return self._name_factory.spectral_template(**format_dict) elif model_type == 'MapCubeSource': return self._name_factory.diffuse_template(**format_dict) else: raise ValueError("Unexpected model_type %s" % model_type) def make_xml_name(self, sourcekey): """ Make the name of an xml file for a model definition of a single component Parameters ---------- sourcekey : str Key to identify this component """ format_dict = self.__dict__.copy() format_dict['sourcekey'] = sourcekey return self._name_factory.srcmdl_xml(**format_dict) def make_diffuse_comp_info(self, source_name, source_ver, diffuse_dict, components=None, comp_key=None): """ Make a dictionary mapping the merged component names to list of template files Parameters ---------- source_name : str Name of the source source_ver : str Key identifying the version of the source diffuse_dict : dict Information about this component comp_key : str Used when we need to keep track of sub-components, i.e., for moving and selection dependent sources. Returns `model_component.ModelComponentInfo` or `model_component.IsoComponentInfo` """ model_type = diffuse_dict['model_type'] sourcekey = '%s_%s' % (source_name, source_ver) if comp_key is None: template_name = self.make_template_name(model_type, sourcekey) srcmdl_name = self.make_xml_name(sourcekey) else: template_name = self.make_template_name( model_type, "%s_%s" % (sourcekey, comp_key)) srcmdl_name = self.make_xml_name("%s_%s" % (sourcekey, comp_key)) template_name = self._name_factory.fullpath(localpath=template_name) srcmdl_name = self._name_factory.fullpath(localpath=srcmdl_name) kwargs = dict(source_name=source_name, source_ver=source_ver, model_type=model_type, srcmdl_name=srcmdl_name, components=components, comp_key=comp_key) kwargs.update(diffuse_dict) if model_type == 'IsoSource': kwargs['Spectral_Filename'] = template_name return IsoComponentInfo(**kwargs) elif model_type == 'MapCubeSource': kwargs['Spatial_Filename'] = template_name return MapCubeComponentInfo(**kwargs) else: raise ValueError("Unexpected model type %s" % model_type) def make_diffuse_comp_info_dict(self, diffuse_sources, components): """ Make a dictionary maping from diffuse component to information about that component Parameters ---------- diffuse_sources : dict Dictionary with diffuse source defintions components : dict Dictionary with event selection defintions, needed for selection depenedent diffuse components Returns ------- ret_dict : dict Dictionary mapping sourcekey to `model_component.ModelComponentInfo` """ ret_dict = {} for key, value in diffuse_sources.items(): model_type = value.get('model_type', 'mapcube') if model_type == 'galprop_rings': continue selection_dependent = value.get('selection_dependent', False) moving = value.get('moving', False) versions = value.get('versions', []) for version in versions: #sourcekey = self._name_factory.sourcekey(source_name=key, # source_ver=version) comp_dict = None if selection_dependent: # For selection dependent diffuse sources we need to split # by binning component comp_dict = {} for comp in components: comp_key = comp.make_key('{ebin_name}_{evtype_name}') comp_dict[comp_key] = self.make_diffuse_comp_info( key, version, value, None, comp_key) elif moving: # For moving diffuse sources we need to split by zmax cut comp_dict = {} zmax_dict = {} for comp in components: zmax_dict[int(comp.zmax)] = True zmax_list = sorted(zmax_dict.keys()) for zmax in zmax_list: comp_key = "zmax%i" % (zmax) comp_dict[comp_key] = self.make_diffuse_comp_info( key, version, value, None, comp_key) comp_info = self.make_diffuse_comp_info( key, version, value, comp_dict) ret_dict[comp_info.sourcekey] = comp_info self._diffuse_comp_info_dict.update(ret_dict) return ret_dict
class CatalogSourceManager(object): """ Small helper class to keep track of how we deal with catalog sources This keeps track of two dictionaries One of the dictionaries is keyed by catalog name, and contains information about complete catalogs catalog_comp_info_dicts[catalog_name] : `model_component.CatalogInfo` The other dictionary is keyed by [{catalog_name}_{split_ver}][{split_key}] Where: {catalog_name} is something like '3FGL' {split_ver} is somthing like 'v00' and specifes how to divide sources in the catalog {split_key} refers to a specific sub-selection of sources split_comp_info_dicts[splitkey] : `model_component.ModelComponentInfo` """ def __init__(self, **kwargs): """ C'tor Keyword arguments ----------------- basedir : str Top level directory for finding files """ self._name_factory = NameFactory(**kwargs) self._catalog_comp_info_dicts = {} self._split_comp_info_dicts = {} def read_catalog_info_yaml(self, splitkey): """ Read the yaml file for a particular split key """ catalog_info_yaml = self._name_factory.catalog_split_yaml( sourcekey=splitkey, fullpath=True) yaml_dict = yaml.safe_load(open(catalog_info_yaml)) # resolve env vars yaml_dict['catalog_file'] = os.path.expandvars( yaml_dict['catalog_file']) yaml_dict['catalog_extdir'] = os.path.expandvars( yaml_dict['catalog_extdir']) return yaml_dict def build_catalog_info(self, catalog_info): """ Build a CatalogInfo object """ cat = SourceFactory.build_catalog(**catalog_info) catalog_info['catalog'] = cat # catalog_info['catalog_table'] = # Table.read(catalog_info['catalog_file']) catalog_info['catalog_table'] = cat.table catalog_info['roi_model'] =\ SourceFactory.make_fermipy_roi_model_from_catalogs([cat]) catalog_info['srcmdl_name'] =\ self._name_factory.srcmdl_xml(sourcekey=catalog_info['catalog_name']) return CatalogInfo(**catalog_info) def catalogs(self): """ Return the list of full catalogs used """ return sorted(self._catalog_comp_info_dicts.keys()) def splitkeys(self): """ Return the list of catalog split keys used """ return sorted(self._split_comp_info_dicts.keys()) def catalog_comp_info_dict(self, catkey): """ Return the roi_model for an entire catalog """ return self._catalog_comp_info_dicts[catkey] def split_comp_info_dict(self, catalog_name, split_ver): """ Return the information about a particular scheme for how to handle catalog sources """ return self._split_comp_info_dicts["%s_%s" % (catalog_name, split_ver)] def catalog_components(self, catalog_name, split_ver): """ Return the set of merged components for a particular split key """ return sorted( self._split_comp_info_dicts["%s_%s" % (catalog_name, split_ver)].keys()) def split_comp_info(self, catalog_name, split_ver, split_key): """ Return the info for a particular split key """ return self._split_comp_info_dicts["%s_%s" % (catalog_name, split_ver)][split_key] def make_catalog_comp_info(self, full_cat_info, split_key, rule_key, rule_val, sources): """ Make the information about a single merged component Parameters ---------- full_cat_info : `_model_component.CatalogInfo` Information about the full catalog split_key : str Key identifying the version of the spliting used rule_key : str Key identifying the specific rule for this component rule_val : list List of the cuts used to define this component sources : list List of the names of the sources in this component Returns `CompositeSourceInfo` or `CatalogSourcesInfo` """ merge = rule_val.get('merge', True) sourcekey = "%s_%s_%s" % (full_cat_info.catalog_name, split_key, rule_key) srcmdl_name = self._name_factory.srcmdl_xml(sourcekey=sourcekey) srcmdl_name = self._name_factory.fullpath(localpath=srcmdl_name) kwargs = dict(source_name="%s_%s" % (full_cat_info.catalog_name, rule_key), source_ver=split_key, sourcekey=sourcekey, srcmdl_name=srcmdl_name, source_names=sources, catalog_info=full_cat_info, roi_model=SourceFactory.copy_selected_sources( full_cat_info.roi_model, sources)) if merge: return CompositeSourceInfo(**kwargs) return CatalogSourcesInfo(**kwargs) def make_catalog_comp_info_dict(self, catalog_sources): """ Make the information about the catalog components Parameters ---------- catalog_sources : dict Dictionary with catalog source defintions Returns ------- catalog_ret_dict : dict Dictionary mapping catalog_name to `model_component.CatalogInfo` split_ret_dict : dict Dictionary mapping sourcekey to `model_component.ModelComponentInfo` """ catalog_ret_dict = {} split_ret_dict = {} for key, value in catalog_sources.items(): if value is None: continue if value['model_type'] != 'catalog': continue versions = value['versions'] for version in versions: ver_key = "%s_%s" % (key, version) source_dict = self.read_catalog_info_yaml(ver_key) try: full_cat_info = catalog_ret_dict[key] except KeyError: full_cat_info = self.build_catalog_info(source_dict) catalog_ret_dict[key] = full_cat_info try: all_sources = [ x.strip() for x in full_cat_info. catalog_table['Source_Name'].astype(str).tolist() ] except KeyError: print(full_cat_info.catalog_table.colnames) used_sources = [] rules_dict = source_dict['rules_dict'] if rules_dict is None: rules_dict = {} split_dict = {} for rule_key, rule_val in rules_dict.items(): # full_key =\ # self._name_factory.merged_sourcekey(catalog=ver_key, # rulekey=rule_key) sources = select_sources(full_cat_info.catalog_table, rule_val['cuts']) used_sources.extend(sources) split_dict[rule_key] = self.make_catalog_comp_info( full_cat_info, version, rule_key, rule_val, sources) # Now deal with the remainder for source in used_sources: try: all_sources.remove(source) except ValueError: continue rule_val = dict(cuts=[], merge=source_dict['remainder'].get( 'merge', False)) split_dict['remain'] = self.make_catalog_comp_info( full_cat_info, version, 'remain', rule_val, all_sources) # Merge in the info for this version of splits split_ret_dict[ver_key] = split_dict self._catalog_comp_info_dicts.update(catalog_ret_dict) self._split_comp_info_dicts.update(split_ret_dict) return (catalog_ret_dict, split_ret_dict)
class ModelManager(object): """ Small helper class to create fitting models and manager XML files for fermipy This class contains a 'library', which is a dictionary of all the source components: specifically it maps: sourcekey : `model_component.ModelComponentInfo` """ def __init__(self, **kwargs): """ Keyword arguments ------------------- basedir : str Top level directory for finding files """ self._name_factory = NameFactory(**kwargs) self._dmm = kwargs.get('DiffuseModelManager', DiffuseModelManager(**kwargs)) self._gmm = kwargs.get('GalpropMapManager', GalpropMapManager(**kwargs)) self._csm = kwargs.get('CatalogSourceManager', CatalogSourceManager(**kwargs)) self._library = {} self._models = {} self._spec_lib = SpectralLibrary({}) def read_model_yaml(self, modelkey): """ Read the yaml file for the diffuse components """ model_yaml = self._name_factory.model_yaml(modelkey=modelkey, fullpath=True) model = yaml.safe_load(open(model_yaml)) return model @property def dmm(self): """ Return the DiffuseModelManager """ return self._dmm @property def gmm(self): """ Return the GalpropMapManager """ return self._gmm @property def csm(self): """ Return the CatalogSourceManager """ return self._csm def make_library(self, diffuse_yaml, catalog_yaml, binning_yaml): """ Build up the library of all the components Parameters ---------- diffuse_yaml : str Name of the yaml file with the library of diffuse component definitions catalog_yaml : str Name of the yaml file width the library of catalog split definitions binning_yaml : str Name of the yaml file with the binning definitions """ ret_dict = {} #catalog_dict = yaml.safe_load(open(catalog_yaml)) components_dict = Component.build_from_yamlfile(binning_yaml) diffuse_ret_dict = make_diffuse_comp_info_dict( GalpropMapManager=self._gmm, DiffuseModelManager=self._dmm, library=diffuse_yaml, components=components_dict) catalog_ret_dict = make_catalog_comp_dict( library=catalog_yaml, CatalogSourceManager=self._csm) ret_dict.update(diffuse_ret_dict['comp_info_dict']) ret_dict.update(catalog_ret_dict['comp_info_dict']) self._library.update(ret_dict) return ret_dict def make_model_info(self, modelkey): """ Build a dictionary with the information for a particular model. Parameters ---------- modelkey : str Key used to identify this particular model Return `ModelInfo` """ model = self.read_model_yaml(modelkey) sources = model['sources'] components = OrderedDict() spec_model_yaml = self._name_factory.fullpath( localpath=model['spectral_models']) self._spec_lib.update(yaml.safe_load(open(spec_model_yaml))) for source, source_info in sources.items(): model_type = source_info.get('model_type', None) par_overrides = source_info.get('par_overides', None) version = source_info['version'] spec_type = source_info['SpectrumType'] edisp_disable = source_info.get('edisp_disable', False) sourcekey = "%s_%s" % (source, version) if model_type == 'galprop_rings': comp_info_dict = self.gmm.diffuse_comp_info_dicts(version) def_spec_type = spec_type['default'] for comp_key, comp_info in comp_info_dict.items(): model_comp = ModelComponent(info=comp_info, spectrum=\ self._spec_lib[spec_type.get(comp_key, def_spec_type)], par_overrides=par_overrides, edisp_disable=edisp_disable) components[comp_key] = model_comp elif model_type == 'Catalog': comp_info_dict = self.csm.split_comp_info_dict(source, version) def_spec_type = spec_type['default'] for comp_key, comp_info in comp_info_dict.items(): model_comp = ModelComponent(info=comp_info, spectrum=\ self._spec_lib[spec_type.get(comp_key, def_spec_type)], par_overrides=par_overrides, edisp_disable=edisp_disable) components[comp_key] = model_comp else: comp_info = self.dmm.diffuse_comp_info(sourcekey) model_comp = ModelComponent(info=comp_info, spectrum=self._spec_lib[spec_type], par_overrides=par_overrides, edisp_disable=edisp_disable) components[sourcekey] = model_comp ret_val = ModelInfo(model_name=modelkey, model_components=components) self._models[modelkey] = ret_val return ret_val def make_srcmap_manifest(self, modelkey, components, data): """Build a yaml file that specfies how to make the srcmap files for a particular model Parameters ---------- modelkey : str Key used to identify this particular model components : list The binning components used in this analysis data : str Path to file containing dataset definition """ try: model_info = self._models[modelkey] except KeyError: model_info = self.make_model_info(modelkey) self._name_factory.update_base_dict(data) outfile = os.path.join('analysis', 'model_%s' % modelkey, 'srcmap_manifest_%s.yaml' % modelkey) manifest = model_info.make_srcmap_manifest(components, self._name_factory) outdir = os.path.dirname(outfile) try: os.makedirs(outdir) except OSError: pass utils.write_yaml(manifest, outfile) def make_fermipy_config_yaml(self, modelkey, components, data, **kwargs): """Build a fermipy top-level yaml configuration file Parameters ---------- modelkey : str Key used to identify this particular model components : list The binning components used in this analysis data : str Path to file containing dataset definition """ model_dir = os.path.join('analysis', 'model_%s' % modelkey) hpx_order = kwargs.get('hpx_order', 9) self._name_factory.update_base_dict(data) try: model_info = self._models[modelkey] except KeyError: model_info = self.make_model_info(modelkey) model_rois = model_info.make_model_rois(components, self._name_factory) #source_names = model_info.component_names master_xml_mdl = os.path.basename( self._name_factory.master_srcmdl_xml(modelkey=modelkey)) master_data = dict(scfile=self._name_factory.ft2file(fullpath=True), cacheft1=False) master_binning = dict(projtype='HPX', roiwidth=360., binsperdec=4, hpx_ordering_scheme="RING", hpx_order=hpx_order, hpx_ebin=True) # master_fileio = dict(outdir=model_dir, # logfile=os.path.join(model_dir, 'fermipy.log')) master_fileio = dict(logfile='fermipy.log') master_gtlike = dict(irfs=self._name_factory.irfs(**kwargs), edisp_disable=model_info.edisp_disable_list(), use_external_srcmap=True) master_selection = dict(glat=0., glon=0., radius=180.) master_model = dict(catalogs=[master_xml_mdl]) master_plotting = dict(label_ts_threshold=1e9) master = dict(data=master_data, binning=master_binning, fileio=master_fileio, selection=master_selection, gtlike=master_gtlike, model=master_model, plotting=master_plotting, components=[]) fermipy_dict = master #comp_rois = {} for comp in components: zcut = "zmax%i" % comp.zmax compkey = "%s_%s" % (zcut, comp.make_key('{ebin_name}_{evtype_name}')) comp_roi = model_rois[compkey] name_keys = dict(zcut=zcut, modelkey=modelkey, component=compkey, mktime='none', coordsys=comp.coordsys, fullpath=True) comp_data = dict(ltcube=self._name_factory.ltcube(**name_keys)) comp_selection = dict(logemin=comp.log_emin, logemax=comp.log_emax, zmax=comp.zmax, evtype=comp.evtype) comp_binning = dict(enumbins=comp.enumbins, hpx_order=min(comp.hpx_order, hpx_order), coordsys=comp.coordsys) comp_gtlike = dict( srcmap=self._name_factory.merged_srcmaps(**name_keys), bexpmap=self._name_factory.bexpcube(**name_keys), use_external_srcmap=True) #comp_roi_source_info = {} diffuse_srcs = [] for src in comp_roi.diffuse_sources: if isinstance(src, MapCubeSource): diffuse_srcs.append(dict(name=src.name, file=src.mapcube)) elif isinstance(src, IsoSource): diffuse_srcs.append( dict(name=src.name, file=src.fileFunction)) else: pass comp_model = dict(diffuse=diffuse_srcs) sub_dict = dict(data=comp_data, binning=comp_binning, selection=comp_selection, gtlike=comp_gtlike, model=comp_model) fermipy_dict['components'].append(sub_dict) # Add placeholder diffuse sources fermipy_dict['model']['diffuse'] = diffuse_srcs outfile = os.path.join(model_dir, 'config.yaml') print("Writing fermipy config file %s" % outfile) utils.write_yaml(fermipy_dict, outfile) return fermipy_dict @staticmethod def get_sub_comp_info(source_info, comp): """Build and return information about a sub-component for a particular selection """ sub_comps = source_info.get('components', None) if sub_comps is None: return source_info.copy() moving = source_info.get('moving', False) selection_dependent = source_info.get('selection_dependent', False) if selection_dependent: key = comp.make_key('{ebin_name}_{evtype_name}') elif moving: key = "zmax%i" % comp.zmax ret_dict = source_info.copy() ret_dict.update(sub_comps[key]) return ret_dict
class DiffuseModelManager(object): """ Small helper class to keep track of diffuse component templates This keeps track of the 'diffuse component infomation' dictionary This keyed by: key = {source_name}_{source_ver} Where: {source_name} is something like 'loopI' {source_ver} is somthinng like v00 The dictioary is diffuse_comp_info_dict[key] - > `model_component.ModelComponentInfo` Note that some components ( those that represent moving sources or are selection depedent ) will have a sub-dictionary of diffuse_comp_info_dict object for each sub-component The compoents are defined in a file called config/diffuse_components.yaml """ def __init__(self, **kwargs): """ C'tor Keyword arguments ----------------- name_policy : str Name of yaml file contain file naming policy definitions basedir : str Top level directory for finding files """ self._name_factory = NameFactory(basedir=kwargs.get('basedir')) self._diffuse_comp_info_dict = {} @staticmethod def read_diffuse_component_yaml(yamlfile): """ Read the yaml file for the diffuse components """ diffuse_components = yaml.safe_load(open(yamlfile)) return diffuse_components def sourcekeys(self): """Return the list of source keys""" return sorted(self._diffuse_comp_info_dict.keys()) def diffuse_comp_info(self, sourcekey): """Return the Component info associated to a particular key """ return self._diffuse_comp_info_dict[sourcekey] def make_template_name(self, model_type, sourcekey): """ Make the name of a template file for particular component Parameters ---------- model_type : str Type of model to use for this component sourcekey : str Key to identify this component Returns filename or None if component does not require a template file """ format_dict = self.__dict__.copy() format_dict['sourcekey'] = sourcekey if model_type == 'IsoSource': return self._name_factory.spectral_template(**format_dict) elif model_type == 'MapCubeSource': return self._name_factory.diffuse_template(**format_dict) else: raise ValueError("Unexpected model_type %s" % model_type) def make_xml_name(self, sourcekey): """ Make the name of an xml file for a model definition of a single component Parameters ---------- sourcekey : str Key to identify this component """ format_dict = self.__dict__.copy() format_dict['sourcekey'] = sourcekey return self._name_factory.srcmdl_xml(**format_dict) def make_diffuse_comp_info(self, source_name, source_ver, diffuse_dict, components=None, comp_key=None): """ Make a dictionary mapping the merged component names to list of template files Parameters ---------- source_name : str Name of the source source_ver : str Key identifying the version of the source diffuse_dict : dict Information about this component comp_key : str Used when we need to keep track of sub-components, i.e., for moving and selection dependent sources. Returns `model_component.ModelComponentInfo` or `model_component.IsoComponentInfo` """ model_type = diffuse_dict['model_type'] sourcekey = '%s_%s' % (source_name, source_ver) if comp_key is None: template_name = self.make_template_name(model_type, sourcekey) srcmdl_name = self.make_xml_name(sourcekey) else: template_name = self.make_template_name( model_type, "%s_%s" % (sourcekey, comp_key)) srcmdl_name = self.make_xml_name("%s_%s" % (sourcekey, comp_key)) template_name = self._name_factory.fullpath(localpath=template_name) srcmdl_name = self._name_factory.fullpath(localpath=srcmdl_name) kwargs = dict(source_name=source_name, source_ver=source_ver, model_type=model_type, srcmdl_name=srcmdl_name, components=components, comp_key=comp_key) kwargs.update(diffuse_dict) if model_type == 'IsoSource': kwargs['Spectral_Filename'] = template_name return IsoComponentInfo(**kwargs) elif model_type == 'MapCubeSource': kwargs['Spatial_Filename'] = template_name return MapCubeComponentInfo(**kwargs) else: raise ValueError("Unexpected model type %s" % model_type) def make_diffuse_comp_info_dict(self, diffuse_sources, components): """ Make a dictionary maping from diffuse component to information about that component Parameters ---------- diffuse_sources : dict Dictionary with diffuse source defintions components : dict Dictionary with event selection defintions, needed for selection depenedent diffuse components Returns ------- ret_dict : dict Dictionary mapping sourcekey to `model_component.ModelComponentInfo` """ ret_dict = {} for key, value in diffuse_sources.items(): model_type = value.get('model_type', 'mapcube') if model_type == 'galprop_rings': continue selection_dependent = value.get('selection_dependent', False) moving = value.get('moving', False) versions = value.get('versions', []) for version in versions: #sourcekey = self._name_factory.sourcekey(source_name=key, # source_ver=version) comp_dict = None if selection_dependent: # For selection dependent diffuse sources we need to split # by binning component comp_dict = {} for comp in components: comp_key = comp.make_key('{ebin_name}_{evtype_name}') comp_dict[comp_key] = self.make_diffuse_comp_info( key, version, value, None, comp_key) elif moving: # For moving diffuse sources we need to split by zmax cut comp_dict = {} zmax_dict = {} for comp in components: zmax_dict[int(comp.zmax)] = True zmax_list = sorted(zmax_dict.keys()) for zmax in zmax_list: comp_key = "zmax%i" % (zmax) comp_dict[comp_key] = self.make_diffuse_comp_info( key, version, value, None, comp_key) comp_info = self.make_diffuse_comp_info( key, version, value, comp_dict) ret_dict[comp_info.sourcekey] = comp_info self._diffuse_comp_info_dict.update(ret_dict) return ret_dict
class CatalogSourceManager(object): """ Small helper class to keep track of how we deal with catalog sources This keeps track of two dictionaries One of the dictionaries is keyed by catalog name, and contains information about complete catalogs catalog_comp_info_dicts[catalog_name] : `model_component.CatalogInfo` The other dictionary is keyed by [{catalog_name}_{split_ver}][{split_key}] Where: {catalog_name} is something like '3FGL' {split_ver} is somthing like 'v00' and specifes how to divide sources in the catalog {split_key} refers to a specific sub-selection of sources split_comp_info_dicts[splitkey] : `model_component.ModelComponentInfo` """ def __init__(self, **kwargs): """ C'tor Keyword arguments ----------------- basedir : str Top level directory for finding files """ self._name_factory = NameFactory(**kwargs) self._catalog_comp_info_dicts = {} self._split_comp_info_dicts = {} def read_catalog_info_yaml(self, splitkey): """ Read the yaml file for a particular split key """ catalog_info_yaml = self._name_factory.catalog_split_yaml(sourcekey=splitkey, fullpath=True) yaml_dict = yaml.safe_load(open(catalog_info_yaml)) # resolve env vars yaml_dict['catalog_file'] = os.path.expandvars(yaml_dict['catalog_file']) yaml_dict['catalog_extdir'] = os.path.expandvars(yaml_dict['catalog_extdir']) return yaml_dict def build_catalog_info(self, catalog_info): """ Build a CatalogInfo object """ cat = SourceFactory.build_catalog(**catalog_info) catalog_info['catalog'] = cat #catalog_info['catalog_table'] = # Table.read(catalog_info['catalog_file']) catalog_info['catalog_table'] = cat.table catalog_info['roi_model'] =\ SourceFactory.make_fermipy_roi_model_from_catalogs([cat]) catalog_info['srcmdl_name'] =\ self._name_factory.srcmdl_xml(sourcekey=catalog_info['catalog_name']) return CatalogInfo(**catalog_info) def catalogs(self): """ Return the list of full catalogs used """ return sorted(self._catalog_comp_info_dicts.keys()) def splitkeys(self): """ Return the list of catalog split keys used """ return sorted(self._split_comp_info_dicts.keys()) def catalog_comp_info_dict(self, catkey): """ Return the roi_model for an entire catalog """ return self._catalog_comp_info_dicts[catkey] def split_comp_info_dict(self, catalog_name, split_ver): """ Return the information about a particular scheme for how to handle catalog sources """ return self._split_comp_info_dicts["%s_%s" % (catalog_name, split_ver)] def catalog_components(self, catalog_name, split_ver): """ Return the set of merged components for a particular split key """ return sorted(self._split_comp_info_dicts["%s_%s" % (catalog_name, split_ver)].keys()) def split_comp_info(self, catalog_name, split_ver, split_key): """ Return the info for a particular split key """ return self._split_comp_info_dicts["%s_%s" % (catalog_name, split_ver)][split_key] def make_catalog_comp_info(self, full_cat_info, split_key, rule_key, rule_val, sources): """ Make the information about a single merged component Parameters ---------- full_cat_info : `_model_component.CatalogInfo` Information about the full catalog split_key : str Key identifying the version of the spliting used rule_key : str Key identifying the specific rule for this component rule_val : list List of the cuts used to define this component sources : list List of the names of the sources in this component Returns `CompositeSourceInfo` or `CatalogSourcesInfo` """ merge = rule_val.get('merge', True) sourcekey = "%s_%s_%s" % ( full_cat_info.catalog_name, split_key, rule_key) srcmdl_name = self._name_factory.srcmdl_xml(sourcekey=sourcekey) srcmdl_name = self._name_factory.fullpath(localpath=srcmdl_name) kwargs = dict(source_name="%s_%s" % (full_cat_info.catalog_name, rule_key), source_ver=split_key, sourcekey=sourcekey, srcmdl_name=srcmdl_name, source_names=sources, catalog_info=full_cat_info, roi_model=\ SourceFactory.copy_selected_sources(full_cat_info.roi_model, sources)) if merge: return CompositeSourceInfo(**kwargs) else: return CatalogSourcesInfo(**kwargs) def make_catalog_comp_info_dict(self, catalog_sources): """ Make the information about the catalog components Parameters ---------- catalog_sources : dict Dictionary with catalog source defintions Returns ------- catalog_ret_dict : dict Dictionary mapping catalog_name to `model_component.CatalogInfo` split_ret_dict : dict Dictionary mapping sourcekey to `model_component.ModelComponentInfo` """ catalog_ret_dict = {} split_ret_dict = {} for key, value in catalog_sources.items(): #model_type = value['model_type'] versions = value['versions'] for version in versions: ver_key = "%s_%s" % (key, version) source_dict = self.read_catalog_info_yaml(ver_key) try: full_cat_info = catalog_ret_dict[key] except KeyError: full_cat_info = self.build_catalog_info(source_dict) catalog_ret_dict[key] = full_cat_info try: all_sources = [x.strip() for x in full_cat_info.catalog_table[ 'Source_Name'].tolist()] except KeyError: print (full_cat_info.catalog_table.colnames) used_sources = [] rules_dict = source_dict['rules_dict'] split_dict = {} for rule_key, rule_val in rules_dict.items(): #full_key =\ # self._name_factory.merged_sourcekey(catalog=ver_key, # rulekey=rule_key) sources = select_sources( full_cat_info.catalog_table, rule_val['cuts']) used_sources.extend(sources) split_dict[rule_key] = self.make_catalog_comp_info( full_cat_info, version, rule_key, rule_val, sources) # Now deal with the remainder for source in used_sources: try: all_sources.remove(source) except ValueError: continue rule_val = dict(cuts=[], merge=source_dict['remainder'].get('merge', False)) split_dict['remain'] = self.make_catalog_comp_info( full_cat_info, version, 'remain', rule_val, all_sources) # Merge in the info for this version of splits split_ret_dict[ver_key] = split_dict self._catalog_comp_info_dicts.update(catalog_ret_dict) self._split_comp_info_dicts.update(split_ret_dict) return (catalog_ret_dict, split_ret_dict)
class ModelManager(object): """ Small helper class to create fitting models and manager XML files for fermipy This class contains a 'library', which is a dictionary of all the source components: specifically it maps: sourcekey : `model_component.ModelComponentInfo` """ def __init__(self, **kwargs): """ Keyword arguments ------------------- basedir : str Top level directory for finding files """ self._name_factory = NameFactory(**kwargs) self._dmm = kwargs.get('DiffuseModelManager', DiffuseModelManager(**kwargs)) self._gmm = kwargs.get('GalpropMapManager', GalpropMapManager(**kwargs)) self._csm = kwargs.get('CatalogSourceManager', CatalogSourceManager(**kwargs)) self._library = {} self._models = {} self._spec_lib = SpectralLibrary({}) def read_model_yaml(self, modelkey): """ Read the yaml file for the diffuse components """ model_yaml = self._name_factory.model_yaml(modelkey=modelkey, fullpath=True) model = yaml.safe_load(open(model_yaml)) return model @property def dmm(self): """ Return the DiffuseModelManager """ return self._dmm @property def gmm(self): """ Return the GalpropMapManager """ return self._gmm @property def csm(self): """ Return the CatalogSourceManager """ return self._csm def make_library(self, diffuse_yaml, catalog_yaml, binning_yaml): """ Build up the library of all the components Parameters ---------- diffuse_yaml : str Name of the yaml file with the library of diffuse component definitions catalog_yaml : str Name of the yaml file with the library of catalog split definitions binning_yaml : str Name of the yaml file with the binning definitions """ ret_dict = {} #catalog_dict = yaml.safe_load(open(catalog_yaml)) components_dict = Component.build_from_yamlfile(binning_yaml) diffuse_ret_dict = make_diffuse_comp_info_dict(GalpropMapManager=self._gmm, DiffuseModelManager=self._dmm, diffuse=diffuse_yaml, components=components_dict) catalog_ret_dict = make_catalog_comp_dict(sources=catalog_yaml, CatalogSourceManager=self._csm) ret_dict.update(diffuse_ret_dict['comp_info_dict']) ret_dict.update(catalog_ret_dict['comp_info_dict']) self._library.update(ret_dict) return ret_dict def make_model_info(self, modelkey): """ Build a dictionary with the information for a particular model. Parameters ---------- modelkey : str Key used to identify this particular model Return `ModelInfo` """ model = self.read_model_yaml(modelkey) sources = model['sources'] components = OrderedDict() spec_model_yaml = self._name_factory.fullpath(localpath=model['spectral_models']) self._spec_lib.update(yaml.safe_load(open(spec_model_yaml))) for source, source_info in sources.items(): model_type = source_info.get('model_type', None) version = source_info['version'] spec_type = source_info['SpectrumType'] sourcekey = "%s_%s" % (source, version) if model_type == 'galprop_rings': comp_info_dict = self.gmm.diffuse_comp_info_dicts(version) def_spec_type = spec_type['default'] for comp_key, comp_info in comp_info_dict.items(): model_comp = ModelComponent(info=comp_info, spectrum=\ self._spec_lib[spec_type.get(comp_key, def_spec_type)]) components[comp_key] = model_comp elif model_type == 'Catalog': comp_info_dict = self.csm.split_comp_info_dict(source, version) def_spec_type = spec_type['default'] for comp_key, comp_info in comp_info_dict.items(): model_comp = ModelComponent(info=comp_info, spectrum=\ self._spec_lib[spec_type.get(comp_key, def_spec_type)]) components[comp_key] = model_comp else: comp_info = self.dmm.diffuse_comp_info(sourcekey) model_comp = ModelComponent(info=comp_info, spectrum=self._spec_lib[spec_type]) components[sourcekey] = model_comp ret_val = ModelInfo(model_name=modelkey, model_components=components) self._models[modelkey] = ret_val return ret_val def make_srcmap_manifest(self, modelkey, components, data): """Build a yaml file that specfies how to make the srcmap files for a particular model Parameters ---------- modelkey : str Key used to identify this particular model components : list The binning components used in this analysis data : str Path to file containing dataset definition """ try: model_info = self._models[modelkey] except KeyError: model_info = self.make_model_info(modelkey) self._name_factory.update_base_dict(data) outfile = os.path.join('analysis', 'model_%s' % modelkey, 'srcmap_manifest_%s.yaml' % modelkey) manifest = model_info.make_srcmap_manifest( components, self._name_factory) outdir = os.path.dirname(outfile) try: os.makedirs(outdir) except OSError: pass utils.write_yaml(manifest, outfile) def make_fermipy_config_yaml(self, modelkey, components, data, **kwargs): """Build a fermipy top-level yaml configuration file Parameters ---------- modelkey : str Key used to identify this particular model components : list The binning components used in this analysis data : str Path to file containing dataset definition """ model_dir = os.path.join('analysis', 'model_%s' % modelkey) hpx_order = kwargs.get('hpx_order', 9) self._name_factory.update_base_dict(data) try: model_info = self._models[modelkey] except KeyError: model_info = self.make_model_info(modelkey) model_info.make_model_rois(components, self._name_factory) #source_names = model_info.component_names master_xml_mdl = os.path.basename( self._name_factory.master_srcmdl_xml(modelkey=modelkey)) master_data = dict(scfile=self._name_factory.ft2file(fullpath=True), cacheft1=False) master_binning = dict(projtype='HPX', coordsys=kwargs.get('coordsys', 'GAL'), roiwidth=180., binsperdec=8, hpx_ordering_scheme="RING", hpx_order=hpx_order, hpx_ebin=True) master_fileio = dict(outdir=model_dir, logfile=os.path.join(model_dir, 'fermipy.log')) master_gtlike = dict(irfs=self._name_factory.irfs(**kwargs), edisp_disable=['isodiff', 'diffuse', 'limb']) master_selection = dict(glat=0., glon=0., radius=180.) master_model = dict(catalogs=[master_xml_mdl]) master = dict(data=master_data, binning=master_binning, fileio=master_fileio, selection=master_selection, gtlike=master_gtlike, model=master_model, components=[]) fermipy_dict = master #comp_rois = {} for comp in components: zcut = "zmax%i" % comp.zmax compkey = "%s_%s" % (zcut, comp.make_key( '{ebin_name}_{evtype_name}')) name_keys = dict(zcut=zcut, modelkey=modelkey, component=compkey, fullpath=True) comp_data = dict(ltcube=self._name_factory.ltcube(**name_keys)) comp_selection = dict(logemin=comp.log_emin, logemax=comp.log_emax, zmax=comp.zmax, evtype=comp.evtype) comp_binning = dict(enumbins=comp.enumbins, hpx_order=min(comp.hpx_order, hpx_order)) comp_gtlike = dict(srcmap=self._name_factory.merged_srcmaps(**name_keys), bexpmap=self._name_factory.bexpcube(**name_keys)) #comp_roi_source_info = {} comp_xml_mdl = os.path.basename(self._name_factory.comp_srcmdl_xml(modelkey=modelkey, component=compkey)) comp_model = dict(catalogs=[master_xml_mdl, comp_xml_mdl]) sub_dict = dict(data=comp_data, binning=comp_binning, selection=comp_selection, gtlike=comp_gtlike, model=comp_model) fermipy_dict['components'].append(sub_dict) outfile = os.path.join(model_dir, 'config.yaml') print ("Writing fermipy config file %s"%outfile) utils.write_yaml(fermipy_dict, outfile) return fermipy_dict @staticmethod def get_sub_comp_info(source_info, comp): """Build and return information about a sub-component for a particular selection """ sub_comps = source_info.get('components', None) if sub_comps is None: return source_info.copy() moving = source_info.get('moving', False) selection_dependent = source_info.get('selection_dependent', False) if selection_dependent: key = comp.make_key('{ebin_name}_{evtype_name}') elif moving: key = "zmax%i" % comp.zmax ret_dict = source_info.copy() ret_dict.update(sub_comps[key]) return ret_dict