class DiffuseModelManager(object): """ Small helper class to keep track of diffuse component templates This keeps track of the 'diffuse component infomation' dictionary This keyed by: key = {source_name}_{source_ver} Where: {source_name} is something like 'loopI' {source_ver} is somthinng like v00 The dictioary is diffuse_comp_info_dict[key] - > `model_component.ModelComponentInfo` Note that some components ( those that represent moving sources or are selection depedent ) will have a sub-dictionary of diffuse_comp_info_dict object for each sub-component The compoents are defined in a file called config/diffuse_components.yaml """ def __init__(self, **kwargs): """ C'tor Keyword arguments ----------------- name_policy : str Name of yaml file contain file naming policy definitions basedir : str Top level directory for finding files """ self._name_factory = NameFactory(basedir=kwargs.get('basedir')) self._diffuse_comp_info_dict = {} @staticmethod def read_diffuse_component_yaml(yamlfile): """ Read the yaml file for the diffuse components """ diffuse_components = yaml.safe_load(open(yamlfile)) return diffuse_components def sourcekeys(self): """Return the list of source keys""" return sorted(self._diffuse_comp_info_dict.keys()) def diffuse_comp_info(self, sourcekey): """Return the Component info associated to a particular key """ return self._diffuse_comp_info_dict[sourcekey] def make_template_name(self, model_type, sourcekey): """ Make the name of a template file for particular component Parameters ---------- model_type : str Type of model to use for this component sourcekey : str Key to identify this component Returns filename or None if component does not require a template file """ format_dict = self.__dict__.copy() format_dict['sourcekey'] = sourcekey if model_type == 'IsoSource': return self._name_factory.spectral_template(**format_dict) elif model_type == 'MapCubeSource': return self._name_factory.diffuse_template(**format_dict) else: raise ValueError("Unexpected model_type %s" % model_type) def make_xml_name(self, sourcekey): """ Make the name of an xml file for a model definition of a single component Parameters ---------- sourcekey : str Key to identify this component """ format_dict = self.__dict__.copy() format_dict['sourcekey'] = sourcekey return self._name_factory.srcmdl_xml(**format_dict) def make_diffuse_comp_info(self, source_name, source_ver, diffuse_dict, components=None, comp_key=None): """ Make a dictionary mapping the merged component names to list of template files Parameters ---------- source_name : str Name of the source source_ver : str Key identifying the version of the source diffuse_dict : dict Information about this component comp_key : str Used when we need to keep track of sub-components, i.e., for moving and selection dependent sources. Returns `model_component.ModelComponentInfo` or `model_component.IsoComponentInfo` """ model_type = diffuse_dict['model_type'] sourcekey = '%s_%s' % (source_name, source_ver) if comp_key is None: template_name = self.make_template_name(model_type, sourcekey) srcmdl_name = self.make_xml_name(sourcekey) else: template_name = self.make_template_name( model_type, "%s_%s" % (sourcekey, comp_key)) srcmdl_name = self.make_xml_name("%s_%s" % (sourcekey, comp_key)) template_name = self._name_factory.fullpath(localpath=template_name) srcmdl_name = self._name_factory.fullpath(localpath=srcmdl_name) kwargs = dict(source_name=source_name, source_ver=source_ver, model_type=model_type, srcmdl_name=srcmdl_name, components=components, comp_key=comp_key) kwargs.update(diffuse_dict) if model_type == 'IsoSource': kwargs['Spectral_Filename'] = template_name return IsoComponentInfo(**kwargs) elif model_type == 'MapCubeSource': kwargs['Spatial_Filename'] = template_name return MapCubeComponentInfo(**kwargs) else: raise ValueError("Unexpected model type %s" % model_type) def make_diffuse_comp_info_dict(self, diffuse_sources, components): """ Make a dictionary maping from diffuse component to information about that component Parameters ---------- diffuse_sources : dict Dictionary with diffuse source defintions components : dict Dictionary with event selection defintions, needed for selection depenedent diffuse components Returns ------- ret_dict : dict Dictionary mapping sourcekey to `model_component.ModelComponentInfo` """ ret_dict = {} for key, value in diffuse_sources.items(): model_type = value.get('model_type', 'mapcube') if model_type == 'galprop_rings': continue selection_dependent = value.get('selection_dependent', False) moving = value.get('moving', False) versions = value.get('versions', []) for version in versions: #sourcekey = self._name_factory.sourcekey(source_name=key, # source_ver=version) comp_dict = None if selection_dependent: # For selection dependent diffuse sources we need to split # by binning component comp_dict = {} for comp in components: comp_key = comp.make_key('{ebin_name}_{evtype_name}') comp_dict[comp_key] = self.make_diffuse_comp_info( key, version, value, None, comp_key) elif moving: # For moving diffuse sources we need to split by zmax cut comp_dict = {} zmax_dict = {} for comp in components: zmax_dict[int(comp.zmax)] = True zmax_list = sorted(zmax_dict.keys()) for zmax in zmax_list: comp_key = "zmax%i" % (zmax) comp_dict[comp_key] = self.make_diffuse_comp_info( key, version, value, None, comp_key) comp_info = self.make_diffuse_comp_info( key, version, value, comp_dict) ret_dict[comp_info.sourcekey] = comp_info self._diffuse_comp_info_dict.update(ret_dict) return ret_dict
class GalpropMapManager(object): """ Small helper class to keep track of Galprop gasmaps This keeps track of two types of dictionaries. Both are keyed by: key = {source_name}_{ring}_{galkey} Where: {source_name} is something like 'merged_C0' {ring} is the ring index {galkey} is a key specifying which version of galprop rings to use. The two dictionaries are: ring_dict[key] = `model_component.GalpropMergedRingInfo` diffuse_comp_info_dict[key] ] `model_component.ModelComponentInfo` The dictionaries are defined in files called. models/galprop_rings_{galkey}.yaml """ def __init__(self, **kwargs): """ C'tor Keyword arguments ----------------- maptype : str [newGasMaps_ST] Used to define path to gasmap files projtype : str [healpix] Used to define path to gasmap files basedir : str Top level directory for finding files """ self.maptype = kwargs.get('maptype', 'newGasMaps_ST') self.projtype = kwargs.get('projtype', 'healpix') self._name_factory = NameFactory(basedir=kwargs.get('basedir')) self._ring_dicts = {} self._diffuse_comp_info_dicts = {} def read_galprop_rings_yaml(self, galkey): """ Read the yaml file for a partiuclar galprop key """ galprop_rings_yaml = self._name_factory.galprop_rings_yaml( galkey=galkey, fullpath=True) galprop_rings = yaml.safe_load(open(galprop_rings_yaml)) return galprop_rings def galkeys(self): """ Return the list of galprop keys used """ return sorted(self._ring_dicts.keys()) def ring_dict(self, galkey): """ Return the ring dictionary for a particular galprop key """ return self._ring_dicts[galkey] def diffuse_comp_info_dicts(self, galkey): """ Return the components info dictionary for a particular galprop key """ return self._diffuse_comp_info_dicts[galkey] def merged_components(self, galkey): """ Return the set of merged components for a particular galprop key """ return sorted(self._diffuse_comp_info_dicts[galkey].keys()) def make_ring_filename(self, source_name, ring, galprop_run): """ Make the name of a gasmap file for a single ring Parameters ---------- source_name : str The galprop component, used to define path to gasmap files ring : int The ring index galprop_run : str String identifying the galprop parameters """ format_dict = self.__dict__.copy() format_dict['sourcekey'] = self._name_factory.galprop_ringkey( source_name=source_name, ringkey="ring_%i" % ring) format_dict['galprop_run'] = galprop_run return self._name_factory.galprop_gasmap(**format_dict) def make_merged_name(self, source_name, galkey, fullpath): """ Make the name of a gasmap file for a set of merged rings Parameters ---------- source_name : str The galprop component, used to define path to gasmap files galkey : str A short key identifying the galprop parameters fullpath : bool Return the full path name """ format_dict = self.__dict__.copy() format_dict['sourcekey'] = self._name_factory.galprop_sourcekey( source_name=source_name, galpropkey=galkey) format_dict['fullpath'] = fullpath return self._name_factory.merged_gasmap(**format_dict) def make_xml_name(self, source_name, galkey, fullpath): """ Make the name of an xml file for a model definition for a set of merged rings Parameters ---------- source_name : str The galprop component, used to define path to gasmap files galkey : str A short key identifying the galprop parameters fullpath : bool Return the full path name """ format_dict = self.__dict__.copy() format_dict['sourcekey'] = self._name_factory.galprop_sourcekey( source_name=source_name, galpropkey=galkey) format_dict['fullpath'] = fullpath return self._name_factory.srcmdl_xml(**format_dict) def make_ring_filelist(self, sourcekeys, rings, galprop_run): """ Make a list of all the template files for a merged component Parameters ---------- sourcekeys : list-like of str The names of the componenents to merge rings : list-like of int The indices of the rings to merge galprop_run : str String identifying the galprop parameters """ flist = [] for sourcekey in sourcekeys: for ring in rings: flist += [ self.make_ring_filename(sourcekey, ring, galprop_run) ] return flist def make_ring_dict(self, galkey): """ Make a dictionary mapping the merged component names to list of template files Parameters ---------- galkey : str Unique key for this ring dictionary Returns `model_component.GalpropMergedRingInfo` """ galprop_rings = self.read_galprop_rings_yaml(galkey) galprop_run = galprop_rings['galprop_run'] ring_limits = galprop_rings['ring_limits'] comp_dict = galprop_rings['diffuse_comp_dict'] ring_dict = {} nring = len(ring_limits) - 1 for source_name, source_value in comp_dict.items(): base_dict = dict(source_name=source_name, galkey=galkey, galprop_run=galprop_run) for iring in range(nring): sourcekey = "%s_%i" % (source_name, iring) full_key = "%s_%s" % (sourcekey, galkey) rings = range(ring_limits[iring], ring_limits[iring + 1]) base_dict.update( dict(ring=iring, sourcekey=sourcekey, files=self.make_ring_filelist(source_value, rings, galprop_run), merged_gasmap=self.make_merged_name( sourcekey, galkey, False))) ring_dict[full_key] = GalpropMergedRingInfo(**base_dict) self._ring_dicts[galkey] = ring_dict return ring_dict def make_diffuse_comp_info(self, merged_name, galkey): """ Make the information about a single merged component Parameters ---------- merged_name : str The name of the merged component galkey : str A short key identifying the galprop parameters Returns `odel_component.ModelComponentInfo` """ kwargs = dict(source_name=merged_name, source_ver=galkey, model_type='MapCubeSource', Spatial_Filename=self.make_merged_name(merged_name, galkey, fullpath=True), srcmdl_name=self.make_xml_name(merged_name, galkey, fullpath=True)) return MapCubeComponentInfo(**kwargs) def make_diffuse_comp_info_dict(self, galkey): """ Make a dictionary maping from merged component to information about that component Parameters ---------- galkey : str A short key identifying the galprop parameters """ galprop_rings = self.read_galprop_rings_yaml(galkey) ring_limits = galprop_rings.get('ring_limits') comp_dict = galprop_rings.get('diffuse_comp_dict') diffuse_comp_info_dict = {} nring = len(ring_limits) - 1 for source_key in sorted(comp_dict.keys()): for iring in range(nring): source_name = "%s_%i" % (source_key, iring) full_key = "%s_%s" % (source_name, galkey) diffuse_comp_info_dict[full_key] =\ self.make_diffuse_comp_info(source_name, galkey) self._diffuse_comp_info_dicts[galkey] = diffuse_comp_info_dict return diffuse_comp_info_dict
class CatalogSourceManager(object): """ Small helper class to keep track of how we deal with catalog sources This keeps track of two dictionaries One of the dictionaries is keyed by catalog name, and contains information about complete catalogs catalog_comp_info_dicts[catalog_name] : `model_component.CatalogInfo` The other dictionary is keyed by [{catalog_name}_{split_ver}][{split_key}] Where: {catalog_name} is something like '3FGL' {split_ver} is somthing like 'v00' and specifes how to divide sources in the catalog {split_key} refers to a specific sub-selection of sources split_comp_info_dicts[splitkey] : `model_component.ModelComponentInfo` """ def __init__(self, **kwargs): """ C'tor Keyword arguments ----------------- basedir : str Top level directory for finding files """ self._name_factory = NameFactory(**kwargs) self._catalog_comp_info_dicts = {} self._split_comp_info_dicts = {} def read_catalog_info_yaml(self, splitkey): """ Read the yaml file for a particular split key """ catalog_info_yaml = self._name_factory.catalog_split_yaml( sourcekey=splitkey, fullpath=True) yaml_dict = yaml.safe_load(open(catalog_info_yaml)) # resolve env vars yaml_dict['catalog_file'] = os.path.expandvars( yaml_dict['catalog_file']) yaml_dict['catalog_extdir'] = os.path.expandvars( yaml_dict['catalog_extdir']) return yaml_dict def build_catalog_info(self, catalog_info): """ Build a CatalogInfo object """ cat = SourceFactory.build_catalog(**catalog_info) catalog_info['catalog'] = cat # catalog_info['catalog_table'] = # Table.read(catalog_info['catalog_file']) catalog_info['catalog_table'] = cat.table catalog_info['roi_model'] =\ SourceFactory.make_fermipy_roi_model_from_catalogs([cat]) catalog_info['srcmdl_name'] =\ self._name_factory.srcmdl_xml(sourcekey=catalog_info['catalog_name']) return CatalogInfo(**catalog_info) def catalogs(self): """ Return the list of full catalogs used """ return sorted(self._catalog_comp_info_dicts.keys()) def splitkeys(self): """ Return the list of catalog split keys used """ return sorted(self._split_comp_info_dicts.keys()) def catalog_comp_info_dict(self, catkey): """ Return the roi_model for an entire catalog """ return self._catalog_comp_info_dicts[catkey] def split_comp_info_dict(self, catalog_name, split_ver): """ Return the information about a particular scheme for how to handle catalog sources """ return self._split_comp_info_dicts["%s_%s" % (catalog_name, split_ver)] def catalog_components(self, catalog_name, split_ver): """ Return the set of merged components for a particular split key """ return sorted( self._split_comp_info_dicts["%s_%s" % (catalog_name, split_ver)].keys()) def split_comp_info(self, catalog_name, split_ver, split_key): """ Return the info for a particular split key """ return self._split_comp_info_dicts["%s_%s" % (catalog_name, split_ver)][split_key] def make_catalog_comp_info(self, full_cat_info, split_key, rule_key, rule_val, sources): """ Make the information about a single merged component Parameters ---------- full_cat_info : `_model_component.CatalogInfo` Information about the full catalog split_key : str Key identifying the version of the spliting used rule_key : str Key identifying the specific rule for this component rule_val : list List of the cuts used to define this component sources : list List of the names of the sources in this component Returns `CompositeSourceInfo` or `CatalogSourcesInfo` """ merge = rule_val.get('merge', True) sourcekey = "%s_%s_%s" % (full_cat_info.catalog_name, split_key, rule_key) srcmdl_name = self._name_factory.srcmdl_xml(sourcekey=sourcekey) srcmdl_name = self._name_factory.fullpath(localpath=srcmdl_name) kwargs = dict(source_name="%s_%s" % (full_cat_info.catalog_name, rule_key), source_ver=split_key, sourcekey=sourcekey, srcmdl_name=srcmdl_name, source_names=sources, catalog_info=full_cat_info, roi_model=SourceFactory.copy_selected_sources( full_cat_info.roi_model, sources)) if merge: return CompositeSourceInfo(**kwargs) return CatalogSourcesInfo(**kwargs) def make_catalog_comp_info_dict(self, catalog_sources): """ Make the information about the catalog components Parameters ---------- catalog_sources : dict Dictionary with catalog source defintions Returns ------- catalog_ret_dict : dict Dictionary mapping catalog_name to `model_component.CatalogInfo` split_ret_dict : dict Dictionary mapping sourcekey to `model_component.ModelComponentInfo` """ catalog_ret_dict = {} split_ret_dict = {} for key, value in catalog_sources.items(): if value is None: continue if value['model_type'] != 'catalog': continue versions = value['versions'] for version in versions: ver_key = "%s_%s" % (key, version) source_dict = self.read_catalog_info_yaml(ver_key) try: full_cat_info = catalog_ret_dict[key] except KeyError: full_cat_info = self.build_catalog_info(source_dict) catalog_ret_dict[key] = full_cat_info try: all_sources = [ x.strip() for x in full_cat_info. catalog_table['Source_Name'].astype(str).tolist() ] except KeyError: print(full_cat_info.catalog_table.colnames) used_sources = [] rules_dict = source_dict['rules_dict'] if rules_dict is None: rules_dict = {} split_dict = {} for rule_key, rule_val in rules_dict.items(): # full_key =\ # self._name_factory.merged_sourcekey(catalog=ver_key, # rulekey=rule_key) sources = select_sources(full_cat_info.catalog_table, rule_val['cuts']) used_sources.extend(sources) split_dict[rule_key] = self.make_catalog_comp_info( full_cat_info, version, rule_key, rule_val, sources) # Now deal with the remainder for source in used_sources: try: all_sources.remove(source) except ValueError: continue rule_val = dict(cuts=[], merge=source_dict['remainder'].get( 'merge', False)) split_dict['remain'] = self.make_catalog_comp_info( full_cat_info, version, 'remain', rule_val, all_sources) # Merge in the info for this version of splits split_ret_dict[ver_key] = split_dict self._catalog_comp_info_dicts.update(catalog_ret_dict) self._split_comp_info_dicts.update(split_ret_dict) return (catalog_ret_dict, split_ret_dict)
class GalpropMapManager(object): """ Small helper class to keep track of Galprop gasmaps This keeps track of two types of dictionaries. Both are keyed by: key = {source_name}_{ring}_{galkey} Where: {source_name} is something like 'merged_C0' {ring} is the ring index {galkey} is a key specifying which version of galprop rings to use. The two dictionaries are: ring_dict[key] = `model_component.GalpropMergedRingInfo` diffuse_comp_info_dict[key] ] `model_component.ModelComponentInfo` The dictionaries are defined in files called. models/galprop_rings_{galkey}.yaml """ def __init__(self, **kwargs): """ C'tor Keyword arguments ----------------- maptype : str [newGasMaps_ST] Used to define path to gasmap files projtype : str [healpix] Used to define path to gasmap files basedir : str Top level directory for finding files """ self.maptype = kwargs.get('maptype', 'newGasMaps_ST') self.projtype = kwargs.get('projtype', 'healpix') self._name_factory = NameFactory(basedir=kwargs.get('basedir')) self._ring_dicts = {} self._diffuse_comp_info_dicts = {} def read_galprop_rings_yaml(self, galkey): """ Read the yaml file for a partiuclar galprop key """ galprop_rings_yaml = self._name_factory.galprop_rings_yaml(galkey=galkey, fullpath=True) galprop_rings = yaml.safe_load(open(galprop_rings_yaml)) return galprop_rings def galkeys(self): """ Return the list of galprop keys used """ return sorted(self._ring_dicts.keys()) def ring_dict(self, galkey): """ Return the ring dictionary for a particular galprop key """ return self._ring_dicts[galkey] def diffuse_comp_info_dicts(self, galkey): """ Return the components info dictionary for a particular galprop key """ return self._diffuse_comp_info_dicts[galkey] def merged_components(self, galkey): """ Return the set of merged components for a particular galprop key """ return sorted(self._diffuse_comp_info_dicts[galkey].keys()) def make_ring_filename(self, source_name, ring, galprop_run): """ Make the name of a gasmap file for a single ring Parameters ---------- source_name : str The galprop component, used to define path to gasmap files ring : int The ring index galprop_run : str String identifying the galprop parameters """ format_dict = self.__dict__.copy() format_dict['sourcekey'] = self._name_factory.galprop_ringkey(source_name=source_name, ringkey="ring_%i" % ring) format_dict['galprop_run'] = galprop_run return self._name_factory.galprop_gasmap(**format_dict) def make_merged_name(self, source_name, galkey, fullpath): """ Make the name of a gasmap file for a set of merged rings Parameters ---------- source_name : str The galprop component, used to define path to gasmap files galkey : str A short key identifying the galprop parameters fullpath : bool Return the full path name """ format_dict = self.__dict__.copy() format_dict['sourcekey'] = self._name_factory.galprop_sourcekey(source_name=source_name, galpropkey=galkey) format_dict['fullpath'] = fullpath return self._name_factory.merged_gasmap(**format_dict) def make_xml_name(self, source_name, galkey, fullpath): """ Make the name of an xml file for a model definition for a set of merged rings Parameters ---------- source_name : str The galprop component, used to define path to gasmap files galkey : str A short key identifying the galprop parameters fullpath : bool Return the full path name """ format_dict = self.__dict__.copy() format_dict['sourcekey'] = self._name_factory.galprop_sourcekey(source_name=source_name, galpropkey=galkey) format_dict['fullpath'] = fullpath return self._name_factory.srcmdl_xml(**format_dict) def make_ring_filelist(self, sourcekeys, rings, galprop_run): """ Make a list of all the template files for a merged component Parameters ---------- sourcekeys : list-like of str The names of the componenents to merge rings : list-like of int The indices of the rings to merge galprop_run : str String identifying the galprop parameters """ flist = [] for sourcekey in sourcekeys: for ring in rings: flist += [self.make_ring_filename(sourcekey, ring, galprop_run)] return flist def make_ring_dict(self, galkey): """ Make a dictionary mapping the merged component names to list of template files Parameters ---------- galkey : str Unique key for this ring dictionary Returns `model_component.GalpropMergedRingInfo` """ galprop_rings = self.read_galprop_rings_yaml(galkey) galprop_run = galprop_rings['galprop_run'] ring_limits = galprop_rings['ring_limits'] comp_dict = galprop_rings['diffuse_comp_dict'] ring_dict = {} nring = len(ring_limits) - 1 for source_name, source_value in comp_dict.items(): base_dict = dict(source_name=source_name, galkey=galkey, galprop_run=galprop_run) for iring in range(nring): sourcekey = "%s_%i" % (source_name, iring) full_key = "%s_%s" % (sourcekey, galkey) rings = range(ring_limits[iring], ring_limits[iring + 1]) base_dict.update(dict(ring=iring, sourcekey=sourcekey, files=self.make_ring_filelist(source_value, rings, galprop_run), merged_gasmap=self.make_merged_name(sourcekey, galkey, False))) ring_dict[full_key] = GalpropMergedRingInfo(**base_dict) self._ring_dicts[galkey] = ring_dict return ring_dict def make_diffuse_comp_info(self, merged_name, galkey): """ Make the information about a single merged component Parameters ---------- merged_name : str The name of the merged component galkey : str A short key identifying the galprop parameters Returns `odel_component.ModelComponentInfo` """ kwargs = dict(source_name=merged_name, source_ver=galkey, model_type='MapCubeSource', Spatial_Filename=self.make_merged_name( merged_name, galkey, fullpath=True), srcmdl_name=self.make_xml_name(merged_name, galkey, fullpath=True)) return MapCubeComponentInfo(**kwargs) def make_diffuse_comp_info_dict(self, galkey): """ Make a dictionary maping from merged component to information about that component Parameters ---------- galkey : str A short key identifying the galprop parameters """ galprop_rings = self.read_galprop_rings_yaml(galkey) ring_limits = galprop_rings.get('ring_limits') comp_dict = galprop_rings.get('diffuse_comp_dict') diffuse_comp_info_dict = {} nring = len(ring_limits) - 1 for source_key in sorted(comp_dict.keys()): for iring in range(nring): source_name = "%s_%i" % (source_key, iring) full_key = "%s_%s" % (source_name, galkey) diffuse_comp_info_dict[full_key] =\ self.make_diffuse_comp_info(source_name, galkey) self._diffuse_comp_info_dicts[galkey] = diffuse_comp_info_dict return diffuse_comp_info_dict
class CatalogSourceManager(object): """ Small helper class to keep track of how we deal with catalog sources This keeps track of two dictionaries One of the dictionaries is keyed by catalog name, and contains information about complete catalogs catalog_comp_info_dicts[catalog_name] : `model_component.CatalogInfo` The other dictionary is keyed by [{catalog_name}_{split_ver}][{split_key}] Where: {catalog_name} is something like '3FGL' {split_ver} is somthing like 'v00' and specifes how to divide sources in the catalog {split_key} refers to a specific sub-selection of sources split_comp_info_dicts[splitkey] : `model_component.ModelComponentInfo` """ def __init__(self, **kwargs): """ C'tor Keyword arguments ----------------- basedir : str Top level directory for finding files """ self._name_factory = NameFactory(**kwargs) self._catalog_comp_info_dicts = {} self._split_comp_info_dicts = {} def read_catalog_info_yaml(self, splitkey): """ Read the yaml file for a particular split key """ catalog_info_yaml = self._name_factory.catalog_split_yaml(sourcekey=splitkey, fullpath=True) yaml_dict = yaml.safe_load(open(catalog_info_yaml)) # resolve env vars yaml_dict['catalog_file'] = os.path.expandvars(yaml_dict['catalog_file']) yaml_dict['catalog_extdir'] = os.path.expandvars(yaml_dict['catalog_extdir']) return yaml_dict def build_catalog_info(self, catalog_info): """ Build a CatalogInfo object """ cat = SourceFactory.build_catalog(**catalog_info) catalog_info['catalog'] = cat #catalog_info['catalog_table'] = # Table.read(catalog_info['catalog_file']) catalog_info['catalog_table'] = cat.table catalog_info['roi_model'] =\ SourceFactory.make_fermipy_roi_model_from_catalogs([cat]) catalog_info['srcmdl_name'] =\ self._name_factory.srcmdl_xml(sourcekey=catalog_info['catalog_name']) return CatalogInfo(**catalog_info) def catalogs(self): """ Return the list of full catalogs used """ return sorted(self._catalog_comp_info_dicts.keys()) def splitkeys(self): """ Return the list of catalog split keys used """ return sorted(self._split_comp_info_dicts.keys()) def catalog_comp_info_dict(self, catkey): """ Return the roi_model for an entire catalog """ return self._catalog_comp_info_dicts[catkey] def split_comp_info_dict(self, catalog_name, split_ver): """ Return the information about a particular scheme for how to handle catalog sources """ return self._split_comp_info_dicts["%s_%s" % (catalog_name, split_ver)] def catalog_components(self, catalog_name, split_ver): """ Return the set of merged components for a particular split key """ return sorted(self._split_comp_info_dicts["%s_%s" % (catalog_name, split_ver)].keys()) def split_comp_info(self, catalog_name, split_ver, split_key): """ Return the info for a particular split key """ return self._split_comp_info_dicts["%s_%s" % (catalog_name, split_ver)][split_key] def make_catalog_comp_info(self, full_cat_info, split_key, rule_key, rule_val, sources): """ Make the information about a single merged component Parameters ---------- full_cat_info : `_model_component.CatalogInfo` Information about the full catalog split_key : str Key identifying the version of the spliting used rule_key : str Key identifying the specific rule for this component rule_val : list List of the cuts used to define this component sources : list List of the names of the sources in this component Returns `CompositeSourceInfo` or `CatalogSourcesInfo` """ merge = rule_val.get('merge', True) sourcekey = "%s_%s_%s" % ( full_cat_info.catalog_name, split_key, rule_key) srcmdl_name = self._name_factory.srcmdl_xml(sourcekey=sourcekey) srcmdl_name = self._name_factory.fullpath(localpath=srcmdl_name) kwargs = dict(source_name="%s_%s" % (full_cat_info.catalog_name, rule_key), source_ver=split_key, sourcekey=sourcekey, srcmdl_name=srcmdl_name, source_names=sources, catalog_info=full_cat_info, roi_model=\ SourceFactory.copy_selected_sources(full_cat_info.roi_model, sources)) if merge: return CompositeSourceInfo(**kwargs) else: return CatalogSourcesInfo(**kwargs) def make_catalog_comp_info_dict(self, catalog_sources): """ Make the information about the catalog components Parameters ---------- catalog_sources : dict Dictionary with catalog source defintions Returns ------- catalog_ret_dict : dict Dictionary mapping catalog_name to `model_component.CatalogInfo` split_ret_dict : dict Dictionary mapping sourcekey to `model_component.ModelComponentInfo` """ catalog_ret_dict = {} split_ret_dict = {} for key, value in catalog_sources.items(): #model_type = value['model_type'] versions = value['versions'] for version in versions: ver_key = "%s_%s" % (key, version) source_dict = self.read_catalog_info_yaml(ver_key) try: full_cat_info = catalog_ret_dict[key] except KeyError: full_cat_info = self.build_catalog_info(source_dict) catalog_ret_dict[key] = full_cat_info try: all_sources = [x.strip() for x in full_cat_info.catalog_table[ 'Source_Name'].tolist()] except KeyError: print (full_cat_info.catalog_table.colnames) used_sources = [] rules_dict = source_dict['rules_dict'] split_dict = {} for rule_key, rule_val in rules_dict.items(): #full_key =\ # self._name_factory.merged_sourcekey(catalog=ver_key, # rulekey=rule_key) sources = select_sources( full_cat_info.catalog_table, rule_val['cuts']) used_sources.extend(sources) split_dict[rule_key] = self.make_catalog_comp_info( full_cat_info, version, rule_key, rule_val, sources) # Now deal with the remainder for source in used_sources: try: all_sources.remove(source) except ValueError: continue rule_val = dict(cuts=[], merge=source_dict['remainder'].get('merge', False)) split_dict['remain'] = self.make_catalog_comp_info( full_cat_info, version, 'remain', rule_val, all_sources) # Merge in the info for this version of splits split_ret_dict[ver_key] = split_dict self._catalog_comp_info_dicts.update(catalog_ret_dict) self._split_comp_info_dicts.update(split_ret_dict) return (catalog_ret_dict, split_ret_dict)