class DiffuseModelManager(object):
""" Small helper class to keep track of diffuse component templates
This keeps track of the 'diffuse component infomation' dictionary
This keyed by: key = {source_name}_{source_ver}
Where:
{source_name} is something like 'loopI'
{source_ver} is somthinng like v00
The dictioary is
diffuse_comp_info_dict[key] - > `model_component.ModelComponentInfo`
Note that some components ( those that represent moving sources or are selection depedent )
will have a sub-dictionary of diffuse_comp_info_dict object for each sub-component
The compoents are defined in a file called
config/diffuse_components.yaml
"""
def __init__(self, **kwargs):
""" C'tor
Keyword arguments
-----------------
name_policy : str
Name of yaml file contain file naming policy definitions
basedir : str
Top level directory for finding files
"""
self._name_factory = NameFactory(basedir=kwargs.get('basedir'))
self._diffuse_comp_info_dict = {}
@staticmethod
def read_diffuse_component_yaml(yamlfile):
""" Read the yaml file for the diffuse components
"""
diffuse_components = yaml.safe_load(open(yamlfile))
return diffuse_components
def sourcekeys(self):
"""Return the list of source keys"""
return sorted(self._diffuse_comp_info_dict.keys())
def diffuse_comp_info(self, sourcekey):
"""Return the Component info associated to a particular key
"""
return self._diffuse_comp_info_dict[sourcekey]
def make_template_name(self, model_type, sourcekey):
""" Make the name of a template file for particular component
Parameters
----------
model_type : str
Type of model to use for this component
sourcekey : str
Key to identify this component
Returns filename or None if component does not require a template file
"""
format_dict = self.__dict__.copy()
format_dict['sourcekey'] = sourcekey
if model_type == 'IsoSource':
return self._name_factory.spectral_template(**format_dict)
elif model_type == 'MapCubeSource':
return self._name_factory.diffuse_template(**format_dict)
else:
raise ValueError("Unexpected model_type %s" % model_type)
def make_xml_name(self, sourcekey):
""" Make the name of an xml file for a model definition of a single component
Parameters
----------
sourcekey : str
Key to identify this component
"""
format_dict = self.__dict__.copy()
format_dict['sourcekey'] = sourcekey
return self._name_factory.srcmdl_xml(**format_dict)
def make_diffuse_comp_info(self,
source_name,
source_ver,
diffuse_dict,
components=None,
comp_key=None):
""" Make a dictionary mapping the merged component names to list of template files
Parameters
----------
source_name : str
Name of the source
source_ver : str
Key identifying the version of the source
diffuse_dict : dict
Information about this component
comp_key : str
Used when we need to keep track of sub-components, i.e.,
for moving and selection dependent sources.
Returns `model_component.ModelComponentInfo` or
`model_component.IsoComponentInfo`
"""
model_type = diffuse_dict['model_type']
sourcekey = '%s_%s' % (source_name, source_ver)
if comp_key is None:
template_name = self.make_template_name(model_type, sourcekey)
srcmdl_name = self.make_xml_name(sourcekey)
else:
template_name = self.make_template_name(
model_type, "%s_%s" % (sourcekey, comp_key))
srcmdl_name = self.make_xml_name("%s_%s" % (sourcekey, comp_key))
template_name = self._name_factory.fullpath(localpath=template_name)
srcmdl_name = self._name_factory.fullpath(localpath=srcmdl_name)
kwargs = dict(source_name=source_name,
source_ver=source_ver,
model_type=model_type,
srcmdl_name=srcmdl_name,
components=components,
comp_key=comp_key)
kwargs.update(diffuse_dict)
if model_type == 'IsoSource':
kwargs['Spectral_Filename'] = template_name
return IsoComponentInfo(**kwargs)
elif model_type == 'MapCubeSource':
kwargs['Spatial_Filename'] = template_name
return MapCubeComponentInfo(**kwargs)
else:
raise ValueError("Unexpected model type %s" % model_type)
def make_diffuse_comp_info_dict(self, diffuse_sources, components):
""" Make a dictionary maping from diffuse component to information about that component
Parameters
----------
diffuse_sources : dict
Dictionary with diffuse source defintions
components : dict
Dictionary with event selection defintions,
needed for selection depenedent diffuse components
Returns
-------
ret_dict : dict
Dictionary mapping sourcekey to `model_component.ModelComponentInfo`
"""
ret_dict = {}
for key, value in diffuse_sources.items():
model_type = value.get('model_type', 'mapcube')
if model_type == 'galprop_rings':
continue
selection_dependent = value.get('selection_dependent', False)
moving = value.get('moving', False)
versions = value.get('versions', [])
for version in versions:
#sourcekey = self._name_factory.sourcekey(source_name=key,
# source_ver=version)
comp_dict = None
if selection_dependent:
# For selection dependent diffuse sources we need to split
# by binning component
comp_dict = {}
for comp in components:
comp_key = comp.make_key('{ebin_name}_{evtype_name}')
comp_dict[comp_key] = self.make_diffuse_comp_info(
key, version, value, None, comp_key)
elif moving:
# For moving diffuse sources we need to split by zmax cut
comp_dict = {}
zmax_dict = {}
for comp in components:
zmax_dict[int(comp.zmax)] = True
zmax_list = sorted(zmax_dict.keys())
for zmax in zmax_list:
comp_key = "zmax%i" % (zmax)
comp_dict[comp_key] = self.make_diffuse_comp_info(
key, version, value, None, comp_key)
comp_info = self.make_diffuse_comp_info(
key, version, value, comp_dict)
ret_dict[comp_info.sourcekey] = comp_info
self._diffuse_comp_info_dict.update(ret_dict)
return ret_dict
class CatalogSourceManager(object):
""" Small helper class to keep track of how we deal with catalog sources
This keeps track of two dictionaries
One of the dictionaries is keyed by catalog name, and contains information
about complete catalogs
catalog_comp_info_dicts[catalog_name] : `model_component.CatalogInfo`
The other dictionary is keyed by [{catalog_name}_{split_ver}][{split_key}]
Where:
{catalog_name} is something like '3FGL'
{split_ver} is somthing like 'v00' and specifes how to divide sources in the catalog
{split_key} refers to a specific sub-selection of sources
split_comp_info_dicts[splitkey] : `model_component.ModelComponentInfo`
"""
def __init__(self, **kwargs):
""" C'tor
Keyword arguments
-----------------
basedir : str
Top level directory for finding files
"""
self._name_factory = NameFactory(**kwargs)
self._catalog_comp_info_dicts = {}
self._split_comp_info_dicts = {}
def read_catalog_info_yaml(self, splitkey):
""" Read the yaml file for a particular split key
"""
catalog_info_yaml = self._name_factory.catalog_split_yaml(
sourcekey=splitkey, fullpath=True)
yaml_dict = yaml.safe_load(open(catalog_info_yaml))
# resolve env vars
yaml_dict['catalog_file'] = os.path.expandvars(
yaml_dict['catalog_file'])
yaml_dict['catalog_extdir'] = os.path.expandvars(
yaml_dict['catalog_extdir'])
return yaml_dict
def build_catalog_info(self, catalog_info):
""" Build a CatalogInfo object """
cat = SourceFactory.build_catalog(**catalog_info)
catalog_info['catalog'] = cat
# catalog_info['catalog_table'] =
# Table.read(catalog_info['catalog_file'])
catalog_info['catalog_table'] = cat.table
catalog_info['roi_model'] =\
SourceFactory.make_fermipy_roi_model_from_catalogs([cat])
catalog_info['srcmdl_name'] =\
self._name_factory.srcmdl_xml(sourcekey=catalog_info['catalog_name'])
return CatalogInfo(**catalog_info)
def catalogs(self):
""" Return the list of full catalogs used """
return sorted(self._catalog_comp_info_dicts.keys())
def splitkeys(self):
""" Return the list of catalog split keys used """
return sorted(self._split_comp_info_dicts.keys())
def catalog_comp_info_dict(self, catkey):
""" Return the roi_model for an entire catalog """
return self._catalog_comp_info_dicts[catkey]
def split_comp_info_dict(self, catalog_name, split_ver):
""" Return the information about a particular scheme for how to handle catalog sources """
return self._split_comp_info_dicts["%s_%s" % (catalog_name, split_ver)]
def catalog_components(self, catalog_name, split_ver):
""" Return the set of merged components for a particular split key """
return sorted(
self._split_comp_info_dicts["%s_%s" %
(catalog_name, split_ver)].keys())
def split_comp_info(self, catalog_name, split_ver, split_key):
""" Return the info for a particular split key """
return self._split_comp_info_dicts["%s_%s" % (catalog_name,
split_ver)][split_key]
def make_catalog_comp_info(self, full_cat_info, split_key, rule_key,
rule_val, sources):
""" Make the information about a single merged component
Parameters
----------
full_cat_info : `_model_component.CatalogInfo`
Information about the full catalog
split_key : str
Key identifying the version of the spliting used
rule_key : str
Key identifying the specific rule for this component
rule_val : list
List of the cuts used to define this component
sources : list
List of the names of the sources in this component
Returns `CompositeSourceInfo` or `CatalogSourcesInfo`
"""
merge = rule_val.get('merge', True)
sourcekey = "%s_%s_%s" % (full_cat_info.catalog_name, split_key,
rule_key)
srcmdl_name = self._name_factory.srcmdl_xml(sourcekey=sourcekey)
srcmdl_name = self._name_factory.fullpath(localpath=srcmdl_name)
kwargs = dict(source_name="%s_%s" %
(full_cat_info.catalog_name, rule_key),
source_ver=split_key,
sourcekey=sourcekey,
srcmdl_name=srcmdl_name,
source_names=sources,
catalog_info=full_cat_info,
roi_model=SourceFactory.copy_selected_sources(
full_cat_info.roi_model, sources))
if merge:
return CompositeSourceInfo(**kwargs)
return CatalogSourcesInfo(**kwargs)
def make_catalog_comp_info_dict(self, catalog_sources):
""" Make the information about the catalog components
Parameters
----------
catalog_sources : dict
Dictionary with catalog source defintions
Returns
-------
catalog_ret_dict : dict
Dictionary mapping catalog_name to `model_component.CatalogInfo`
split_ret_dict : dict
Dictionary mapping sourcekey to `model_component.ModelComponentInfo`
"""
catalog_ret_dict = {}
split_ret_dict = {}
for key, value in catalog_sources.items():
if value is None:
continue
if value['model_type'] != 'catalog':
continue
versions = value['versions']
for version in versions:
ver_key = "%s_%s" % (key, version)
source_dict = self.read_catalog_info_yaml(ver_key)
try:
full_cat_info = catalog_ret_dict[key]
except KeyError:
full_cat_info = self.build_catalog_info(source_dict)
catalog_ret_dict[key] = full_cat_info
try:
all_sources = [
x.strip() for x in full_cat_info.
catalog_table['Source_Name'].astype(str).tolist()
]
except KeyError:
print(full_cat_info.catalog_table.colnames)
used_sources = []
rules_dict = source_dict['rules_dict']
if rules_dict is None:
rules_dict = {}
split_dict = {}
for rule_key, rule_val in rules_dict.items():
# full_key =\
# self._name_factory.merged_sourcekey(catalog=ver_key,
# rulekey=rule_key)
sources = select_sources(full_cat_info.catalog_table,
rule_val['cuts'])
used_sources.extend(sources)
split_dict[rule_key] = self.make_catalog_comp_info(
full_cat_info, version, rule_key, rule_val, sources)
# Now deal with the remainder
for source in used_sources:
try:
all_sources.remove(source)
except ValueError:
continue
rule_val = dict(cuts=[],
merge=source_dict['remainder'].get(
'merge', False))
split_dict['remain'] = self.make_catalog_comp_info(
full_cat_info, version, 'remain', rule_val, all_sources)
# Merge in the info for this version of splits
split_ret_dict[ver_key] = split_dict
self._catalog_comp_info_dicts.update(catalog_ret_dict)
self._split_comp_info_dicts.update(split_ret_dict)
return (catalog_ret_dict, split_ret_dict)
class ModelManager(object):
""" Small helper class to create fitting models and manager XML files for fermipy
This class contains a 'library', which is a dictionary of all the source components:
specifically it maps:
sourcekey : `model_component.ModelComponentInfo`
"""
def __init__(self, **kwargs):
"""
Keyword arguments
-------------------
basedir : str
Top level directory for finding files
"""
self._name_factory = NameFactory(**kwargs)
self._dmm = kwargs.get('DiffuseModelManager',
DiffuseModelManager(**kwargs))
self._gmm = kwargs.get('GalpropMapManager',
GalpropMapManager(**kwargs))
self._csm = kwargs.get('CatalogSourceManager',
CatalogSourceManager(**kwargs))
self._library = {}
self._models = {}
self._spec_lib = SpectralLibrary({})
def read_model_yaml(self, modelkey):
""" Read the yaml file for the diffuse components
"""
model_yaml = self._name_factory.model_yaml(modelkey=modelkey,
fullpath=True)
model = yaml.safe_load(open(model_yaml))
return model
@property
def dmm(self):
""" Return the DiffuseModelManager """
return self._dmm
@property
def gmm(self):
""" Return the GalpropMapManager """
return self._gmm
@property
def csm(self):
""" Return the CatalogSourceManager """
return self._csm
def make_library(self, diffuse_yaml, catalog_yaml, binning_yaml):
""" Build up the library of all the components
Parameters
----------
diffuse_yaml : str
Name of the yaml file with the library of diffuse component definitions
catalog_yaml : str
Name of the yaml file width the library of catalog split definitions
binning_yaml : str
Name of the yaml file with the binning definitions
"""
ret_dict = {}
#catalog_dict = yaml.safe_load(open(catalog_yaml))
components_dict = Component.build_from_yamlfile(binning_yaml)
diffuse_ret_dict = make_diffuse_comp_info_dict(
GalpropMapManager=self._gmm,
DiffuseModelManager=self._dmm,
library=diffuse_yaml,
components=components_dict)
catalog_ret_dict = make_catalog_comp_dict(
library=catalog_yaml, CatalogSourceManager=self._csm)
ret_dict.update(diffuse_ret_dict['comp_info_dict'])
ret_dict.update(catalog_ret_dict['comp_info_dict'])
self._library.update(ret_dict)
return ret_dict
def make_model_info(self, modelkey):
""" Build a dictionary with the information for a particular model.
Parameters
----------
modelkey : str
Key used to identify this particular model
Return `ModelInfo`
"""
model = self.read_model_yaml(modelkey)
sources = model['sources']
components = OrderedDict()
spec_model_yaml = self._name_factory.fullpath(
localpath=model['spectral_models'])
self._spec_lib.update(yaml.safe_load(open(spec_model_yaml)))
for source, source_info in sources.items():
model_type = source_info.get('model_type', None)
par_overrides = source_info.get('par_overides', None)
version = source_info['version']
spec_type = source_info['SpectrumType']
edisp_disable = source_info.get('edisp_disable', False)
sourcekey = "%s_%s" % (source, version)
if model_type == 'galprop_rings':
comp_info_dict = self.gmm.diffuse_comp_info_dicts(version)
def_spec_type = spec_type['default']
for comp_key, comp_info in comp_info_dict.items():
model_comp = ModelComponent(info=comp_info,
spectrum=\
self._spec_lib[spec_type.get(comp_key,
def_spec_type)],
par_overrides=par_overrides,
edisp_disable=edisp_disable)
components[comp_key] = model_comp
elif model_type == 'Catalog':
comp_info_dict = self.csm.split_comp_info_dict(source, version)
def_spec_type = spec_type['default']
for comp_key, comp_info in comp_info_dict.items():
model_comp = ModelComponent(info=comp_info,
spectrum=\
self._spec_lib[spec_type.get(comp_key,
def_spec_type)],
par_overrides=par_overrides,
edisp_disable=edisp_disable)
components[comp_key] = model_comp
else:
comp_info = self.dmm.diffuse_comp_info(sourcekey)
model_comp = ModelComponent(info=comp_info,
spectrum=self._spec_lib[spec_type],
par_overrides=par_overrides,
edisp_disable=edisp_disable)
components[sourcekey] = model_comp
ret_val = ModelInfo(model_name=modelkey, model_components=components)
self._models[modelkey] = ret_val
return ret_val
def make_srcmap_manifest(self, modelkey, components, data):
"""Build a yaml file that specfies how to make the srcmap files for a particular model
Parameters
----------
modelkey : str
Key used to identify this particular model
components : list
The binning components used in this analysis
data : str
Path to file containing dataset definition
"""
try:
model_info = self._models[modelkey]
except KeyError:
model_info = self.make_model_info(modelkey)
self._name_factory.update_base_dict(data)
outfile = os.path.join('analysis', 'model_%s' % modelkey,
'srcmap_manifest_%s.yaml' % modelkey)
manifest = model_info.make_srcmap_manifest(components,
self._name_factory)
outdir = os.path.dirname(outfile)
try:
os.makedirs(outdir)
except OSError:
pass
utils.write_yaml(manifest, outfile)
def make_fermipy_config_yaml(self, modelkey, components, data, **kwargs):
"""Build a fermipy top-level yaml configuration file
Parameters
----------
modelkey : str
Key used to identify this particular model
components : list
The binning components used in this analysis
data : str
Path to file containing dataset definition
"""
model_dir = os.path.join('analysis', 'model_%s' % modelkey)
hpx_order = kwargs.get('hpx_order', 9)
self._name_factory.update_base_dict(data)
try:
model_info = self._models[modelkey]
except KeyError:
model_info = self.make_model_info(modelkey)
model_rois = model_info.make_model_rois(components, self._name_factory)
#source_names = model_info.component_names
master_xml_mdl = os.path.basename(
self._name_factory.master_srcmdl_xml(modelkey=modelkey))
master_data = dict(scfile=self._name_factory.ft2file(fullpath=True),
cacheft1=False)
master_binning = dict(projtype='HPX',
roiwidth=360.,
binsperdec=4,
hpx_ordering_scheme="RING",
hpx_order=hpx_order,
hpx_ebin=True)
# master_fileio = dict(outdir=model_dir,
# logfile=os.path.join(model_dir, 'fermipy.log'))
master_fileio = dict(logfile='fermipy.log')
master_gtlike = dict(irfs=self._name_factory.irfs(**kwargs),
edisp_disable=model_info.edisp_disable_list(),
use_external_srcmap=True)
master_selection = dict(glat=0., glon=0., radius=180.)
master_model = dict(catalogs=[master_xml_mdl])
master_plotting = dict(label_ts_threshold=1e9)
master = dict(data=master_data,
binning=master_binning,
fileio=master_fileio,
selection=master_selection,
gtlike=master_gtlike,
model=master_model,
plotting=master_plotting,
components=[])
fermipy_dict = master
#comp_rois = {}
for comp in components:
zcut = "zmax%i" % comp.zmax
compkey = "%s_%s" % (zcut,
comp.make_key('{ebin_name}_{evtype_name}'))
comp_roi = model_rois[compkey]
name_keys = dict(zcut=zcut,
modelkey=modelkey,
component=compkey,
mktime='none',
coordsys=comp.coordsys,
fullpath=True)
comp_data = dict(ltcube=self._name_factory.ltcube(**name_keys))
comp_selection = dict(logemin=comp.log_emin,
logemax=comp.log_emax,
zmax=comp.zmax,
evtype=comp.evtype)
comp_binning = dict(enumbins=comp.enumbins,
hpx_order=min(comp.hpx_order, hpx_order),
coordsys=comp.coordsys)
comp_gtlike = dict(
srcmap=self._name_factory.merged_srcmaps(**name_keys),
bexpmap=self._name_factory.bexpcube(**name_keys),
use_external_srcmap=True)
#comp_roi_source_info = {}
diffuse_srcs = []
for src in comp_roi.diffuse_sources:
if isinstance(src, MapCubeSource):
diffuse_srcs.append(dict(name=src.name, file=src.mapcube))
elif isinstance(src, IsoSource):
diffuse_srcs.append(
dict(name=src.name, file=src.fileFunction))
else:
pass
comp_model = dict(diffuse=diffuse_srcs)
sub_dict = dict(data=comp_data,
binning=comp_binning,
selection=comp_selection,
gtlike=comp_gtlike,
model=comp_model)
fermipy_dict['components'].append(sub_dict)
# Add placeholder diffuse sources
fermipy_dict['model']['diffuse'] = diffuse_srcs
outfile = os.path.join(model_dir, 'config.yaml')
print("Writing fermipy config file %s" % outfile)
utils.write_yaml(fermipy_dict, outfile)
return fermipy_dict
@staticmethod
def get_sub_comp_info(source_info, comp):
"""Build and return information about a sub-component for a particular selection
"""
sub_comps = source_info.get('components', None)
if sub_comps is None:
return source_info.copy()
moving = source_info.get('moving', False)
selection_dependent = source_info.get('selection_dependent', False)
if selection_dependent:
key = comp.make_key('{ebin_name}_{evtype_name}')
elif moving:
key = "zmax%i" % comp.zmax
ret_dict = source_info.copy()
ret_dict.update(sub_comps[key])
return ret_dict
class DiffuseModelManager(object):
""" Small helper class to keep track of diffuse component templates
This keeps track of the 'diffuse component infomation' dictionary
This keyed by: key = {source_name}_{source_ver}
Where:
{source_name} is something like 'loopI'
{source_ver} is somthinng like v00
The dictioary is
diffuse_comp_info_dict[key] - > `model_component.ModelComponentInfo`
Note that some components ( those that represent moving sources or are selection depedent )
will have a sub-dictionary of diffuse_comp_info_dict object for each sub-component
The compoents are defined in a file called
config/diffuse_components.yaml
"""
def __init__(self, **kwargs):
""" C'tor
Keyword arguments
-----------------
name_policy : str
Name of yaml file contain file naming policy definitions
basedir : str
Top level directory for finding files
"""
self._name_factory = NameFactory(basedir=kwargs.get('basedir'))
self._diffuse_comp_info_dict = {}
@staticmethod
def read_diffuse_component_yaml(yamlfile):
""" Read the yaml file for the diffuse components
"""
diffuse_components = yaml.safe_load(open(yamlfile))
return diffuse_components
def sourcekeys(self):
"""Return the list of source keys"""
return sorted(self._diffuse_comp_info_dict.keys())
def diffuse_comp_info(self, sourcekey):
"""Return the Component info associated to a particular key
"""
return self._diffuse_comp_info_dict[sourcekey]
def make_template_name(self, model_type, sourcekey):
""" Make the name of a template file for particular component
Parameters
----------
model_type : str
Type of model to use for this component
sourcekey : str
Key to identify this component
Returns filename or None if component does not require a template file
"""
format_dict = self.__dict__.copy()
format_dict['sourcekey'] = sourcekey
if model_type == 'IsoSource':
return self._name_factory.spectral_template(**format_dict)
elif model_type == 'MapCubeSource':
return self._name_factory.diffuse_template(**format_dict)
else:
raise ValueError("Unexpected model_type %s" % model_type)
def make_xml_name(self, sourcekey):
""" Make the name of an xml file for a model definition of a single component
Parameters
----------
sourcekey : str
Key to identify this component
"""
format_dict = self.__dict__.copy()
format_dict['sourcekey'] = sourcekey
return self._name_factory.srcmdl_xml(**format_dict)
def make_diffuse_comp_info(self, source_name, source_ver, diffuse_dict,
components=None, comp_key=None):
""" Make a dictionary mapping the merged component names to list of template files
Parameters
----------
source_name : str
Name of the source
source_ver : str
Key identifying the version of the source
diffuse_dict : dict
Information about this component
comp_key : str
Used when we need to keep track of sub-components, i.e.,
for moving and selection dependent sources.
Returns `model_component.ModelComponentInfo` or
`model_component.IsoComponentInfo`
"""
model_type = diffuse_dict['model_type']
sourcekey = '%s_%s' % (source_name, source_ver)
if comp_key is None:
template_name = self.make_template_name(model_type, sourcekey)
srcmdl_name = self.make_xml_name(sourcekey)
else:
template_name = self.make_template_name(
model_type, "%s_%s" % (sourcekey, comp_key))
srcmdl_name = self.make_xml_name("%s_%s" % (sourcekey, comp_key))
template_name = self._name_factory.fullpath(localpath=template_name)
srcmdl_name = self._name_factory.fullpath(localpath=srcmdl_name)
kwargs = dict(source_name=source_name,
source_ver=source_ver,
model_type=model_type,
srcmdl_name=srcmdl_name,
components=components,
comp_key=comp_key)
kwargs.update(diffuse_dict)
if model_type == 'IsoSource':
kwargs['Spectral_Filename'] = template_name
return IsoComponentInfo(**kwargs)
elif model_type == 'MapCubeSource':
kwargs['Spatial_Filename'] = template_name
return MapCubeComponentInfo(**kwargs)
else:
raise ValueError("Unexpected model type %s" % model_type)
def make_diffuse_comp_info_dict(self, diffuse_sources, components):
""" Make a dictionary maping from diffuse component to information about that component
Parameters
----------
diffuse_sources : dict
Dictionary with diffuse source defintions
components : dict
Dictionary with event selection defintions,
needed for selection depenedent diffuse components
Returns
-------
ret_dict : dict
Dictionary mapping sourcekey to `model_component.ModelComponentInfo`
"""
ret_dict = {}
for key, value in diffuse_sources.items():
model_type = value.get('model_type', 'mapcube')
if model_type == 'galprop_rings':
continue
selection_dependent = value.get('selection_dependent', False)
moving = value.get('moving', False)
versions = value.get('versions', [])
for version in versions:
#sourcekey = self._name_factory.sourcekey(source_name=key,
# source_ver=version)
comp_dict = None
if selection_dependent:
# For selection dependent diffuse sources we need to split
# by binning component
comp_dict = {}
for comp in components:
comp_key = comp.make_key('{ebin_name}_{evtype_name}')
comp_dict[comp_key] = self.make_diffuse_comp_info(
key, version, value, None, comp_key)
elif moving:
# For moving diffuse sources we need to split by zmax cut
comp_dict = {}
zmax_dict = {}
for comp in components:
zmax_dict[int(comp.zmax)] = True
zmax_list = sorted(zmax_dict.keys())
for zmax in zmax_list:
comp_key = "zmax%i" % (zmax)
comp_dict[comp_key] = self.make_diffuse_comp_info(
key, version, value, None, comp_key)
comp_info = self.make_diffuse_comp_info(
key, version, value, comp_dict)
ret_dict[comp_info.sourcekey] = comp_info
self._diffuse_comp_info_dict.update(ret_dict)
return ret_dict
class CatalogSourceManager(object):
""" Small helper class to keep track of how we deal with catalog sources
This keeps track of two dictionaries
One of the dictionaries is keyed by catalog name, and contains information
about complete catalogs
catalog_comp_info_dicts[catalog_name] : `model_component.CatalogInfo`
The other dictionary is keyed by [{catalog_name}_{split_ver}][{split_key}]
Where:
{catalog_name} is something like '3FGL'
{split_ver} is somthing like 'v00' and specifes how to divide sources in the catalog
{split_key} refers to a specific sub-selection of sources
split_comp_info_dicts[splitkey] : `model_component.ModelComponentInfo`
"""
def __init__(self, **kwargs):
""" C'tor
Keyword arguments
-----------------
basedir : str
Top level directory for finding files
"""
self._name_factory = NameFactory(**kwargs)
self._catalog_comp_info_dicts = {}
self._split_comp_info_dicts = {}
def read_catalog_info_yaml(self, splitkey):
""" Read the yaml file for a particular split key
"""
catalog_info_yaml = self._name_factory.catalog_split_yaml(sourcekey=splitkey,
fullpath=True)
yaml_dict = yaml.safe_load(open(catalog_info_yaml))
# resolve env vars
yaml_dict['catalog_file'] = os.path.expandvars(yaml_dict['catalog_file'])
yaml_dict['catalog_extdir'] = os.path.expandvars(yaml_dict['catalog_extdir'])
return yaml_dict
def build_catalog_info(self, catalog_info):
""" Build a CatalogInfo object """
cat = SourceFactory.build_catalog(**catalog_info)
catalog_info['catalog'] = cat
#catalog_info['catalog_table'] =
# Table.read(catalog_info['catalog_file'])
catalog_info['catalog_table'] = cat.table
catalog_info['roi_model'] =\
SourceFactory.make_fermipy_roi_model_from_catalogs([cat])
catalog_info['srcmdl_name'] =\
self._name_factory.srcmdl_xml(sourcekey=catalog_info['catalog_name'])
return CatalogInfo(**catalog_info)
def catalogs(self):
""" Return the list of full catalogs used """
return sorted(self._catalog_comp_info_dicts.keys())
def splitkeys(self):
""" Return the list of catalog split keys used """
return sorted(self._split_comp_info_dicts.keys())
def catalog_comp_info_dict(self, catkey):
""" Return the roi_model for an entire catalog """
return self._catalog_comp_info_dicts[catkey]
def split_comp_info_dict(self, catalog_name, split_ver):
""" Return the information about a particular scheme for how to handle catalog sources """
return self._split_comp_info_dicts["%s_%s" % (catalog_name, split_ver)]
def catalog_components(self, catalog_name, split_ver):
""" Return the set of merged components for a particular split key """
return sorted(self._split_comp_info_dicts["%s_%s" % (catalog_name, split_ver)].keys())
def split_comp_info(self, catalog_name, split_ver, split_key):
""" Return the info for a particular split key """
return self._split_comp_info_dicts["%s_%s" % (catalog_name, split_ver)][split_key]
def make_catalog_comp_info(self, full_cat_info, split_key, rule_key, rule_val, sources):
""" Make the information about a single merged component
Parameters
----------
full_cat_info : `_model_component.CatalogInfo`
Information about the full catalog
split_key : str
Key identifying the version of the spliting used
rule_key : str
Key identifying the specific rule for this component
rule_val : list
List of the cuts used to define this component
sources : list
List of the names of the sources in this component
Returns `CompositeSourceInfo` or `CatalogSourcesInfo`
"""
merge = rule_val.get('merge', True)
sourcekey = "%s_%s_%s" % (
full_cat_info.catalog_name, split_key, rule_key)
srcmdl_name = self._name_factory.srcmdl_xml(sourcekey=sourcekey)
srcmdl_name = self._name_factory.fullpath(localpath=srcmdl_name)
kwargs = dict(source_name="%s_%s" % (full_cat_info.catalog_name, rule_key),
source_ver=split_key,
sourcekey=sourcekey,
srcmdl_name=srcmdl_name,
source_names=sources,
catalog_info=full_cat_info,
roi_model=\
SourceFactory.copy_selected_sources(full_cat_info.roi_model, sources))
if merge:
return CompositeSourceInfo(**kwargs)
else:
return CatalogSourcesInfo(**kwargs)
def make_catalog_comp_info_dict(self, catalog_sources):
""" Make the information about the catalog components
Parameters
----------
catalog_sources : dict
Dictionary with catalog source defintions
Returns
-------
catalog_ret_dict : dict
Dictionary mapping catalog_name to `model_component.CatalogInfo`
split_ret_dict : dict
Dictionary mapping sourcekey to `model_component.ModelComponentInfo`
"""
catalog_ret_dict = {}
split_ret_dict = {}
for key, value in catalog_sources.items():
#model_type = value['model_type']
versions = value['versions']
for version in versions:
ver_key = "%s_%s" % (key, version)
source_dict = self.read_catalog_info_yaml(ver_key)
try:
full_cat_info = catalog_ret_dict[key]
except KeyError:
full_cat_info = self.build_catalog_info(source_dict)
catalog_ret_dict[key] = full_cat_info
try:
all_sources = [x.strip() for x in full_cat_info.catalog_table[
'Source_Name'].tolist()]
except KeyError:
print (full_cat_info.catalog_table.colnames)
used_sources = []
rules_dict = source_dict['rules_dict']
split_dict = {}
for rule_key, rule_val in rules_dict.items():
#full_key =\
# self._name_factory.merged_sourcekey(catalog=ver_key,
# rulekey=rule_key)
sources = select_sources(
full_cat_info.catalog_table, rule_val['cuts'])
used_sources.extend(sources)
split_dict[rule_key] = self.make_catalog_comp_info(
full_cat_info, version, rule_key, rule_val, sources)
# Now deal with the remainder
for source in used_sources:
try:
all_sources.remove(source)
except ValueError:
continue
rule_val = dict(cuts=[],
merge=source_dict['remainder'].get('merge', False))
split_dict['remain'] = self.make_catalog_comp_info(
full_cat_info, version, 'remain', rule_val, all_sources)
# Merge in the info for this version of splits
split_ret_dict[ver_key] = split_dict
self._catalog_comp_info_dicts.update(catalog_ret_dict)
self._split_comp_info_dicts.update(split_ret_dict)
return (catalog_ret_dict, split_ret_dict)
class ModelManager(object):
""" Small helper class to create fitting models and manager XML files for fermipy
This class contains a 'library', which is a dictionary of all the source components:
specifically it maps:
sourcekey : `model_component.ModelComponentInfo`
"""
def __init__(self, **kwargs):
"""
Keyword arguments
-------------------
basedir : str
Top level directory for finding files
"""
self._name_factory = NameFactory(**kwargs)
self._dmm = kwargs.get('DiffuseModelManager',
DiffuseModelManager(**kwargs))
self._gmm = kwargs.get('GalpropMapManager',
GalpropMapManager(**kwargs))
self._csm = kwargs.get('CatalogSourceManager',
CatalogSourceManager(**kwargs))
self._library = {}
self._models = {}
self._spec_lib = SpectralLibrary({})
def read_model_yaml(self, modelkey):
""" Read the yaml file for the diffuse components
"""
model_yaml = self._name_factory.model_yaml(modelkey=modelkey,
fullpath=True)
model = yaml.safe_load(open(model_yaml))
return model
@property
def dmm(self):
""" Return the DiffuseModelManager """
return self._dmm
@property
def gmm(self):
""" Return the GalpropMapManager """
return self._gmm
@property
def csm(self):
""" Return the CatalogSourceManager """
return self._csm
def make_library(self, diffuse_yaml, catalog_yaml, binning_yaml):
""" Build up the library of all the components
Parameters
----------
diffuse_yaml : str
Name of the yaml file with the library of diffuse component definitions
catalog_yaml : str
Name of the yaml file with the library of catalog split definitions
binning_yaml : str
Name of the yaml file with the binning definitions
"""
ret_dict = {}
#catalog_dict = yaml.safe_load(open(catalog_yaml))
components_dict = Component.build_from_yamlfile(binning_yaml)
diffuse_ret_dict = make_diffuse_comp_info_dict(GalpropMapManager=self._gmm,
DiffuseModelManager=self._dmm,
diffuse=diffuse_yaml,
components=components_dict)
catalog_ret_dict = make_catalog_comp_dict(sources=catalog_yaml,
CatalogSourceManager=self._csm)
ret_dict.update(diffuse_ret_dict['comp_info_dict'])
ret_dict.update(catalog_ret_dict['comp_info_dict'])
self._library.update(ret_dict)
return ret_dict
def make_model_info(self, modelkey):
""" Build a dictionary with the information for a particular model.
Parameters
----------
modelkey : str
Key used to identify this particular model
Return `ModelInfo`
"""
model = self.read_model_yaml(modelkey)
sources = model['sources']
components = OrderedDict()
spec_model_yaml = self._name_factory.fullpath(localpath=model['spectral_models'])
self._spec_lib.update(yaml.safe_load(open(spec_model_yaml)))
for source, source_info in sources.items():
model_type = source_info.get('model_type', None)
version = source_info['version']
spec_type = source_info['SpectrumType']
sourcekey = "%s_%s" % (source, version)
if model_type == 'galprop_rings':
comp_info_dict = self.gmm.diffuse_comp_info_dicts(version)
def_spec_type = spec_type['default']
for comp_key, comp_info in comp_info_dict.items():
model_comp = ModelComponent(info=comp_info,
spectrum=\
self._spec_lib[spec_type.get(comp_key,
def_spec_type)])
components[comp_key] = model_comp
elif model_type == 'Catalog':
comp_info_dict = self.csm.split_comp_info_dict(source, version)
def_spec_type = spec_type['default']
for comp_key, comp_info in comp_info_dict.items():
model_comp = ModelComponent(info=comp_info,
spectrum=\
self._spec_lib[spec_type.get(comp_key,
def_spec_type)])
components[comp_key] = model_comp
else:
comp_info = self.dmm.diffuse_comp_info(sourcekey)
model_comp = ModelComponent(info=comp_info,
spectrum=self._spec_lib[spec_type])
components[sourcekey] = model_comp
ret_val = ModelInfo(model_name=modelkey,
model_components=components)
self._models[modelkey] = ret_val
return ret_val
def make_srcmap_manifest(self, modelkey, components, data):
"""Build a yaml file that specfies how to make the srcmap files for a particular model
Parameters
----------
modelkey : str
Key used to identify this particular model
components : list
The binning components used in this analysis
data : str
Path to file containing dataset definition
"""
try:
model_info = self._models[modelkey]
except KeyError:
model_info = self.make_model_info(modelkey)
self._name_factory.update_base_dict(data)
outfile = os.path.join('analysis', 'model_%s' %
modelkey, 'srcmap_manifest_%s.yaml' % modelkey)
manifest = model_info.make_srcmap_manifest(
components, self._name_factory)
outdir = os.path.dirname(outfile)
try:
os.makedirs(outdir)
except OSError:
pass
utils.write_yaml(manifest, outfile)
def make_fermipy_config_yaml(self, modelkey, components, data, **kwargs):
"""Build a fermipy top-level yaml configuration file
Parameters
----------
modelkey : str
Key used to identify this particular model
components : list
The binning components used in this analysis
data : str
Path to file containing dataset definition
"""
model_dir = os.path.join('analysis', 'model_%s' % modelkey)
hpx_order = kwargs.get('hpx_order', 9)
self._name_factory.update_base_dict(data)
try:
model_info = self._models[modelkey]
except KeyError:
model_info = self.make_model_info(modelkey)
model_info.make_model_rois(components, self._name_factory)
#source_names = model_info.component_names
master_xml_mdl = os.path.basename(
self._name_factory.master_srcmdl_xml(modelkey=modelkey))
master_data = dict(scfile=self._name_factory.ft2file(fullpath=True),
cacheft1=False)
master_binning = dict(projtype='HPX',
coordsys=kwargs.get('coordsys', 'GAL'),
roiwidth=180.,
binsperdec=8,
hpx_ordering_scheme="RING",
hpx_order=hpx_order,
hpx_ebin=True)
master_fileio = dict(outdir=model_dir,
logfile=os.path.join(model_dir, 'fermipy.log'))
master_gtlike = dict(irfs=self._name_factory.irfs(**kwargs),
edisp_disable=['isodiff', 'diffuse', 'limb'])
master_selection = dict(glat=0., glon=0., radius=180.)
master_model = dict(catalogs=[master_xml_mdl])
master = dict(data=master_data,
binning=master_binning,
fileio=master_fileio,
selection=master_selection,
gtlike=master_gtlike,
model=master_model,
components=[])
fermipy_dict = master
#comp_rois = {}
for comp in components:
zcut = "zmax%i" % comp.zmax
compkey = "%s_%s" % (zcut, comp.make_key(
'{ebin_name}_{evtype_name}'))
name_keys = dict(zcut=zcut,
modelkey=modelkey,
component=compkey,
fullpath=True)
comp_data = dict(ltcube=self._name_factory.ltcube(**name_keys))
comp_selection = dict(logemin=comp.log_emin,
logemax=comp.log_emax,
zmax=comp.zmax,
evtype=comp.evtype)
comp_binning = dict(enumbins=comp.enumbins,
hpx_order=min(comp.hpx_order, hpx_order))
comp_gtlike = dict(srcmap=self._name_factory.merged_srcmaps(**name_keys),
bexpmap=self._name_factory.bexpcube(**name_keys))
#comp_roi_source_info = {}
comp_xml_mdl = os.path.basename(self._name_factory.comp_srcmdl_xml(modelkey=modelkey,
component=compkey))
comp_model = dict(catalogs=[master_xml_mdl, comp_xml_mdl])
sub_dict = dict(data=comp_data,
binning=comp_binning,
selection=comp_selection,
gtlike=comp_gtlike,
model=comp_model)
fermipy_dict['components'].append(sub_dict)
outfile = os.path.join(model_dir, 'config.yaml')
print ("Writing fermipy config file %s"%outfile)
utils.write_yaml(fermipy_dict, outfile)
return fermipy_dict
@staticmethod
def get_sub_comp_info(source_info, comp):
"""Build and return information about a sub-component for a particular selection
"""
sub_comps = source_info.get('components', None)
if sub_comps is None:
return source_info.copy()
moving = source_info.get('moving', False)
selection_dependent = source_info.get('selection_dependent', False)
if selection_dependent:
key = comp.make_key('{ebin_name}_{evtype_name}')
elif moving:
key = "zmax%i" % comp.zmax
ret_dict = source_info.copy()
ret_dict.update(sub_comps[key])
return ret_dict