def __init__(self, **kwargs):
"""
Keyword arguments
-------------------
basedir : str
Top level directory for finding files
"""
self._name_factory = NameFactory(**kwargs)
self._dmm = kwargs.get('DiffuseModelManager',
DiffuseModelManager(**kwargs))
self._gmm = kwargs.get('GalpropMapManager',
GalpropMapManager(**kwargs))
self._csm = kwargs.get('CatalogSourceManager',
CatalogSourceManager(**kwargs))
self._library = {}
self._models = {}
self._spec_lib = SpectralLibrary({})
def __init__(self, **kwargs):
"""
Keyword arguments
-------------------
basedir : str
Top level directory for finding files
"""
self._name_factory = NameFactory(**kwargs)
self._dmm = kwargs.get('DiffuseModelManager',
DiffuseModelManager(**kwargs))
self._gmm = kwargs.get('GalpropMapManager',
GalpropMapManager(**kwargs))
self._csm = kwargs.get('CatalogSourceManager',
CatalogSourceManager(**kwargs))
self._library = {}
self._models = {}
self._spec_lib = SpectralLibrary({})
class ModelManager(object):
""" Small helper class to create fitting models and manager XML files for fermipy
This class contains a 'library', which is a dictionary of all the source components:
specifically it maps:
sourcekey : `model_component.ModelComponentInfo`
"""
def __init__(self, **kwargs):
"""
Keyword arguments
-------------------
basedir : str
Top level directory for finding files
"""
self._name_factory = NameFactory(**kwargs)
self._dmm = kwargs.get('DiffuseModelManager',
DiffuseModelManager(**kwargs))
self._gmm = kwargs.get('GalpropMapManager',
GalpropMapManager(**kwargs))
self._csm = kwargs.get('CatalogSourceManager',
CatalogSourceManager(**kwargs))
self._library = {}
self._models = {}
self._spec_lib = SpectralLibrary({})
def read_model_yaml(self, modelkey):
""" Read the yaml file for the diffuse components
"""
model_yaml = self._name_factory.model_yaml(modelkey=modelkey,
fullpath=True)
model = yaml.safe_load(open(model_yaml))
return model
@property
def dmm(self):
""" Return the DiffuseModelManager """
return self._dmm
@property
def gmm(self):
""" Return the GalpropMapManager """
return self._gmm
@property
def csm(self):
""" Return the CatalogSourceManager """
return self._csm
def make_library(self, diffuse_yaml, catalog_yaml, binning_yaml):
""" Build up the library of all the components
Parameters
----------
diffuse_yaml : str
Name of the yaml file with the library of diffuse component definitions
catalog_yaml : str
Name of the yaml file width the library of catalog split definitions
binning_yaml : str
Name of the yaml file with the binning definitions
"""
ret_dict = {}
#catalog_dict = yaml.safe_load(open(catalog_yaml))
components_dict = Component.build_from_yamlfile(binning_yaml)
diffuse_ret_dict = make_diffuse_comp_info_dict(
GalpropMapManager=self._gmm,
DiffuseModelManager=self._dmm,
library=diffuse_yaml,
components=components_dict)
catalog_ret_dict = make_catalog_comp_dict(
library=catalog_yaml, CatalogSourceManager=self._csm)
ret_dict.update(diffuse_ret_dict['comp_info_dict'])
ret_dict.update(catalog_ret_dict['comp_info_dict'])
self._library.update(ret_dict)
return ret_dict
def make_model_info(self, modelkey):
""" Build a dictionary with the information for a particular model.
Parameters
----------
modelkey : str
Key used to identify this particular model
Return `ModelInfo`
"""
model = self.read_model_yaml(modelkey)
sources = model['sources']
components = OrderedDict()
spec_model_yaml = self._name_factory.fullpath(
localpath=model['spectral_models'])
self._spec_lib.update(yaml.safe_load(open(spec_model_yaml)))
for source, source_info in sources.items():
model_type = source_info.get('model_type', None)
par_overrides = source_info.get('par_overides', None)
version = source_info['version']
spec_type = source_info['SpectrumType']
edisp_disable = source_info.get('edisp_disable', False)
sourcekey = "%s_%s" % (source, version)
if model_type == 'galprop_rings':
comp_info_dict = self.gmm.diffuse_comp_info_dicts(version)
def_spec_type = spec_type['default']
for comp_key, comp_info in comp_info_dict.items():
model_comp = ModelComponent(info=comp_info,
spectrum=\
self._spec_lib[spec_type.get(comp_key,
def_spec_type)],
par_overrides=par_overrides,
edisp_disable=edisp_disable)
components[comp_key] = model_comp
elif model_type == 'Catalog':
comp_info_dict = self.csm.split_comp_info_dict(source, version)
def_spec_type = spec_type['default']
for comp_key, comp_info in comp_info_dict.items():
model_comp = ModelComponent(info=comp_info,
spectrum=\
self._spec_lib[spec_type.get(comp_key,
def_spec_type)],
par_overrides=par_overrides,
edisp_disable=edisp_disable)
components[comp_key] = model_comp
else:
comp_info = self.dmm.diffuse_comp_info(sourcekey)
model_comp = ModelComponent(info=comp_info,
spectrum=self._spec_lib[spec_type],
par_overrides=par_overrides,
edisp_disable=edisp_disable)
components[sourcekey] = model_comp
ret_val = ModelInfo(model_name=modelkey, model_components=components)
self._models[modelkey] = ret_val
return ret_val
def make_srcmap_manifest(self, modelkey, components, data):
"""Build a yaml file that specfies how to make the srcmap files for a particular model
Parameters
----------
modelkey : str
Key used to identify this particular model
components : list
The binning components used in this analysis
data : str
Path to file containing dataset definition
"""
try:
model_info = self._models[modelkey]
except KeyError:
model_info = self.make_model_info(modelkey)
self._name_factory.update_base_dict(data)
outfile = os.path.join('analysis', 'model_%s' % modelkey,
'srcmap_manifest_%s.yaml' % modelkey)
manifest = model_info.make_srcmap_manifest(components,
self._name_factory)
outdir = os.path.dirname(outfile)
try:
os.makedirs(outdir)
except OSError:
pass
utils.write_yaml(manifest, outfile)
def make_fermipy_config_yaml(self, modelkey, components, data, **kwargs):
"""Build a fermipy top-level yaml configuration file
Parameters
----------
modelkey : str
Key used to identify this particular model
components : list
The binning components used in this analysis
data : str
Path to file containing dataset definition
"""
model_dir = os.path.join('analysis', 'model_%s' % modelkey)
hpx_order = kwargs.get('hpx_order', 9)
self._name_factory.update_base_dict(data)
try:
model_info = self._models[modelkey]
except KeyError:
model_info = self.make_model_info(modelkey)
model_rois = model_info.make_model_rois(components, self._name_factory)
#source_names = model_info.component_names
master_xml_mdl = os.path.basename(
self._name_factory.master_srcmdl_xml(modelkey=modelkey))
master_data = dict(scfile=self._name_factory.ft2file(fullpath=True),
cacheft1=False)
master_binning = dict(projtype='HPX',
roiwidth=360.,
binsperdec=4,
hpx_ordering_scheme="RING",
hpx_order=hpx_order,
hpx_ebin=True)
# master_fileio = dict(outdir=model_dir,
# logfile=os.path.join(model_dir, 'fermipy.log'))
master_fileio = dict(logfile='fermipy.log')
master_gtlike = dict(irfs=self._name_factory.irfs(**kwargs),
edisp_disable=model_info.edisp_disable_list(),
use_external_srcmap=True)
master_selection = dict(glat=0., glon=0., radius=180.)
master_model = dict(catalogs=[master_xml_mdl])
master_plotting = dict(label_ts_threshold=1e9)
master = dict(data=master_data,
binning=master_binning,
fileio=master_fileio,
selection=master_selection,
gtlike=master_gtlike,
model=master_model,
plotting=master_plotting,
components=[])
fermipy_dict = master
#comp_rois = {}
for comp in components:
zcut = "zmax%i" % comp.zmax
compkey = "%s_%s" % (zcut,
comp.make_key('{ebin_name}_{evtype_name}'))
comp_roi = model_rois[compkey]
name_keys = dict(zcut=zcut,
modelkey=modelkey,
component=compkey,
mktime='none',
coordsys=comp.coordsys,
fullpath=True)
comp_data = dict(ltcube=self._name_factory.ltcube(**name_keys))
comp_selection = dict(logemin=comp.log_emin,
logemax=comp.log_emax,
zmax=comp.zmax,
evtype=comp.evtype)
comp_binning = dict(enumbins=comp.enumbins,
hpx_order=min(comp.hpx_order, hpx_order),
coordsys=comp.coordsys)
comp_gtlike = dict(
srcmap=self._name_factory.merged_srcmaps(**name_keys),
bexpmap=self._name_factory.bexpcube(**name_keys),
use_external_srcmap=True)
#comp_roi_source_info = {}
diffuse_srcs = []
for src in comp_roi.diffuse_sources:
if isinstance(src, MapCubeSource):
diffuse_srcs.append(dict(name=src.name, file=src.mapcube))
elif isinstance(src, IsoSource):
diffuse_srcs.append(
dict(name=src.name, file=src.fileFunction))
else:
pass
comp_model = dict(diffuse=diffuse_srcs)
sub_dict = dict(data=comp_data,
binning=comp_binning,
selection=comp_selection,
gtlike=comp_gtlike,
model=comp_model)
fermipy_dict['components'].append(sub_dict)
# Add placeholder diffuse sources
fermipy_dict['model']['diffuse'] = diffuse_srcs
outfile = os.path.join(model_dir, 'config.yaml')
print("Writing fermipy config file %s" % outfile)
utils.write_yaml(fermipy_dict, outfile)
return fermipy_dict
@staticmethod
def get_sub_comp_info(source_info, comp):
"""Build and return information about a sub-component for a particular selection
"""
sub_comps = source_info.get('components', None)
if sub_comps is None:
return source_info.copy()
moving = source_info.get('moving', False)
selection_dependent = source_info.get('selection_dependent', False)
if selection_dependent:
key = comp.make_key('{ebin_name}_{evtype_name}')
elif moving:
key = "zmax%i" % comp.zmax
ret_dict = source_info.copy()
ret_dict.update(sub_comps[key])
return ret_dict
def test_spectral():
the_yaml = """
Constant_Correction :
SpectrumType : ConstantValue
spectral_pars :
Value :
name : Value
scale : 1.0
value : 1.0
min : 1e-4
max: 1e4
free : False
Powerlaw_Correction :
SpectrumType : PowerLaw
spectral_pars :
Prefactor :
name : Prefactor
scale : 1.0
value : 1.0
min : 0.1
max: 10.0
free : False
Index :
name: Index
scale : -1.0
value : 0.0
min : -1.0
max : 1.0
free : False
Scale :
name: Scale
scale : 1.0
value : 1000.0
min : 1000.0
max : 1000.0
free : False
LogParabola_Correction :
SpectrumType : LogParabola
spectral_pars :
norm :
name : norm
scale : 1.0
value : 1.0
min : 0.1
max: 10.0
free : False
alpha :
name: alpha
scale : 1.0
value : 0.0
min : -2.0
max : 2.0
free : False
beta :
name: beta
scale : 1.0
value : 0.0
min : -10.0
max : 10.0
free : False
Eb :
name: Eb
scale : 1.0
value : 1000.0
min : 10.0
max : 100000.0
free : False
"""
spectra = SpectralLibrary.create_from_yamlstr(the_yaml)
# spot check some values
assert(spectra['Constant_Correction']['SpectrumType'] == 'ConstantValue')
assert(len(spectra['Constant_Correction']['spectral_pars']) == 1)
assert(spectra['Constant_Correction'][
'spectral_pars']['Value']['value'] == 1.0)
assert(spectra['Constant_Correction'][
'spectral_pars']['Value']['scale'] == 1.0)
#assert(spectra['Constant_Correction']['spectral_pars']['Value']['min'] == 1e-4)
#assert(spectra['Constant_Correction']['spectral_pars']['Value']['max'] == 1e4)
assert(spectra['Constant_Correction'][
'spectral_pars']['Value']['free'] is False)
assert(spectra['LogParabola_Correction']['SpectrumType'] == 'LogParabola')
assert(len(spectra['LogParabola_Correction']['spectral_pars']) == 4)
assert(spectra['LogParabola_Correction'][
'spectral_pars']['norm']['value'] == 1.0)
assert(spectra['LogParabola_Correction'][
'spectral_pars']['norm']['scale'] == 1.0)
assert(spectra['LogParabola_Correction'][
'spectral_pars']['norm']['min'] == 0.1)
assert(spectra['LogParabola_Correction'][
'spectral_pars']['norm']['max'] == 10.0)
assert(spectra['LogParabola_Correction'][
'spectral_pars']['norm']['free'] is False)
class ModelManager(object):
""" Small helper class to create fitting models and manager XML files for fermipy
This class contains a 'library', which is a dictionary of all the source components:
specifically it maps:
sourcekey : `model_component.ModelComponentInfo`
"""
def __init__(self, **kwargs):
"""
Keyword arguments
-------------------
basedir : str
Top level directory for finding files
"""
self._name_factory = NameFactory(**kwargs)
self._dmm = kwargs.get('DiffuseModelManager',
DiffuseModelManager(**kwargs))
self._gmm = kwargs.get('GalpropMapManager',
GalpropMapManager(**kwargs))
self._csm = kwargs.get('CatalogSourceManager',
CatalogSourceManager(**kwargs))
self._library = {}
self._models = {}
self._spec_lib = SpectralLibrary({})
def read_model_yaml(self, modelkey):
""" Read the yaml file for the diffuse components
"""
model_yaml = self._name_factory.model_yaml(modelkey=modelkey,
fullpath=True)
model = yaml.safe_load(open(model_yaml))
return model
@property
def dmm(self):
""" Return the DiffuseModelManager """
return self._dmm
@property
def gmm(self):
""" Return the GalpropMapManager """
return self._gmm
@property
def csm(self):
""" Return the CatalogSourceManager """
return self._csm
def make_library(self, diffuse_yaml, catalog_yaml, binning_yaml):
""" Build up the library of all the components
Parameters
----------
diffuse_yaml : str
Name of the yaml file with the library of diffuse component definitions
catalog_yaml : str
Name of the yaml file with the library of catalog split definitions
binning_yaml : str
Name of the yaml file with the binning definitions
"""
ret_dict = {}
#catalog_dict = yaml.safe_load(open(catalog_yaml))
components_dict = Component.build_from_yamlfile(binning_yaml)
diffuse_ret_dict = make_diffuse_comp_info_dict(GalpropMapManager=self._gmm,
DiffuseModelManager=self._dmm,
diffuse=diffuse_yaml,
components=components_dict)
catalog_ret_dict = make_catalog_comp_dict(sources=catalog_yaml,
CatalogSourceManager=self._csm)
ret_dict.update(diffuse_ret_dict['comp_info_dict'])
ret_dict.update(catalog_ret_dict['comp_info_dict'])
self._library.update(ret_dict)
return ret_dict
def make_model_info(self, modelkey):
""" Build a dictionary with the information for a particular model.
Parameters
----------
modelkey : str
Key used to identify this particular model
Return `ModelInfo`
"""
model = self.read_model_yaml(modelkey)
sources = model['sources']
components = OrderedDict()
spec_model_yaml = self._name_factory.fullpath(localpath=model['spectral_models'])
self._spec_lib.update(yaml.safe_load(open(spec_model_yaml)))
for source, source_info in sources.items():
model_type = source_info.get('model_type', None)
version = source_info['version']
spec_type = source_info['SpectrumType']
sourcekey = "%s_%s" % (source, version)
if model_type == 'galprop_rings':
comp_info_dict = self.gmm.diffuse_comp_info_dicts(version)
def_spec_type = spec_type['default']
for comp_key, comp_info in comp_info_dict.items():
model_comp = ModelComponent(info=comp_info,
spectrum=\
self._spec_lib[spec_type.get(comp_key,
def_spec_type)])
components[comp_key] = model_comp
elif model_type == 'Catalog':
comp_info_dict = self.csm.split_comp_info_dict(source, version)
def_spec_type = spec_type['default']
for comp_key, comp_info in comp_info_dict.items():
model_comp = ModelComponent(info=comp_info,
spectrum=\
self._spec_lib[spec_type.get(comp_key,
def_spec_type)])
components[comp_key] = model_comp
else:
comp_info = self.dmm.diffuse_comp_info(sourcekey)
model_comp = ModelComponent(info=comp_info,
spectrum=self._spec_lib[spec_type])
components[sourcekey] = model_comp
ret_val = ModelInfo(model_name=modelkey,
model_components=components)
self._models[modelkey] = ret_val
return ret_val
def make_srcmap_manifest(self, modelkey, components, data):
"""Build a yaml file that specfies how to make the srcmap files for a particular model
Parameters
----------
modelkey : str
Key used to identify this particular model
components : list
The binning components used in this analysis
data : str
Path to file containing dataset definition
"""
try:
model_info = self._models[modelkey]
except KeyError:
model_info = self.make_model_info(modelkey)
self._name_factory.update_base_dict(data)
outfile = os.path.join('analysis', 'model_%s' %
modelkey, 'srcmap_manifest_%s.yaml' % modelkey)
manifest = model_info.make_srcmap_manifest(
components, self._name_factory)
outdir = os.path.dirname(outfile)
try:
os.makedirs(outdir)
except OSError:
pass
utils.write_yaml(manifest, outfile)
def make_fermipy_config_yaml(self, modelkey, components, data, **kwargs):
"""Build a fermipy top-level yaml configuration file
Parameters
----------
modelkey : str
Key used to identify this particular model
components : list
The binning components used in this analysis
data : str
Path to file containing dataset definition
"""
model_dir = os.path.join('analysis', 'model_%s' % modelkey)
hpx_order = kwargs.get('hpx_order', 9)
self._name_factory.update_base_dict(data)
try:
model_info = self._models[modelkey]
except KeyError:
model_info = self.make_model_info(modelkey)
model_info.make_model_rois(components, self._name_factory)
#source_names = model_info.component_names
master_xml_mdl = os.path.basename(
self._name_factory.master_srcmdl_xml(modelkey=modelkey))
master_data = dict(scfile=self._name_factory.ft2file(fullpath=True),
cacheft1=False)
master_binning = dict(projtype='HPX',
coordsys=kwargs.get('coordsys', 'GAL'),
roiwidth=180.,
binsperdec=8,
hpx_ordering_scheme="RING",
hpx_order=hpx_order,
hpx_ebin=True)
master_fileio = dict(outdir=model_dir,
logfile=os.path.join(model_dir, 'fermipy.log'))
master_gtlike = dict(irfs=self._name_factory.irfs(**kwargs),
edisp_disable=['isodiff', 'diffuse', 'limb'])
master_selection = dict(glat=0., glon=0., radius=180.)
master_model = dict(catalogs=[master_xml_mdl])
master = dict(data=master_data,
binning=master_binning,
fileio=master_fileio,
selection=master_selection,
gtlike=master_gtlike,
model=master_model,
components=[])
fermipy_dict = master
#comp_rois = {}
for comp in components:
zcut = "zmax%i" % comp.zmax
compkey = "%s_%s" % (zcut, comp.make_key(
'{ebin_name}_{evtype_name}'))
name_keys = dict(zcut=zcut,
modelkey=modelkey,
component=compkey,
fullpath=True)
comp_data = dict(ltcube=self._name_factory.ltcube(**name_keys))
comp_selection = dict(logemin=comp.log_emin,
logemax=comp.log_emax,
zmax=comp.zmax,
evtype=comp.evtype)
comp_binning = dict(enumbins=comp.enumbins,
hpx_order=min(comp.hpx_order, hpx_order))
comp_gtlike = dict(srcmap=self._name_factory.merged_srcmaps(**name_keys),
bexpmap=self._name_factory.bexpcube(**name_keys))
#comp_roi_source_info = {}
comp_xml_mdl = os.path.basename(self._name_factory.comp_srcmdl_xml(modelkey=modelkey,
component=compkey))
comp_model = dict(catalogs=[master_xml_mdl, comp_xml_mdl])
sub_dict = dict(data=comp_data,
binning=comp_binning,
selection=comp_selection,
gtlike=comp_gtlike,
model=comp_model)
fermipy_dict['components'].append(sub_dict)
outfile = os.path.join(model_dir, 'config.yaml')
print ("Writing fermipy config file %s"%outfile)
utils.write_yaml(fermipy_dict, outfile)
return fermipy_dict
@staticmethod
def get_sub_comp_info(source_info, comp):
"""Build and return information about a sub-component for a particular selection
"""
sub_comps = source_info.get('components', None)
if sub_comps is None:
return source_info.copy()
moving = source_info.get('moving', False)
selection_dependent = source_info.get('selection_dependent', False)
if selection_dependent:
key = comp.make_key('{ebin_name}_{evtype_name}')
elif moving:
key = "zmax%i" % comp.zmax
ret_dict = source_info.copy()
ret_dict.update(sub_comps[key])
return ret_dict
def test_spectral():
the_yaml = """
Constant_Correction :
SpectrumType : ConstantValue
spectral_pars :
Value :
name : Value
scale : 1.0
value : 1.0
min : 1e-4
max: 1e4
free : False
Powerlaw_Correction :
SpectrumType : PowerLaw
spectral_pars :
Prefactor :
name : Prefactor
scale : 1.0
value : 1.0
min : 0.1
max: 10.0
free : False
Index :
name: Index
scale : -1.0
value : 0.0
min : -1.0
max : 1.0
free : False
Scale :
name: Scale
scale : 1.0
value : 1000.0
min : 1000.0
max : 1000.0
free : False
LogParabola_Correction :
SpectrumType : LogParabola
spectral_pars :
norm :
name : norm
scale : 1.0
value : 1.0
min : 0.1
max: 10.0
free : False
alpha :
name: alpha
scale : 1.0
value : 0.0
min : -2.0
max : 2.0
free : False
beta :
name: beta
scale : 1.0
value : 0.0
min : -10.0
max : 10.0
free : False
Eb :
name: Eb
scale : 1.0
value : 1000.0
min : 10.0
max : 100000.0
free : False
"""
spectra = SpectralLibrary.create_from_yamlstr(the_yaml)
# spot check some values
assert (spectra['Constant_Correction']['SpectrumType'] == 'ConstantValue')
assert (len(spectra['Constant_Correction']['spectral_pars']) == 1)
assert (spectra['Constant_Correction']['spectral_pars']['Value']['value']
== 1.0)
assert (spectra['Constant_Correction']['spectral_pars']['Value']['scale']
== 1.0)
#assert(spectra['Constant_Correction']['spectral_pars']['Value']['min'] == 1e-4)
#assert(spectra['Constant_Correction']['spectral_pars']['Value']['max'] == 1e4)
assert (spectra['Constant_Correction']['spectral_pars']['Value']['free'] is
False)
assert (spectra['LogParabola_Correction']['SpectrumType'] == 'LogParabola')
assert (len(spectra['LogParabola_Correction']['spectral_pars']) == 4)
assert (spectra['LogParabola_Correction']['spectral_pars']['norm']['value']
== 1.0)
assert (spectra['LogParabola_Correction']['spectral_pars']['norm']['scale']
== 1.0)
assert (spectra['LogParabola_Correction']['spectral_pars']['norm']['min']
== 0.1)
assert (spectra['LogParabola_Correction']['spectral_pars']['norm']['max']
== 10.0)
assert (spectra['LogParabola_Correction']['spectral_pars']['norm']['free']
is False)