def test_name_factory():
names = dict(data_pass='******',
data_ver='P302',
evclass='source',
data_time='8years',
zcut='zmax105',
psftype='PSF3',
ebin='E3',
coordsys='GAL',
irf_ver='V6',
sourcekey='ptsrc')
name_fact = NameFactory(**names)
test_dict = name_fact.make_filenames(**names)
test_irfs = name_fact.irfs(**names)
assert(test_dict['ltcube'] == 'lt_cubes/ltcube_8years_zmax105.fits')
assert(test_dict['ft1file'] == 'P8_P302_8years_source_zmax105.lst')
assert(test_dict['bexpcube'] ==
'bexp_cubes/bexcube_P8_P302_8years_source_zmax105_E3_PSF3_GAL_V6.fits')
assert(test_dict[
'srcmaps'] == 'srcmaps/srcmaps_ptsrc_P8_P302_8years_source_zmax105_E3_PSF3_GAL_V6.fits')
assert(test_dict[
'ccube'] == 'counts_cubes/ccube_P8_P302_8years_source_zmax105_E3_PSF3_GAL.fits')
assert(test_dict[
'mcube'] == 'model_cubes/mcube_ptsrc_P8_P302_8years_source_zmax105_E3_PSF3_GAL_V6.fits')
assert(test_irfs == 'P8R2_SOURCE_V6')
def test_name_factory():
names = dict(data_pass='******',
data_ver='P302',
evclass='source',
data_time='8years',
zcut='zmax105',
psftype='PSF3',
mktime='none',
ebin='E3',
coordsys='GAL',
irf_ver='V6',
sourcekey='ptsrc')
name_fact = NameFactory(**names)
test_dict = name_fact.make_filenames(**names)
test_irfs = name_fact.irfs(**names)
assert (test_dict['ltcube'] == 'lt_cubes/ltcube_8years_none_zmax105.fits')
assert (test_dict['ft1file'] == 'P8_P302_8years_source_zmax105.lst')
assert (
test_dict['bexpcube'] ==
'bexp_cubes/bexcube_P8_P302_8years_source_none_zmax105_E3_PSF3_GAL_V6.fits'
)
assert (
test_dict['srcmaps'] ==
'srcmaps/srcmaps_ptsrc_P8_P302_8years_source_none_zmax105_E3_PSF3_GAL_V6.fits'
)
assert (
test_dict['ccube'] ==
'counts_cubes/ccube_P8_P302_8years_source_none_zmax105_E3_PSF3_GAL.fits'
)
assert (
test_dict['mcube'] ==
'model_cubes/mcube_ptsrc_P8_P302_8years_source_none_zmax105_E3_PSF3_GAL_V6.fits'
)
assert (test_irfs == 'P8R2_SOURCE_V6')
class ModelManager(object):
""" Small helper class to create fitting models and manager XML files for fermipy
This class contains a 'library', which is a dictionary of all the source components:
specifically it maps:
sourcekey : `model_component.ModelComponentInfo`
"""
def __init__(self, **kwargs):
"""
Keyword arguments
-------------------
basedir : str
Top level directory for finding files
"""
self._name_factory = NameFactory(**kwargs)
self._dmm = kwargs.get('DiffuseModelManager',
DiffuseModelManager(**kwargs))
self._gmm = kwargs.get('GalpropMapManager',
GalpropMapManager(**kwargs))
self._csm = kwargs.get('CatalogSourceManager',
CatalogSourceManager(**kwargs))
self._library = {}
self._models = {}
self._spec_lib = SpectralLibrary({})
def read_model_yaml(self, modelkey):
""" Read the yaml file for the diffuse components
"""
model_yaml = self._name_factory.model_yaml(modelkey=modelkey,
fullpath=True)
model = yaml.safe_load(open(model_yaml))
return model
@property
def dmm(self):
""" Return the DiffuseModelManager """
return self._dmm
@property
def gmm(self):
""" Return the GalpropMapManager """
return self._gmm
@property
def csm(self):
""" Return the CatalogSourceManager """
return self._csm
def make_library(self, diffuse_yaml, catalog_yaml, binning_yaml):
""" Build up the library of all the components
Parameters
----------
diffuse_yaml : str
Name of the yaml file with the library of diffuse component definitions
catalog_yaml : str
Name of the yaml file width the library of catalog split definitions
binning_yaml : str
Name of the yaml file with the binning definitions
"""
ret_dict = {}
#catalog_dict = yaml.safe_load(open(catalog_yaml))
components_dict = Component.build_from_yamlfile(binning_yaml)
diffuse_ret_dict = make_diffuse_comp_info_dict(
GalpropMapManager=self._gmm,
DiffuseModelManager=self._dmm,
library=diffuse_yaml,
components=components_dict)
catalog_ret_dict = make_catalog_comp_dict(
library=catalog_yaml, CatalogSourceManager=self._csm)
ret_dict.update(diffuse_ret_dict['comp_info_dict'])
ret_dict.update(catalog_ret_dict['comp_info_dict'])
self._library.update(ret_dict)
return ret_dict
def make_model_info(self, modelkey):
""" Build a dictionary with the information for a particular model.
Parameters
----------
modelkey : str
Key used to identify this particular model
Return `ModelInfo`
"""
model = self.read_model_yaml(modelkey)
sources = model['sources']
components = OrderedDict()
spec_model_yaml = self._name_factory.fullpath(
localpath=model['spectral_models'])
self._spec_lib.update(yaml.safe_load(open(spec_model_yaml)))
for source, source_info in sources.items():
model_type = source_info.get('model_type', None)
par_overrides = source_info.get('par_overides', None)
version = source_info['version']
spec_type = source_info['SpectrumType']
edisp_disable = source_info.get('edisp_disable', False)
sourcekey = "%s_%s" % (source, version)
if model_type == 'galprop_rings':
comp_info_dict = self.gmm.diffuse_comp_info_dicts(version)
def_spec_type = spec_type['default']
for comp_key, comp_info in comp_info_dict.items():
model_comp = ModelComponent(info=comp_info,
spectrum=\
self._spec_lib[spec_type.get(comp_key,
def_spec_type)],
par_overrides=par_overrides,
edisp_disable=edisp_disable)
components[comp_key] = model_comp
elif model_type == 'Catalog':
comp_info_dict = self.csm.split_comp_info_dict(source, version)
def_spec_type = spec_type['default']
for comp_key, comp_info in comp_info_dict.items():
model_comp = ModelComponent(info=comp_info,
spectrum=\
self._spec_lib[spec_type.get(comp_key,
def_spec_type)],
par_overrides=par_overrides,
edisp_disable=edisp_disable)
components[comp_key] = model_comp
else:
comp_info = self.dmm.diffuse_comp_info(sourcekey)
model_comp = ModelComponent(info=comp_info,
spectrum=self._spec_lib[spec_type],
par_overrides=par_overrides,
edisp_disable=edisp_disable)
components[sourcekey] = model_comp
ret_val = ModelInfo(model_name=modelkey, model_components=components)
self._models[modelkey] = ret_val
return ret_val
def make_srcmap_manifest(self, modelkey, components, data):
"""Build a yaml file that specfies how to make the srcmap files for a particular model
Parameters
----------
modelkey : str
Key used to identify this particular model
components : list
The binning components used in this analysis
data : str
Path to file containing dataset definition
"""
try:
model_info = self._models[modelkey]
except KeyError:
model_info = self.make_model_info(modelkey)
self._name_factory.update_base_dict(data)
outfile = os.path.join('analysis', 'model_%s' % modelkey,
'srcmap_manifest_%s.yaml' % modelkey)
manifest = model_info.make_srcmap_manifest(components,
self._name_factory)
outdir = os.path.dirname(outfile)
try:
os.makedirs(outdir)
except OSError:
pass
utils.write_yaml(manifest, outfile)
def make_fermipy_config_yaml(self, modelkey, components, data, **kwargs):
"""Build a fermipy top-level yaml configuration file
Parameters
----------
modelkey : str
Key used to identify this particular model
components : list
The binning components used in this analysis
data : str
Path to file containing dataset definition
"""
model_dir = os.path.join('analysis', 'model_%s' % modelkey)
hpx_order = kwargs.get('hpx_order', 9)
self._name_factory.update_base_dict(data)
try:
model_info = self._models[modelkey]
except KeyError:
model_info = self.make_model_info(modelkey)
model_rois = model_info.make_model_rois(components, self._name_factory)
#source_names = model_info.component_names
master_xml_mdl = os.path.basename(
self._name_factory.master_srcmdl_xml(modelkey=modelkey))
master_data = dict(scfile=self._name_factory.ft2file(fullpath=True),
cacheft1=False)
master_binning = dict(projtype='HPX',
roiwidth=360.,
binsperdec=4,
hpx_ordering_scheme="RING",
hpx_order=hpx_order,
hpx_ebin=True)
# master_fileio = dict(outdir=model_dir,
# logfile=os.path.join(model_dir, 'fermipy.log'))
master_fileio = dict(logfile='fermipy.log')
master_gtlike = dict(irfs=self._name_factory.irfs(**kwargs),
edisp_disable=model_info.edisp_disable_list(),
use_external_srcmap=True)
master_selection = dict(glat=0., glon=0., radius=180.)
master_model = dict(catalogs=[master_xml_mdl])
master_plotting = dict(label_ts_threshold=1e9)
master = dict(data=master_data,
binning=master_binning,
fileio=master_fileio,
selection=master_selection,
gtlike=master_gtlike,
model=master_model,
plotting=master_plotting,
components=[])
fermipy_dict = master
#comp_rois = {}
for comp in components:
zcut = "zmax%i" % comp.zmax
compkey = "%s_%s" % (zcut,
comp.make_key('{ebin_name}_{evtype_name}'))
comp_roi = model_rois[compkey]
name_keys = dict(zcut=zcut,
modelkey=modelkey,
component=compkey,
mktime='none',
coordsys=comp.coordsys,
fullpath=True)
comp_data = dict(ltcube=self._name_factory.ltcube(**name_keys))
comp_selection = dict(logemin=comp.log_emin,
logemax=comp.log_emax,
zmax=comp.zmax,
evtype=comp.evtype)
comp_binning = dict(enumbins=comp.enumbins,
hpx_order=min(comp.hpx_order, hpx_order),
coordsys=comp.coordsys)
comp_gtlike = dict(
srcmap=self._name_factory.merged_srcmaps(**name_keys),
bexpmap=self._name_factory.bexpcube(**name_keys),
use_external_srcmap=True)
#comp_roi_source_info = {}
diffuse_srcs = []
for src in comp_roi.diffuse_sources:
if isinstance(src, MapCubeSource):
diffuse_srcs.append(dict(name=src.name, file=src.mapcube))
elif isinstance(src, IsoSource):
diffuse_srcs.append(
dict(name=src.name, file=src.fileFunction))
else:
pass
comp_model = dict(diffuse=diffuse_srcs)
sub_dict = dict(data=comp_data,
binning=comp_binning,
selection=comp_selection,
gtlike=comp_gtlike,
model=comp_model)
fermipy_dict['components'].append(sub_dict)
# Add placeholder diffuse sources
fermipy_dict['model']['diffuse'] = diffuse_srcs
outfile = os.path.join(model_dir, 'config.yaml')
print("Writing fermipy config file %s" % outfile)
utils.write_yaml(fermipy_dict, outfile)
return fermipy_dict
@staticmethod
def get_sub_comp_info(source_info, comp):
"""Build and return information about a sub-component for a particular selection
"""
sub_comps = source_info.get('components', None)
if sub_comps is None:
return source_info.copy()
moving = source_info.get('moving', False)
selection_dependent = source_info.get('selection_dependent', False)
if selection_dependent:
key = comp.make_key('{ebin_name}_{evtype_name}')
elif moving:
key = "zmax%i" % comp.zmax
ret_dict = source_info.copy()
ret_dict.update(sub_comps[key])
return ret_dict
class ModelManager(object):
""" Small helper class to create fitting models and manager XML files for fermipy
This class contains a 'library', which is a dictionary of all the source components:
specifically it maps:
sourcekey : `model_component.ModelComponentInfo`
"""
def __init__(self, **kwargs):
"""
Keyword arguments
-------------------
basedir : str
Top level directory for finding files
"""
self._name_factory = NameFactory(**kwargs)
self._dmm = kwargs.get('DiffuseModelManager',
DiffuseModelManager(**kwargs))
self._gmm = kwargs.get('GalpropMapManager',
GalpropMapManager(**kwargs))
self._csm = kwargs.get('CatalogSourceManager',
CatalogSourceManager(**kwargs))
self._library = {}
self._models = {}
self._spec_lib = SpectralLibrary({})
def read_model_yaml(self, modelkey):
""" Read the yaml file for the diffuse components
"""
model_yaml = self._name_factory.model_yaml(modelkey=modelkey,
fullpath=True)
model = yaml.safe_load(open(model_yaml))
return model
@property
def dmm(self):
""" Return the DiffuseModelManager """
return self._dmm
@property
def gmm(self):
""" Return the GalpropMapManager """
return self._gmm
@property
def csm(self):
""" Return the CatalogSourceManager """
return self._csm
def make_library(self, diffuse_yaml, catalog_yaml, binning_yaml):
""" Build up the library of all the components
Parameters
----------
diffuse_yaml : str
Name of the yaml file with the library of diffuse component definitions
catalog_yaml : str
Name of the yaml file with the library of catalog split definitions
binning_yaml : str
Name of the yaml file with the binning definitions
"""
ret_dict = {}
#catalog_dict = yaml.safe_load(open(catalog_yaml))
components_dict = Component.build_from_yamlfile(binning_yaml)
diffuse_ret_dict = make_diffuse_comp_info_dict(GalpropMapManager=self._gmm,
DiffuseModelManager=self._dmm,
diffuse=diffuse_yaml,
components=components_dict)
catalog_ret_dict = make_catalog_comp_dict(sources=catalog_yaml,
CatalogSourceManager=self._csm)
ret_dict.update(diffuse_ret_dict['comp_info_dict'])
ret_dict.update(catalog_ret_dict['comp_info_dict'])
self._library.update(ret_dict)
return ret_dict
def make_model_info(self, modelkey):
""" Build a dictionary with the information for a particular model.
Parameters
----------
modelkey : str
Key used to identify this particular model
Return `ModelInfo`
"""
model = self.read_model_yaml(modelkey)
sources = model['sources']
components = OrderedDict()
spec_model_yaml = self._name_factory.fullpath(localpath=model['spectral_models'])
self._spec_lib.update(yaml.safe_load(open(spec_model_yaml)))
for source, source_info in sources.items():
model_type = source_info.get('model_type', None)
version = source_info['version']
spec_type = source_info['SpectrumType']
sourcekey = "%s_%s" % (source, version)
if model_type == 'galprop_rings':
comp_info_dict = self.gmm.diffuse_comp_info_dicts(version)
def_spec_type = spec_type['default']
for comp_key, comp_info in comp_info_dict.items():
model_comp = ModelComponent(info=comp_info,
spectrum=\
self._spec_lib[spec_type.get(comp_key,
def_spec_type)])
components[comp_key] = model_comp
elif model_type == 'Catalog':
comp_info_dict = self.csm.split_comp_info_dict(source, version)
def_spec_type = spec_type['default']
for comp_key, comp_info in comp_info_dict.items():
model_comp = ModelComponent(info=comp_info,
spectrum=\
self._spec_lib[spec_type.get(comp_key,
def_spec_type)])
components[comp_key] = model_comp
else:
comp_info = self.dmm.diffuse_comp_info(sourcekey)
model_comp = ModelComponent(info=comp_info,
spectrum=self._spec_lib[spec_type])
components[sourcekey] = model_comp
ret_val = ModelInfo(model_name=modelkey,
model_components=components)
self._models[modelkey] = ret_val
return ret_val
def make_srcmap_manifest(self, modelkey, components, data):
"""Build a yaml file that specfies how to make the srcmap files for a particular model
Parameters
----------
modelkey : str
Key used to identify this particular model
components : list
The binning components used in this analysis
data : str
Path to file containing dataset definition
"""
try:
model_info = self._models[modelkey]
except KeyError:
model_info = self.make_model_info(modelkey)
self._name_factory.update_base_dict(data)
outfile = os.path.join('analysis', 'model_%s' %
modelkey, 'srcmap_manifest_%s.yaml' % modelkey)
manifest = model_info.make_srcmap_manifest(
components, self._name_factory)
outdir = os.path.dirname(outfile)
try:
os.makedirs(outdir)
except OSError:
pass
utils.write_yaml(manifest, outfile)
def make_fermipy_config_yaml(self, modelkey, components, data, **kwargs):
"""Build a fermipy top-level yaml configuration file
Parameters
----------
modelkey : str
Key used to identify this particular model
components : list
The binning components used in this analysis
data : str
Path to file containing dataset definition
"""
model_dir = os.path.join('analysis', 'model_%s' % modelkey)
hpx_order = kwargs.get('hpx_order', 9)
self._name_factory.update_base_dict(data)
try:
model_info = self._models[modelkey]
except KeyError:
model_info = self.make_model_info(modelkey)
model_info.make_model_rois(components, self._name_factory)
#source_names = model_info.component_names
master_xml_mdl = os.path.basename(
self._name_factory.master_srcmdl_xml(modelkey=modelkey))
master_data = dict(scfile=self._name_factory.ft2file(fullpath=True),
cacheft1=False)
master_binning = dict(projtype='HPX',
coordsys=kwargs.get('coordsys', 'GAL'),
roiwidth=180.,
binsperdec=8,
hpx_ordering_scheme="RING",
hpx_order=hpx_order,
hpx_ebin=True)
master_fileio = dict(outdir=model_dir,
logfile=os.path.join(model_dir, 'fermipy.log'))
master_gtlike = dict(irfs=self._name_factory.irfs(**kwargs),
edisp_disable=['isodiff', 'diffuse', 'limb'])
master_selection = dict(glat=0., glon=0., radius=180.)
master_model = dict(catalogs=[master_xml_mdl])
master = dict(data=master_data,
binning=master_binning,
fileio=master_fileio,
selection=master_selection,
gtlike=master_gtlike,
model=master_model,
components=[])
fermipy_dict = master
#comp_rois = {}
for comp in components:
zcut = "zmax%i" % comp.zmax
compkey = "%s_%s" % (zcut, comp.make_key(
'{ebin_name}_{evtype_name}'))
name_keys = dict(zcut=zcut,
modelkey=modelkey,
component=compkey,
fullpath=True)
comp_data = dict(ltcube=self._name_factory.ltcube(**name_keys))
comp_selection = dict(logemin=comp.log_emin,
logemax=comp.log_emax,
zmax=comp.zmax,
evtype=comp.evtype)
comp_binning = dict(enumbins=comp.enumbins,
hpx_order=min(comp.hpx_order, hpx_order))
comp_gtlike = dict(srcmap=self._name_factory.merged_srcmaps(**name_keys),
bexpmap=self._name_factory.bexpcube(**name_keys))
#comp_roi_source_info = {}
comp_xml_mdl = os.path.basename(self._name_factory.comp_srcmdl_xml(modelkey=modelkey,
component=compkey))
comp_model = dict(catalogs=[master_xml_mdl, comp_xml_mdl])
sub_dict = dict(data=comp_data,
binning=comp_binning,
selection=comp_selection,
gtlike=comp_gtlike,
model=comp_model)
fermipy_dict['components'].append(sub_dict)
outfile = os.path.join(model_dir, 'config.yaml')
print ("Writing fermipy config file %s"%outfile)
utils.write_yaml(fermipy_dict, outfile)
return fermipy_dict
@staticmethod
def get_sub_comp_info(source_info, comp):
"""Build and return information about a sub-component for a particular selection
"""
sub_comps = source_info.get('components', None)
if sub_comps is None:
return source_info.copy()
moving = source_info.get('moving', False)
selection_dependent = source_info.get('selection_dependent', False)
if selection_dependent:
key = comp.make_key('{ebin_name}_{evtype_name}')
elif moving:
key = "zmax%i" % comp.zmax
ret_dict = source_info.copy()
ret_dict.update(sub_comps[key])
return ret_dict
