def test_name_factory():
names = dict(data_pass='******',
data_ver='P302',
evclass='source',
data_time='8years',
zcut='zmax105',
psftype='PSF3',
ebin='E3',
coordsys='GAL',
irf_ver='V6',
sourcekey='ptsrc')
name_fact = NameFactory(**names)
test_dict = name_fact.make_filenames(**names)
test_irfs = name_fact.irfs(**names)
assert(test_dict['ltcube'] == 'lt_cubes/ltcube_8years_zmax105.fits')
assert(test_dict['ft1file'] == 'P8_P302_8years_source_zmax105.lst')
assert(test_dict['bexpcube'] ==
'bexp_cubes/bexcube_P8_P302_8years_source_zmax105_E3_PSF3_GAL_V6.fits')
assert(test_dict[
'srcmaps'] == 'srcmaps/srcmaps_ptsrc_P8_P302_8years_source_zmax105_E3_PSF3_GAL_V6.fits')
assert(test_dict[
'ccube'] == 'counts_cubes/ccube_P8_P302_8years_source_zmax105_E3_PSF3_GAL.fits')
assert(test_dict[
'mcube'] == 'model_cubes/mcube_ptsrc_P8_P302_8years_source_zmax105_E3_PSF3_GAL_V6.fits')
assert(test_irfs == 'P8R2_SOURCE_V6')
def __init__(self, **kwargs):
""" C'tor
Keyword arguments
-----------------
basedir : str
Top level directory for finding files
"""
self._name_factory = NameFactory(**kwargs)
self._catalog_comp_info_dicts = {}
self._split_comp_info_dicts = {}
def __init__(self, **kwargs):
""" C'tor
Keyword arguments
-----------------
name_policy : str
Name of yaml file contain file naming policy definitions
basedir : str
Top level directory for finding files
"""
self._name_factory = NameFactory(basedir=kwargs.get('basedir'))
self._diffuse_comp_info_dict = {}
def __init__(self, **kwargs):
""" C'tor
Keyword arguments
-----------------
projtype : str [healpix]
Used to define path to gasmap files
basedir : str
Top level directory for finding files
"""
self.projtype = kwargs.get('projtype', 'healpix')
self._name_factory = NameFactory(basedir=kwargs.get('basedir'))
self._ring_dicts = {}
self._diffuse_comp_info_dicts = {}
def __init__(self, **kwargs):
"""
Keyword arguments
-------------------
basedir : str
Top level directory for finding files
"""
self._name_factory = NameFactory(**kwargs)
self._dmm = kwargs.get('DiffuseModelManager',
DiffuseModelManager(**kwargs))
self._gmm = kwargs.get('GalpropMapManager',
GalpropMapManager(**kwargs))
self._csm = kwargs.get('CatalogSourceManager',
CatalogSourceManager(**kwargs))
self._library = {}
self._models = {}
self._spec_lib = SpectralLibrary({})
def __init__(self, **kwargs):
""" C'tor
Keyword arguments
-----------------
basedir : str
Top level directory for finding files
"""
self._name_factory = NameFactory(**kwargs)
self._catalog_comp_info_dicts = {}
self._split_comp_info_dicts = {}
def __init__(self, **kwargs):
""" C'tor
Keyword arguments
-----------------
name_policy : str
Name of yaml file contain file naming policy definitions
basedir : str
Top level directory for finding files
"""
self._name_factory = NameFactory(basedir=kwargs.get('basedir'))
self._diffuse_comp_info_dict = {}
def test_name_factory():
names = dict(data_pass='******',
data_ver='P302',
evclass='source',
data_time='8years',
zcut='zmax105',
psftype='PSF3',
mktime='none',
ebin='E3',
coordsys='GAL',
irf_ver='V6',
sourcekey='ptsrc')
name_fact = NameFactory(**names)
test_dict = name_fact.make_filenames(**names)
test_irfs = name_fact.irfs(**names)
assert (test_dict['ltcube'] == 'lt_cubes/ltcube_8years_none_zmax105.fits')
assert (test_dict['ft1file'] == 'P8_P302_8years_source_zmax105.lst')
assert (
test_dict['bexpcube'] ==
'bexp_cubes/bexcube_P8_P302_8years_source_none_zmax105_E3_PSF3_GAL_V6.fits'
)
assert (
test_dict['srcmaps'] ==
'srcmaps/srcmaps_ptsrc_P8_P302_8years_source_none_zmax105_E3_PSF3_GAL_V6.fits'
)
assert (
test_dict['ccube'] ==
'counts_cubes/ccube_P8_P302_8years_source_none_zmax105_E3_PSF3_GAL.fits'
)
assert (
test_dict['mcube'] ==
'model_cubes/mcube_ptsrc_P8_P302_8years_source_none_zmax105_E3_PSF3_GAL_V6.fits'
)
assert (test_irfs == 'P8R2_SOURCE_V6')
def __init__(self, **kwargs):
""" C'tor
Keyword arguments
-----------------
projtype : str [healpix]
Used to define path to gasmap files
basedir : str
Top level directory for finding files
"""
self.projtype = kwargs.get('projtype', 'healpix')
self._name_factory = NameFactory(basedir=kwargs.get('basedir'))
self._ring_dicts = {}
self._diffuse_comp_info_dicts = {}
def __init__(self, **kwargs):
"""
Keyword arguments
-------------------
basedir : str
Top level directory for finding files
"""
self._name_factory = NameFactory(**kwargs)
self._dmm = kwargs.get('DiffuseModelManager',
DiffuseModelManager(**kwargs))
self._gmm = kwargs.get('GalpropMapManager',
GalpropMapManager(**kwargs))
self._csm = kwargs.get('CatalogSourceManager',
CatalogSourceManager(**kwargs))
self._library = {}
self._models = {}
self._spec_lib = SpectralLibrary({})
class ModelManager(object):
""" Small helper class to create fitting models and manager XML files for fermipy
This class contains a 'library', which is a dictionary of all the source components:
specifically it maps:
sourcekey : `model_component.ModelComponentInfo`
"""
def __init__(self, **kwargs):
"""
Keyword arguments
-------------------
basedir : str
Top level directory for finding files
"""
self._name_factory = NameFactory(**kwargs)
self._dmm = kwargs.get('DiffuseModelManager',
DiffuseModelManager(**kwargs))
self._gmm = kwargs.get('GalpropMapManager',
GalpropMapManager(**kwargs))
self._csm = kwargs.get('CatalogSourceManager',
CatalogSourceManager(**kwargs))
self._library = {}
self._models = {}
self._spec_lib = SpectralLibrary({})
def read_model_yaml(self, modelkey):
""" Read the yaml file for the diffuse components
"""
model_yaml = self._name_factory.model_yaml(modelkey=modelkey,
fullpath=True)
model = yaml.safe_load(open(model_yaml))
return model
@property
def dmm(self):
""" Return the DiffuseModelManager """
return self._dmm
@property
def gmm(self):
""" Return the GalpropMapManager """
return self._gmm
@property
def csm(self):
""" Return the CatalogSourceManager """
return self._csm
def make_library(self, diffuse_yaml, catalog_yaml, binning_yaml):
""" Build up the library of all the components
Parameters
----------
diffuse_yaml : str
Name of the yaml file with the library of diffuse component definitions
catalog_yaml : str
Name of the yaml file width the library of catalog split definitions
binning_yaml : str
Name of the yaml file with the binning definitions
"""
ret_dict = {}
#catalog_dict = yaml.safe_load(open(catalog_yaml))
components_dict = Component.build_from_yamlfile(binning_yaml)
diffuse_ret_dict = make_diffuse_comp_info_dict(
GalpropMapManager=self._gmm,
DiffuseModelManager=self._dmm,
library=diffuse_yaml,
components=components_dict)
catalog_ret_dict = make_catalog_comp_dict(
library=catalog_yaml, CatalogSourceManager=self._csm)
ret_dict.update(diffuse_ret_dict['comp_info_dict'])
ret_dict.update(catalog_ret_dict['comp_info_dict'])
self._library.update(ret_dict)
return ret_dict
def make_model_info(self, modelkey):
""" Build a dictionary with the information for a particular model.
Parameters
----------
modelkey : str
Key used to identify this particular model
Return `ModelInfo`
"""
model = self.read_model_yaml(modelkey)
sources = model['sources']
components = OrderedDict()
spec_model_yaml = self._name_factory.fullpath(
localpath=model['spectral_models'])
self._spec_lib.update(yaml.safe_load(open(spec_model_yaml)))
for source, source_info in sources.items():
model_type = source_info.get('model_type', None)
par_overrides = source_info.get('par_overides', None)
version = source_info['version']
spec_type = source_info['SpectrumType']
edisp_disable = source_info.get('edisp_disable', False)
sourcekey = "%s_%s" % (source, version)
if model_type == 'galprop_rings':
comp_info_dict = self.gmm.diffuse_comp_info_dicts(version)
def_spec_type = spec_type['default']
for comp_key, comp_info in comp_info_dict.items():
model_comp = ModelComponent(info=comp_info,
spectrum=\
self._spec_lib[spec_type.get(comp_key,
def_spec_type)],
par_overrides=par_overrides,
edisp_disable=edisp_disable)
components[comp_key] = model_comp
elif model_type == 'Catalog':
comp_info_dict = self.csm.split_comp_info_dict(source, version)
def_spec_type = spec_type['default']
for comp_key, comp_info in comp_info_dict.items():
model_comp = ModelComponent(info=comp_info,
spectrum=\
self._spec_lib[spec_type.get(comp_key,
def_spec_type)],
par_overrides=par_overrides,
edisp_disable=edisp_disable)
components[comp_key] = model_comp
else:
comp_info = self.dmm.diffuse_comp_info(sourcekey)
model_comp = ModelComponent(info=comp_info,
spectrum=self._spec_lib[spec_type],
par_overrides=par_overrides,
edisp_disable=edisp_disable)
components[sourcekey] = model_comp
ret_val = ModelInfo(model_name=modelkey, model_components=components)
self._models[modelkey] = ret_val
return ret_val
def make_srcmap_manifest(self, modelkey, components, data):
"""Build a yaml file that specfies how to make the srcmap files for a particular model
Parameters
----------
modelkey : str
Key used to identify this particular model
components : list
The binning components used in this analysis
data : str
Path to file containing dataset definition
"""
try:
model_info = self._models[modelkey]
except KeyError:
model_info = self.make_model_info(modelkey)
self._name_factory.update_base_dict(data)
outfile = os.path.join('analysis', 'model_%s' % modelkey,
'srcmap_manifest_%s.yaml' % modelkey)
manifest = model_info.make_srcmap_manifest(components,
self._name_factory)
outdir = os.path.dirname(outfile)
try:
os.makedirs(outdir)
except OSError:
pass
utils.write_yaml(manifest, outfile)
def make_fermipy_config_yaml(self, modelkey, components, data, **kwargs):
"""Build a fermipy top-level yaml configuration file
Parameters
----------
modelkey : str
Key used to identify this particular model
components : list
The binning components used in this analysis
data : str
Path to file containing dataset definition
"""
model_dir = os.path.join('analysis', 'model_%s' % modelkey)
hpx_order = kwargs.get('hpx_order', 9)
self._name_factory.update_base_dict(data)
try:
model_info = self._models[modelkey]
except KeyError:
model_info = self.make_model_info(modelkey)
model_rois = model_info.make_model_rois(components, self._name_factory)
#source_names = model_info.component_names
master_xml_mdl = os.path.basename(
self._name_factory.master_srcmdl_xml(modelkey=modelkey))
master_data = dict(scfile=self._name_factory.ft2file(fullpath=True),
cacheft1=False)
master_binning = dict(projtype='HPX',
roiwidth=360.,
binsperdec=4,
hpx_ordering_scheme="RING",
hpx_order=hpx_order,
hpx_ebin=True)
# master_fileio = dict(outdir=model_dir,
# logfile=os.path.join(model_dir, 'fermipy.log'))
master_fileio = dict(logfile='fermipy.log')
master_gtlike = dict(irfs=self._name_factory.irfs(**kwargs),
edisp_disable=model_info.edisp_disable_list(),
use_external_srcmap=True)
master_selection = dict(glat=0., glon=0., radius=180.)
master_model = dict(catalogs=[master_xml_mdl])
master_plotting = dict(label_ts_threshold=1e9)
master = dict(data=master_data,
binning=master_binning,
fileio=master_fileio,
selection=master_selection,
gtlike=master_gtlike,
model=master_model,
plotting=master_plotting,
components=[])
fermipy_dict = master
#comp_rois = {}
for comp in components:
zcut = "zmax%i" % comp.zmax
compkey = "%s_%s" % (zcut,
comp.make_key('{ebin_name}_{evtype_name}'))
comp_roi = model_rois[compkey]
name_keys = dict(zcut=zcut,
modelkey=modelkey,
component=compkey,
mktime='none',
coordsys=comp.coordsys,
fullpath=True)
comp_data = dict(ltcube=self._name_factory.ltcube(**name_keys))
comp_selection = dict(logemin=comp.log_emin,
logemax=comp.log_emax,
zmax=comp.zmax,
evtype=comp.evtype)
comp_binning = dict(enumbins=comp.enumbins,
hpx_order=min(comp.hpx_order, hpx_order),
coordsys=comp.coordsys)
comp_gtlike = dict(
srcmap=self._name_factory.merged_srcmaps(**name_keys),
bexpmap=self._name_factory.bexpcube(**name_keys),
use_external_srcmap=True)
#comp_roi_source_info = {}
diffuse_srcs = []
for src in comp_roi.diffuse_sources:
if isinstance(src, MapCubeSource):
diffuse_srcs.append(dict(name=src.name, file=src.mapcube))
elif isinstance(src, IsoSource):
diffuse_srcs.append(
dict(name=src.name, file=src.fileFunction))
else:
pass
comp_model = dict(diffuse=diffuse_srcs)
sub_dict = dict(data=comp_data,
binning=comp_binning,
selection=comp_selection,
gtlike=comp_gtlike,
model=comp_model)
fermipy_dict['components'].append(sub_dict)
# Add placeholder diffuse sources
fermipy_dict['model']['diffuse'] = diffuse_srcs
outfile = os.path.join(model_dir, 'config.yaml')
print("Writing fermipy config file %s" % outfile)
utils.write_yaml(fermipy_dict, outfile)
return fermipy_dict
@staticmethod
def get_sub_comp_info(source_info, comp):
"""Build and return information about a sub-component for a particular selection
"""
sub_comps = source_info.get('components', None)
if sub_comps is None:
return source_info.copy()
moving = source_info.get('moving', False)
selection_dependent = source_info.get('selection_dependent', False)
if selection_dependent:
key = comp.make_key('{ebin_name}_{evtype_name}')
elif moving:
key = "zmax%i" % comp.zmax
ret_dict = source_info.copy()
ret_dict.update(sub_comps[key])
return ret_dict
class DiffuseModelManager(object):
""" Small helper class to keep track of diffuse component templates
This keeps track of the 'diffuse component infomation' dictionary
This keyed by: key = {source_name}_{source_ver}
Where:
{source_name} is something like 'loopI'
{source_ver} is somthinng like v00
The dictioary is
diffuse_comp_info_dict[key] - > `model_component.ModelComponentInfo`
Note that some components ( those that represent moving sources or are selection depedent )
will have a sub-dictionary of diffuse_comp_info_dict object for each sub-component
The compoents are defined in a file called
config/diffuse_components.yaml
"""
def __init__(self, **kwargs):
""" C'tor
Keyword arguments
-----------------
name_policy : str
Name of yaml file contain file naming policy definitions
basedir : str
Top level directory for finding files
"""
self._name_factory = NameFactory(basedir=kwargs.get('basedir'))
self._diffuse_comp_info_dict = {}
@staticmethod
def read_diffuse_component_yaml(yamlfile):
""" Read the yaml file for the diffuse components
"""
diffuse_components = yaml.safe_load(open(yamlfile))
return diffuse_components
def sourcekeys(self):
"""Return the list of source keys"""
return sorted(self._diffuse_comp_info_dict.keys())
def diffuse_comp_info(self, sourcekey):
"""Return the Component info associated to a particular key
"""
return self._diffuse_comp_info_dict[sourcekey]
def make_template_name(self, model_type, sourcekey):
""" Make the name of a template file for particular component
Parameters
----------
model_type : str
Type of model to use for this component
sourcekey : str
Key to identify this component
Returns filename or None if component does not require a template file
"""
format_dict = self.__dict__.copy()
format_dict['sourcekey'] = sourcekey
if model_type == 'IsoSource':
return self._name_factory.spectral_template(**format_dict)
elif model_type == 'MapCubeSource':
return self._name_factory.diffuse_template(**format_dict)
else:
raise ValueError("Unexpected model_type %s" % model_type)
def make_xml_name(self, sourcekey):
""" Make the name of an xml file for a model definition of a single component
Parameters
----------
sourcekey : str
Key to identify this component
"""
format_dict = self.__dict__.copy()
format_dict['sourcekey'] = sourcekey
return self._name_factory.srcmdl_xml(**format_dict)
def make_diffuse_comp_info(self,
source_name,
source_ver,
diffuse_dict,
components=None,
comp_key=None):
""" Make a dictionary mapping the merged component names to list of template files
Parameters
----------
source_name : str
Name of the source
source_ver : str
Key identifying the version of the source
diffuse_dict : dict
Information about this component
comp_key : str
Used when we need to keep track of sub-components, i.e.,
for moving and selection dependent sources.
Returns `model_component.ModelComponentInfo` or
`model_component.IsoComponentInfo`
"""
model_type = diffuse_dict['model_type']
sourcekey = '%s_%s' % (source_name, source_ver)
if comp_key is None:
template_name = self.make_template_name(model_type, sourcekey)
srcmdl_name = self.make_xml_name(sourcekey)
else:
template_name = self.make_template_name(
model_type, "%s_%s" % (sourcekey, comp_key))
srcmdl_name = self.make_xml_name("%s_%s" % (sourcekey, comp_key))
template_name = self._name_factory.fullpath(localpath=template_name)
srcmdl_name = self._name_factory.fullpath(localpath=srcmdl_name)
kwargs = dict(source_name=source_name,
source_ver=source_ver,
model_type=model_type,
srcmdl_name=srcmdl_name,
components=components,
comp_key=comp_key)
kwargs.update(diffuse_dict)
if model_type == 'IsoSource':
kwargs['Spectral_Filename'] = template_name
return IsoComponentInfo(**kwargs)
elif model_type == 'MapCubeSource':
kwargs['Spatial_Filename'] = template_name
return MapCubeComponentInfo(**kwargs)
else:
raise ValueError("Unexpected model type %s" % model_type)
def make_diffuse_comp_info_dict(self, diffuse_sources, components):
""" Make a dictionary maping from diffuse component to information about that component
Parameters
----------
diffuse_sources : dict
Dictionary with diffuse source defintions
components : dict
Dictionary with event selection defintions,
needed for selection depenedent diffuse components
Returns
-------
ret_dict : dict
Dictionary mapping sourcekey to `model_component.ModelComponentInfo`
"""
ret_dict = {}
for key, value in diffuse_sources.items():
model_type = value.get('model_type', 'mapcube')
if model_type == 'galprop_rings':
continue
selection_dependent = value.get('selection_dependent', False)
moving = value.get('moving', False)
versions = value.get('versions', [])
for version in versions:
#sourcekey = self._name_factory.sourcekey(source_name=key,
# source_ver=version)
comp_dict = None
if selection_dependent:
# For selection dependent diffuse sources we need to split
# by binning component
comp_dict = {}
for comp in components:
comp_key = comp.make_key('{ebin_name}_{evtype_name}')
comp_dict[comp_key] = self.make_diffuse_comp_info(
key, version, value, None, comp_key)
elif moving:
# For moving diffuse sources we need to split by zmax cut
comp_dict = {}
zmax_dict = {}
for comp in components:
zmax_dict[int(comp.zmax)] = True
zmax_list = sorted(zmax_dict.keys())
for zmax in zmax_list:
comp_key = "zmax%i" % (zmax)
comp_dict[comp_key] = self.make_diffuse_comp_info(
key, version, value, None, comp_key)
comp_info = self.make_diffuse_comp_info(
key, version, value, comp_dict)
ret_dict[comp_info.sourcekey] = comp_info
self._diffuse_comp_info_dict.update(ret_dict)
return ret_dict
class GalpropMapManager(object):
""" Small helper class to keep track of Galprop gasmaps
This keeps track of two types of dictionaries.
Both are keyed by: key = {source_name}_{ring}_{galkey}
Where:
{source_name} is something like 'merged_C0'
{ring} is the ring index
{galkey} is a key specifying which version of galprop rings to use.
The two dictionaries are:
ring_dict[key] = `model_component.GalpropMergedRingInfo`
diffuse_comp_info_dict[key] ] `model_component.ModelComponentInfo`
The dictionaries are defined in files called.
models/galprop_rings_{galkey}.yaml
"""
def __init__(self, **kwargs):
""" C'tor
Keyword arguments
-----------------
maptype : str [newGasMaps_ST]
Used to define path to gasmap files
projtype : str [healpix]
Used to define path to gasmap files
basedir : str
Top level directory for finding files
"""
self.maptype = kwargs.get('maptype', 'newGasMaps_ST')
self.projtype = kwargs.get('projtype', 'healpix')
self._name_factory = NameFactory(basedir=kwargs.get('basedir'))
self._ring_dicts = {}
self._diffuse_comp_info_dicts = {}
def read_galprop_rings_yaml(self, galkey):
""" Read the yaml file for a partiuclar galprop key
"""
galprop_rings_yaml = self._name_factory.galprop_rings_yaml(
galkey=galkey, fullpath=True)
galprop_rings = yaml.safe_load(open(galprop_rings_yaml))
return galprop_rings
def galkeys(self):
""" Return the list of galprop keys used """
return sorted(self._ring_dicts.keys())
def ring_dict(self, galkey):
""" Return the ring dictionary for a particular galprop key
"""
return self._ring_dicts[galkey]
def diffuse_comp_info_dicts(self, galkey):
""" Return the components info dictionary for a particular galprop key
"""
return self._diffuse_comp_info_dicts[galkey]
def merged_components(self, galkey):
""" Return the set of merged components for a particular galprop key
"""
return sorted(self._diffuse_comp_info_dicts[galkey].keys())
def make_ring_filename(self, source_name, ring, galprop_run):
""" Make the name of a gasmap file for a single ring
Parameters
----------
source_name : str
The galprop component, used to define path to gasmap files
ring : int
The ring index
galprop_run : str
String identifying the galprop parameters
"""
format_dict = self.__dict__.copy()
format_dict['sourcekey'] = self._name_factory.galprop_ringkey(
source_name=source_name, ringkey="ring_%i" % ring)
format_dict['galprop_run'] = galprop_run
return self._name_factory.galprop_gasmap(**format_dict)
def make_merged_name(self, source_name, galkey, fullpath):
""" Make the name of a gasmap file for a set of merged rings
Parameters
----------
source_name : str
The galprop component, used to define path to gasmap files
galkey : str
A short key identifying the galprop parameters
fullpath : bool
Return the full path name
"""
format_dict = self.__dict__.copy()
format_dict['sourcekey'] = self._name_factory.galprop_sourcekey(
source_name=source_name, galpropkey=galkey)
format_dict['fullpath'] = fullpath
return self._name_factory.merged_gasmap(**format_dict)
def make_xml_name(self, source_name, galkey, fullpath):
""" Make the name of an xml file for a model definition for a set of merged rings
Parameters
----------
source_name : str
The galprop component, used to define path to gasmap files
galkey : str
A short key identifying the galprop parameters
fullpath : bool
Return the full path name
"""
format_dict = self.__dict__.copy()
format_dict['sourcekey'] = self._name_factory.galprop_sourcekey(
source_name=source_name, galpropkey=galkey)
format_dict['fullpath'] = fullpath
return self._name_factory.srcmdl_xml(**format_dict)
def make_ring_filelist(self, sourcekeys, rings, galprop_run):
""" Make a list of all the template files for a merged component
Parameters
----------
sourcekeys : list-like of str
The names of the componenents to merge
rings : list-like of int
The indices of the rings to merge
galprop_run : str
String identifying the galprop parameters
"""
flist = []
for sourcekey in sourcekeys:
for ring in rings:
flist += [
self.make_ring_filename(sourcekey, ring, galprop_run)
]
return flist
def make_ring_dict(self, galkey):
""" Make a dictionary mapping the merged component names to list of template files
Parameters
----------
galkey : str
Unique key for this ring dictionary
Returns `model_component.GalpropMergedRingInfo`
"""
galprop_rings = self.read_galprop_rings_yaml(galkey)
galprop_run = galprop_rings['galprop_run']
ring_limits = galprop_rings['ring_limits']
comp_dict = galprop_rings['diffuse_comp_dict']
ring_dict = {}
nring = len(ring_limits) - 1
for source_name, source_value in comp_dict.items():
base_dict = dict(source_name=source_name,
galkey=galkey,
galprop_run=galprop_run)
for iring in range(nring):
sourcekey = "%s_%i" % (source_name, iring)
full_key = "%s_%s" % (sourcekey, galkey)
rings = range(ring_limits[iring], ring_limits[iring + 1])
base_dict.update(
dict(ring=iring,
sourcekey=sourcekey,
files=self.make_ring_filelist(source_value, rings,
galprop_run),
merged_gasmap=self.make_merged_name(
sourcekey, galkey, False)))
ring_dict[full_key] = GalpropMergedRingInfo(**base_dict)
self._ring_dicts[galkey] = ring_dict
return ring_dict
def make_diffuse_comp_info(self, merged_name, galkey):
""" Make the information about a single merged component
Parameters
----------
merged_name : str
The name of the merged component
galkey : str
A short key identifying the galprop parameters
Returns `odel_component.ModelComponentInfo`
"""
kwargs = dict(source_name=merged_name,
source_ver=galkey,
model_type='MapCubeSource',
Spatial_Filename=self.make_merged_name(merged_name,
galkey,
fullpath=True),
srcmdl_name=self.make_xml_name(merged_name,
galkey,
fullpath=True))
return MapCubeComponentInfo(**kwargs)
def make_diffuse_comp_info_dict(self, galkey):
""" Make a dictionary maping from merged component to information about that component
Parameters
----------
galkey : str
A short key identifying the galprop parameters
"""
galprop_rings = self.read_galprop_rings_yaml(galkey)
ring_limits = galprop_rings.get('ring_limits')
comp_dict = galprop_rings.get('diffuse_comp_dict')
diffuse_comp_info_dict = {}
nring = len(ring_limits) - 1
for source_key in sorted(comp_dict.keys()):
for iring in range(nring):
source_name = "%s_%i" % (source_key, iring)
full_key = "%s_%s" % (source_name, galkey)
diffuse_comp_info_dict[full_key] =\
self.make_diffuse_comp_info(source_name, galkey)
self._diffuse_comp_info_dicts[galkey] = diffuse_comp_info_dict
return diffuse_comp_info_dict
from fermipy import fits_utils
from fermipy.utils import load_yaml
from fermipy.jobs.file_archive import FileFlags
from fermipy.jobs.scatter_gather import ScatterGather
from fermipy.jobs.slac_impl import make_nfs_path
from fermipy.jobs.link import Link
from fermipy.jobs.chain import Chain
from fermipy.diffuse.binning import Component
from fermipy.diffuse.name_policy import NameFactory
from fermipy.diffuse import defaults as diffuse_defaults
from fermipy.diffuse.gt_split_and_mktime import SplitAndMktimeChain
NAME_FACTORY = NameFactory()
NAME_FACTORY_CLEAN = NameFactory()
NAME_FACTORY_DIRTY = NameFactory()
class ResidualCR(Link):
"""Small class to analyze the residual cosmic-ray contaimination.
"""
appname = 'fermipy-residual-cr'
linkname_default = 'residual-cr'
usage = '%s [options]' % (appname)
description = "Compute the residual cosmic-ray contamination"
default_options = dict(ccube_dirty=diffuse_defaults.residual_cr['ccube_dirty'],
ccube_clean=diffuse_defaults.residual_cr['ccube_clean'],
bexpcube_dirty=diffuse_defaults.residual_cr['bexpcube_dirty'],
class CatalogSourceManager(object):
""" Small helper class to keep track of how we deal with catalog sources
This keeps track of two dictionaries
One of the dictionaries is keyed by catalog name, and contains information
about complete catalogs
catalog_comp_info_dicts[catalog_name] : `model_component.CatalogInfo`
The other dictionary is keyed by [{catalog_name}_{split_ver}][{split_key}]
Where:
{catalog_name} is something like '3FGL'
{split_ver} is somthing like 'v00' and specifes how to divide sources in the catalog
{split_key} refers to a specific sub-selection of sources
split_comp_info_dicts[splitkey] : `model_component.ModelComponentInfo`
"""
def __init__(self, **kwargs):
""" C'tor
Keyword arguments
-----------------
basedir : str
Top level directory for finding files
"""
self._name_factory = NameFactory(**kwargs)
self._catalog_comp_info_dicts = {}
self._split_comp_info_dicts = {}
def read_catalog_info_yaml(self, splitkey):
""" Read the yaml file for a particular split key
"""
catalog_info_yaml = self._name_factory.catalog_split_yaml(
sourcekey=splitkey, fullpath=True)
yaml_dict = yaml.safe_load(open(catalog_info_yaml))
# resolve env vars
yaml_dict['catalog_file'] = os.path.expandvars(
yaml_dict['catalog_file'])
yaml_dict['catalog_extdir'] = os.path.expandvars(
yaml_dict['catalog_extdir'])
return yaml_dict
def build_catalog_info(self, catalog_info):
""" Build a CatalogInfo object """
cat = SourceFactory.build_catalog(**catalog_info)
catalog_info['catalog'] = cat
# catalog_info['catalog_table'] =
# Table.read(catalog_info['catalog_file'])
catalog_info['catalog_table'] = cat.table
catalog_info['roi_model'] =\
SourceFactory.make_fermipy_roi_model_from_catalogs([cat])
catalog_info['srcmdl_name'] =\
self._name_factory.srcmdl_xml(sourcekey=catalog_info['catalog_name'])
return CatalogInfo(**catalog_info)
def catalogs(self):
""" Return the list of full catalogs used """
return sorted(self._catalog_comp_info_dicts.keys())
def splitkeys(self):
""" Return the list of catalog split keys used """
return sorted(self._split_comp_info_dicts.keys())
def catalog_comp_info_dict(self, catkey):
""" Return the roi_model for an entire catalog """
return self._catalog_comp_info_dicts[catkey]
def split_comp_info_dict(self, catalog_name, split_ver):
""" Return the information about a particular scheme for how to handle catalog sources """
return self._split_comp_info_dicts["%s_%s" % (catalog_name, split_ver)]
def catalog_components(self, catalog_name, split_ver):
""" Return the set of merged components for a particular split key """
return sorted(
self._split_comp_info_dicts["%s_%s" %
(catalog_name, split_ver)].keys())
def split_comp_info(self, catalog_name, split_ver, split_key):
""" Return the info for a particular split key """
return self._split_comp_info_dicts["%s_%s" % (catalog_name,
split_ver)][split_key]
def make_catalog_comp_info(self, full_cat_info, split_key, rule_key,
rule_val, sources):
""" Make the information about a single merged component
Parameters
----------
full_cat_info : `_model_component.CatalogInfo`
Information about the full catalog
split_key : str
Key identifying the version of the spliting used
rule_key : str
Key identifying the specific rule for this component
rule_val : list
List of the cuts used to define this component
sources : list
List of the names of the sources in this component
Returns `CompositeSourceInfo` or `CatalogSourcesInfo`
"""
merge = rule_val.get('merge', True)
sourcekey = "%s_%s_%s" % (full_cat_info.catalog_name, split_key,
rule_key)
srcmdl_name = self._name_factory.srcmdl_xml(sourcekey=sourcekey)
srcmdl_name = self._name_factory.fullpath(localpath=srcmdl_name)
kwargs = dict(source_name="%s_%s" %
(full_cat_info.catalog_name, rule_key),
source_ver=split_key,
sourcekey=sourcekey,
srcmdl_name=srcmdl_name,
source_names=sources,
catalog_info=full_cat_info,
roi_model=SourceFactory.copy_selected_sources(
full_cat_info.roi_model, sources))
if merge:
return CompositeSourceInfo(**kwargs)
return CatalogSourcesInfo(**kwargs)
def make_catalog_comp_info_dict(self, catalog_sources):
""" Make the information about the catalog components
Parameters
----------
catalog_sources : dict
Dictionary with catalog source defintions
Returns
-------
catalog_ret_dict : dict
Dictionary mapping catalog_name to `model_component.CatalogInfo`
split_ret_dict : dict
Dictionary mapping sourcekey to `model_component.ModelComponentInfo`
"""
catalog_ret_dict = {}
split_ret_dict = {}
for key, value in catalog_sources.items():
if value is None:
continue
if value['model_type'] != 'catalog':
continue
versions = value['versions']
for version in versions:
ver_key = "%s_%s" % (key, version)
source_dict = self.read_catalog_info_yaml(ver_key)
try:
full_cat_info = catalog_ret_dict[key]
except KeyError:
full_cat_info = self.build_catalog_info(source_dict)
catalog_ret_dict[key] = full_cat_info
try:
all_sources = [
x.strip() for x in full_cat_info.
catalog_table['Source_Name'].astype(str).tolist()
]
except KeyError:
print(full_cat_info.catalog_table.colnames)
used_sources = []
rules_dict = source_dict['rules_dict']
if rules_dict is None:
rules_dict = {}
split_dict = {}
for rule_key, rule_val in rules_dict.items():
# full_key =\
# self._name_factory.merged_sourcekey(catalog=ver_key,
# rulekey=rule_key)
sources = select_sources(full_cat_info.catalog_table,
rule_val['cuts'])
used_sources.extend(sources)
split_dict[rule_key] = self.make_catalog_comp_info(
full_cat_info, version, rule_key, rule_val, sources)
# Now deal with the remainder
for source in used_sources:
try:
all_sources.remove(source)
except ValueError:
continue
rule_val = dict(cuts=[],
merge=source_dict['remainder'].get(
'merge', False))
split_dict['remain'] = self.make_catalog_comp_info(
full_cat_info, version, 'remain', rule_val, all_sources)
# Merge in the info for this version of splits
split_ret_dict[ver_key] = split_dict
self._catalog_comp_info_dicts.update(catalog_ret_dict)
self._split_comp_info_dicts.update(split_ret_dict)
return (catalog_ret_dict, split_ret_dict)
class CatalogSourceManager(object):
""" Small helper class to keep track of how we deal with catalog sources
This keeps track of two dictionaries
One of the dictionaries is keyed by catalog name, and contains information
about complete catalogs
catalog_comp_info_dicts[catalog_name] : `model_component.CatalogInfo`
The other dictionary is keyed by [{catalog_name}_{split_ver}][{split_key}]
Where:
{catalog_name} is something like '3FGL'
{split_ver} is somthing like 'v00' and specifes how to divide sources in the catalog
{split_key} refers to a specific sub-selection of sources
split_comp_info_dicts[splitkey] : `model_component.ModelComponentInfo`
"""
def __init__(self, **kwargs):
""" C'tor
Keyword arguments
-----------------
basedir : str
Top level directory for finding files
"""
self._name_factory = NameFactory(**kwargs)
self._catalog_comp_info_dicts = {}
self._split_comp_info_dicts = {}
def read_catalog_info_yaml(self, splitkey):
""" Read the yaml file for a particular split key
"""
catalog_info_yaml = self._name_factory.catalog_split_yaml(sourcekey=splitkey,
fullpath=True)
yaml_dict = yaml.safe_load(open(catalog_info_yaml))
# resolve env vars
yaml_dict['catalog_file'] = os.path.expandvars(yaml_dict['catalog_file'])
yaml_dict['catalog_extdir'] = os.path.expandvars(yaml_dict['catalog_extdir'])
return yaml_dict
def build_catalog_info(self, catalog_info):
""" Build a CatalogInfo object """
cat = SourceFactory.build_catalog(**catalog_info)
catalog_info['catalog'] = cat
#catalog_info['catalog_table'] =
# Table.read(catalog_info['catalog_file'])
catalog_info['catalog_table'] = cat.table
catalog_info['roi_model'] =\
SourceFactory.make_fermipy_roi_model_from_catalogs([cat])
catalog_info['srcmdl_name'] =\
self._name_factory.srcmdl_xml(sourcekey=catalog_info['catalog_name'])
return CatalogInfo(**catalog_info)
def catalogs(self):
""" Return the list of full catalogs used """
return sorted(self._catalog_comp_info_dicts.keys())
def splitkeys(self):
""" Return the list of catalog split keys used """
return sorted(self._split_comp_info_dicts.keys())
def catalog_comp_info_dict(self, catkey):
""" Return the roi_model for an entire catalog """
return self._catalog_comp_info_dicts[catkey]
def split_comp_info_dict(self, catalog_name, split_ver):
""" Return the information about a particular scheme for how to handle catalog sources """
return self._split_comp_info_dicts["%s_%s" % (catalog_name, split_ver)]
def catalog_components(self, catalog_name, split_ver):
""" Return the set of merged components for a particular split key """
return sorted(self._split_comp_info_dicts["%s_%s" % (catalog_name, split_ver)].keys())
def split_comp_info(self, catalog_name, split_ver, split_key):
""" Return the info for a particular split key """
return self._split_comp_info_dicts["%s_%s" % (catalog_name, split_ver)][split_key]
def make_catalog_comp_info(self, full_cat_info, split_key, rule_key, rule_val, sources):
""" Make the information about a single merged component
Parameters
----------
full_cat_info : `_model_component.CatalogInfo`
Information about the full catalog
split_key : str
Key identifying the version of the spliting used
rule_key : str
Key identifying the specific rule for this component
rule_val : list
List of the cuts used to define this component
sources : list
List of the names of the sources in this component
Returns `CompositeSourceInfo` or `CatalogSourcesInfo`
"""
merge = rule_val.get('merge', True)
sourcekey = "%s_%s_%s" % (
full_cat_info.catalog_name, split_key, rule_key)
srcmdl_name = self._name_factory.srcmdl_xml(sourcekey=sourcekey)
srcmdl_name = self._name_factory.fullpath(localpath=srcmdl_name)
kwargs = dict(source_name="%s_%s" % (full_cat_info.catalog_name, rule_key),
source_ver=split_key,
sourcekey=sourcekey,
srcmdl_name=srcmdl_name,
source_names=sources,
catalog_info=full_cat_info,
roi_model=\
SourceFactory.copy_selected_sources(full_cat_info.roi_model, sources))
if merge:
return CompositeSourceInfo(**kwargs)
else:
return CatalogSourcesInfo(**kwargs)
def make_catalog_comp_info_dict(self, catalog_sources):
""" Make the information about the catalog components
Parameters
----------
catalog_sources : dict
Dictionary with catalog source defintions
Returns
-------
catalog_ret_dict : dict
Dictionary mapping catalog_name to `model_component.CatalogInfo`
split_ret_dict : dict
Dictionary mapping sourcekey to `model_component.ModelComponentInfo`
"""
catalog_ret_dict = {}
split_ret_dict = {}
for key, value in catalog_sources.items():
#model_type = value['model_type']
versions = value['versions']
for version in versions:
ver_key = "%s_%s" % (key, version)
source_dict = self.read_catalog_info_yaml(ver_key)
try:
full_cat_info = catalog_ret_dict[key]
except KeyError:
full_cat_info = self.build_catalog_info(source_dict)
catalog_ret_dict[key] = full_cat_info
try:
all_sources = [x.strip() for x in full_cat_info.catalog_table[
'Source_Name'].tolist()]
except KeyError:
print (full_cat_info.catalog_table.colnames)
used_sources = []
rules_dict = source_dict['rules_dict']
split_dict = {}
for rule_key, rule_val in rules_dict.items():
#full_key =\
# self._name_factory.merged_sourcekey(catalog=ver_key,
# rulekey=rule_key)
sources = select_sources(
full_cat_info.catalog_table, rule_val['cuts'])
used_sources.extend(sources)
split_dict[rule_key] = self.make_catalog_comp_info(
full_cat_info, version, rule_key, rule_val, sources)
# Now deal with the remainder
for source in used_sources:
try:
all_sources.remove(source)
except ValueError:
continue
rule_val = dict(cuts=[],
merge=source_dict['remainder'].get('merge', False))
split_dict['remain'] = self.make_catalog_comp_info(
full_cat_info, version, 'remain', rule_val, all_sources)
# Merge in the info for this version of splits
split_ret_dict[ver_key] = split_dict
self._catalog_comp_info_dicts.update(catalog_ret_dict)
self._split_comp_info_dicts.update(split_ret_dict)
return (catalog_ret_dict, split_ret_dict)
class DiffuseModelManager(object):
""" Small helper class to keep track of diffuse component templates
This keeps track of the 'diffuse component infomation' dictionary
This keyed by: key = {source_name}_{source_ver}
Where:
{source_name} is something like 'loopI'
{source_ver} is somthinng like v00
The dictioary is
diffuse_comp_info_dict[key] - > `model_component.ModelComponentInfo`
Note that some components ( those that represent moving sources or are selection depedent )
will have a sub-dictionary of diffuse_comp_info_dict object for each sub-component
The compoents are defined in a file called
config/diffuse_components.yaml
"""
def __init__(self, **kwargs):
""" C'tor
Keyword arguments
-----------------
name_policy : str
Name of yaml file contain file naming policy definitions
basedir : str
Top level directory for finding files
"""
self._name_factory = NameFactory(basedir=kwargs.get('basedir'))
self._diffuse_comp_info_dict = {}
@staticmethod
def read_diffuse_component_yaml(yamlfile):
""" Read the yaml file for the diffuse components
"""
diffuse_components = yaml.safe_load(open(yamlfile))
return diffuse_components
def sourcekeys(self):
"""Return the list of source keys"""
return sorted(self._diffuse_comp_info_dict.keys())
def diffuse_comp_info(self, sourcekey):
"""Return the Component info associated to a particular key
"""
return self._diffuse_comp_info_dict[sourcekey]
def make_template_name(self, model_type, sourcekey):
""" Make the name of a template file for particular component
Parameters
----------
model_type : str
Type of model to use for this component
sourcekey : str
Key to identify this component
Returns filename or None if component does not require a template file
"""
format_dict = self.__dict__.copy()
format_dict['sourcekey'] = sourcekey
if model_type == 'IsoSource':
return self._name_factory.spectral_template(**format_dict)
elif model_type == 'MapCubeSource':
return self._name_factory.diffuse_template(**format_dict)
else:
raise ValueError("Unexpected model_type %s" % model_type)
def make_xml_name(self, sourcekey):
""" Make the name of an xml file for a model definition of a single component
Parameters
----------
sourcekey : str
Key to identify this component
"""
format_dict = self.__dict__.copy()
format_dict['sourcekey'] = sourcekey
return self._name_factory.srcmdl_xml(**format_dict)
def make_diffuse_comp_info(self, source_name, source_ver, diffuse_dict,
components=None, comp_key=None):
""" Make a dictionary mapping the merged component names to list of template files
Parameters
----------
source_name : str
Name of the source
source_ver : str
Key identifying the version of the source
diffuse_dict : dict
Information about this component
comp_key : str
Used when we need to keep track of sub-components, i.e.,
for moving and selection dependent sources.
Returns `model_component.ModelComponentInfo` or
`model_component.IsoComponentInfo`
"""
model_type = diffuse_dict['model_type']
sourcekey = '%s_%s' % (source_name, source_ver)
if comp_key is None:
template_name = self.make_template_name(model_type, sourcekey)
srcmdl_name = self.make_xml_name(sourcekey)
else:
template_name = self.make_template_name(
model_type, "%s_%s" % (sourcekey, comp_key))
srcmdl_name = self.make_xml_name("%s_%s" % (sourcekey, comp_key))
template_name = self._name_factory.fullpath(localpath=template_name)
srcmdl_name = self._name_factory.fullpath(localpath=srcmdl_name)
kwargs = dict(source_name=source_name,
source_ver=source_ver,
model_type=model_type,
srcmdl_name=srcmdl_name,
components=components,
comp_key=comp_key)
kwargs.update(diffuse_dict)
if model_type == 'IsoSource':
kwargs['Spectral_Filename'] = template_name
return IsoComponentInfo(**kwargs)
elif model_type == 'MapCubeSource':
kwargs['Spatial_Filename'] = template_name
return MapCubeComponentInfo(**kwargs)
else:
raise ValueError("Unexpected model type %s" % model_type)
def make_diffuse_comp_info_dict(self, diffuse_sources, components):
""" Make a dictionary maping from diffuse component to information about that component
Parameters
----------
diffuse_sources : dict
Dictionary with diffuse source defintions
components : dict
Dictionary with event selection defintions,
needed for selection depenedent diffuse components
Returns
-------
ret_dict : dict
Dictionary mapping sourcekey to `model_component.ModelComponentInfo`
"""
ret_dict = {}
for key, value in diffuse_sources.items():
model_type = value.get('model_type', 'mapcube')
if model_type == 'galprop_rings':
continue
selection_dependent = value.get('selection_dependent', False)
moving = value.get('moving', False)
versions = value.get('versions', [])
for version in versions:
#sourcekey = self._name_factory.sourcekey(source_name=key,
# source_ver=version)
comp_dict = None
if selection_dependent:
# For selection dependent diffuse sources we need to split
# by binning component
comp_dict = {}
for comp in components:
comp_key = comp.make_key('{ebin_name}_{evtype_name}')
comp_dict[comp_key] = self.make_diffuse_comp_info(
key, version, value, None, comp_key)
elif moving:
# For moving diffuse sources we need to split by zmax cut
comp_dict = {}
zmax_dict = {}
for comp in components:
zmax_dict[int(comp.zmax)] = True
zmax_list = sorted(zmax_dict.keys())
for zmax in zmax_list:
comp_key = "zmax%i" % (zmax)
comp_dict[comp_key] = self.make_diffuse_comp_info(
key, version, value, None, comp_key)
comp_info = self.make_diffuse_comp_info(
key, version, value, comp_dict)
ret_dict[comp_info.sourcekey] = comp_info
self._diffuse_comp_info_dict.update(ret_dict)
return ret_dict
class GalpropMapManager(object):
""" Small helper class to keep track of Galprop gasmaps
This keeps track of two types of dictionaries.
Both are keyed by: key = {source_name}_{ring}_{galkey}
Where:
{source_name} is something like 'merged_C0'
{ring} is the ring index
{galkey} is a key specifying which version of galprop rings to use.
The two dictionaries are:
ring_dict[key] = `model_component.GalpropMergedRingInfo`
diffuse_comp_info_dict[key] ] `model_component.ModelComponentInfo`
The dictionaries are defined in files called.
models/galprop_rings_{galkey}.yaml
"""
def __init__(self, **kwargs):
""" C'tor
Keyword arguments
-----------------
maptype : str [newGasMaps_ST]
Used to define path to gasmap files
projtype : str [healpix]
Used to define path to gasmap files
basedir : str
Top level directory for finding files
"""
self.maptype = kwargs.get('maptype', 'newGasMaps_ST')
self.projtype = kwargs.get('projtype', 'healpix')
self._name_factory = NameFactory(basedir=kwargs.get('basedir'))
self._ring_dicts = {}
self._diffuse_comp_info_dicts = {}
def read_galprop_rings_yaml(self, galkey):
""" Read the yaml file for a partiuclar galprop key
"""
galprop_rings_yaml = self._name_factory.galprop_rings_yaml(galkey=galkey,
fullpath=True)
galprop_rings = yaml.safe_load(open(galprop_rings_yaml))
return galprop_rings
def galkeys(self):
""" Return the list of galprop keys used """
return sorted(self._ring_dicts.keys())
def ring_dict(self, galkey):
""" Return the ring dictionary for a particular galprop key
"""
return self._ring_dicts[galkey]
def diffuse_comp_info_dicts(self, galkey):
""" Return the components info dictionary for a particular galprop key
"""
return self._diffuse_comp_info_dicts[galkey]
def merged_components(self, galkey):
""" Return the set of merged components for a particular galprop key
"""
return sorted(self._diffuse_comp_info_dicts[galkey].keys())
def make_ring_filename(self, source_name, ring, galprop_run):
""" Make the name of a gasmap file for a single ring
Parameters
----------
source_name : str
The galprop component, used to define path to gasmap files
ring : int
The ring index
galprop_run : str
String identifying the galprop parameters
"""
format_dict = self.__dict__.copy()
format_dict['sourcekey'] = self._name_factory.galprop_ringkey(source_name=source_name,
ringkey="ring_%i" % ring)
format_dict['galprop_run'] = galprop_run
return self._name_factory.galprop_gasmap(**format_dict)
def make_merged_name(self, source_name, galkey, fullpath):
""" Make the name of a gasmap file for a set of merged rings
Parameters
----------
source_name : str
The galprop component, used to define path to gasmap files
galkey : str
A short key identifying the galprop parameters
fullpath : bool
Return the full path name
"""
format_dict = self.__dict__.copy()
format_dict['sourcekey'] = self._name_factory.galprop_sourcekey(source_name=source_name,
galpropkey=galkey)
format_dict['fullpath'] = fullpath
return self._name_factory.merged_gasmap(**format_dict)
def make_xml_name(self, source_name, galkey, fullpath):
""" Make the name of an xml file for a model definition for a set of merged rings
Parameters
----------
source_name : str
The galprop component, used to define path to gasmap files
galkey : str
A short key identifying the galprop parameters
fullpath : bool
Return the full path name
"""
format_dict = self.__dict__.copy()
format_dict['sourcekey'] = self._name_factory.galprop_sourcekey(source_name=source_name,
galpropkey=galkey)
format_dict['fullpath'] = fullpath
return self._name_factory.srcmdl_xml(**format_dict)
def make_ring_filelist(self, sourcekeys, rings, galprop_run):
""" Make a list of all the template files for a merged component
Parameters
----------
sourcekeys : list-like of str
The names of the componenents to merge
rings : list-like of int
The indices of the rings to merge
galprop_run : str
String identifying the galprop parameters
"""
flist = []
for sourcekey in sourcekeys:
for ring in rings:
flist += [self.make_ring_filename(sourcekey,
ring, galprop_run)]
return flist
def make_ring_dict(self, galkey):
""" Make a dictionary mapping the merged component names to list of template files
Parameters
----------
galkey : str
Unique key for this ring dictionary
Returns `model_component.GalpropMergedRingInfo`
"""
galprop_rings = self.read_galprop_rings_yaml(galkey)
galprop_run = galprop_rings['galprop_run']
ring_limits = galprop_rings['ring_limits']
comp_dict = galprop_rings['diffuse_comp_dict']
ring_dict = {}
nring = len(ring_limits) - 1
for source_name, source_value in comp_dict.items():
base_dict = dict(source_name=source_name,
galkey=galkey,
galprop_run=galprop_run)
for iring in range(nring):
sourcekey = "%s_%i" % (source_name, iring)
full_key = "%s_%s" % (sourcekey, galkey)
rings = range(ring_limits[iring], ring_limits[iring + 1])
base_dict.update(dict(ring=iring,
sourcekey=sourcekey,
files=self.make_ring_filelist(source_value,
rings, galprop_run),
merged_gasmap=self.make_merged_name(sourcekey,
galkey, False)))
ring_dict[full_key] = GalpropMergedRingInfo(**base_dict)
self._ring_dicts[galkey] = ring_dict
return ring_dict
def make_diffuse_comp_info(self, merged_name, galkey):
""" Make the information about a single merged component
Parameters
----------
merged_name : str
The name of the merged component
galkey : str
A short key identifying the galprop parameters
Returns `odel_component.ModelComponentInfo`
"""
kwargs = dict(source_name=merged_name,
source_ver=galkey,
model_type='MapCubeSource',
Spatial_Filename=self.make_merged_name(
merged_name, galkey, fullpath=True),
srcmdl_name=self.make_xml_name(merged_name, galkey, fullpath=True))
return MapCubeComponentInfo(**kwargs)
def make_diffuse_comp_info_dict(self, galkey):
""" Make a dictionary maping from merged component to information about that component
Parameters
----------
galkey : str
A short key identifying the galprop parameters
"""
galprop_rings = self.read_galprop_rings_yaml(galkey)
ring_limits = galprop_rings.get('ring_limits')
comp_dict = galprop_rings.get('diffuse_comp_dict')
diffuse_comp_info_dict = {}
nring = len(ring_limits) - 1
for source_key in sorted(comp_dict.keys()):
for iring in range(nring):
source_name = "%s_%i" % (source_key, iring)
full_key = "%s_%s" % (source_name, galkey)
diffuse_comp_info_dict[full_key] =\
self.make_diffuse_comp_info(source_name, galkey)
self._diffuse_comp_info_dicts[galkey] = diffuse_comp_info_dict
return diffuse_comp_info_dict
import math
from fermipy.jobs.file_archive import FileFlags
from fermipy.jobs.chain import Link
from fermipy.jobs.gtlink import Gtlink
from fermipy.jobs.scatter_gather import ConfigMaker, build_sg_from_link
from fermipy.jobs.lsf_impl import make_nfs_path, get_lsf_default_args, LSF_Interface
from fermipy.diffuse.utils import create_inputlist
from fermipy.diffuse.name_policy import NameFactory
from fermipy.diffuse.binning import Component
from fermipy.diffuse.diffuse_src_manager import make_ring_dicts,\
make_diffuse_comp_info_dict
from fermipy.diffuse.catalog_src_manager import make_catalog_comp_dict
from fermipy.diffuse import defaults as diffuse_defaults
NAME_FACTORY = NameFactory()
def make_input_file_list(binnedfile, num_files):
"""Make the list of input files for a particular energy bin X psf type """
outdir_base = os.path.dirname(binnedfile)
outbasename = os.path.basename(binnedfile)
filelist = ""
for i in range(num_files):
split_key = "%06i" % i
output_dir = os.path.join(outdir_base, split_key)
filepath = os.path.join(
output_dir, outbasename.replace('.fits',
'_%s.fits.gz' % split_key))
filelist += ' %s' % filepath
return filelist
class ModelManager(object):
""" Small helper class to create fitting models and manager XML files for fermipy
This class contains a 'library', which is a dictionary of all the source components:
specifically it maps:
sourcekey : `model_component.ModelComponentInfo`
"""
def __init__(self, **kwargs):
"""
Keyword arguments
-------------------
basedir : str
Top level directory for finding files
"""
self._name_factory = NameFactory(**kwargs)
self._dmm = kwargs.get('DiffuseModelManager',
DiffuseModelManager(**kwargs))
self._gmm = kwargs.get('GalpropMapManager',
GalpropMapManager(**kwargs))
self._csm = kwargs.get('CatalogSourceManager',
CatalogSourceManager(**kwargs))
self._library = {}
self._models = {}
self._spec_lib = SpectralLibrary({})
def read_model_yaml(self, modelkey):
""" Read the yaml file for the diffuse components
"""
model_yaml = self._name_factory.model_yaml(modelkey=modelkey,
fullpath=True)
model = yaml.safe_load(open(model_yaml))
return model
@property
def dmm(self):
""" Return the DiffuseModelManager """
return self._dmm
@property
def gmm(self):
""" Return the GalpropMapManager """
return self._gmm
@property
def csm(self):
""" Return the CatalogSourceManager """
return self._csm
def make_library(self, diffuse_yaml, catalog_yaml, binning_yaml):
""" Build up the library of all the components
Parameters
----------
diffuse_yaml : str
Name of the yaml file with the library of diffuse component definitions
catalog_yaml : str
Name of the yaml file with the library of catalog split definitions
binning_yaml : str
Name of the yaml file with the binning definitions
"""
ret_dict = {}
#catalog_dict = yaml.safe_load(open(catalog_yaml))
components_dict = Component.build_from_yamlfile(binning_yaml)
diffuse_ret_dict = make_diffuse_comp_info_dict(GalpropMapManager=self._gmm,
DiffuseModelManager=self._dmm,
diffuse=diffuse_yaml,
components=components_dict)
catalog_ret_dict = make_catalog_comp_dict(sources=catalog_yaml,
CatalogSourceManager=self._csm)
ret_dict.update(diffuse_ret_dict['comp_info_dict'])
ret_dict.update(catalog_ret_dict['comp_info_dict'])
self._library.update(ret_dict)
return ret_dict
def make_model_info(self, modelkey):
""" Build a dictionary with the information for a particular model.
Parameters
----------
modelkey : str
Key used to identify this particular model
Return `ModelInfo`
"""
model = self.read_model_yaml(modelkey)
sources = model['sources']
components = OrderedDict()
spec_model_yaml = self._name_factory.fullpath(localpath=model['spectral_models'])
self._spec_lib.update(yaml.safe_load(open(spec_model_yaml)))
for source, source_info in sources.items():
model_type = source_info.get('model_type', None)
version = source_info['version']
spec_type = source_info['SpectrumType']
sourcekey = "%s_%s" % (source, version)
if model_type == 'galprop_rings':
comp_info_dict = self.gmm.diffuse_comp_info_dicts(version)
def_spec_type = spec_type['default']
for comp_key, comp_info in comp_info_dict.items():
model_comp = ModelComponent(info=comp_info,
spectrum=\
self._spec_lib[spec_type.get(comp_key,
def_spec_type)])
components[comp_key] = model_comp
elif model_type == 'Catalog':
comp_info_dict = self.csm.split_comp_info_dict(source, version)
def_spec_type = spec_type['default']
for comp_key, comp_info in comp_info_dict.items():
model_comp = ModelComponent(info=comp_info,
spectrum=\
self._spec_lib[spec_type.get(comp_key,
def_spec_type)])
components[comp_key] = model_comp
else:
comp_info = self.dmm.diffuse_comp_info(sourcekey)
model_comp = ModelComponent(info=comp_info,
spectrum=self._spec_lib[spec_type])
components[sourcekey] = model_comp
ret_val = ModelInfo(model_name=modelkey,
model_components=components)
self._models[modelkey] = ret_val
return ret_val
def make_srcmap_manifest(self, modelkey, components, data):
"""Build a yaml file that specfies how to make the srcmap files for a particular model
Parameters
----------
modelkey : str
Key used to identify this particular model
components : list
The binning components used in this analysis
data : str
Path to file containing dataset definition
"""
try:
model_info = self._models[modelkey]
except KeyError:
model_info = self.make_model_info(modelkey)
self._name_factory.update_base_dict(data)
outfile = os.path.join('analysis', 'model_%s' %
modelkey, 'srcmap_manifest_%s.yaml' % modelkey)
manifest = model_info.make_srcmap_manifest(
components, self._name_factory)
outdir = os.path.dirname(outfile)
try:
os.makedirs(outdir)
except OSError:
pass
utils.write_yaml(manifest, outfile)
def make_fermipy_config_yaml(self, modelkey, components, data, **kwargs):
"""Build a fermipy top-level yaml configuration file
Parameters
----------
modelkey : str
Key used to identify this particular model
components : list
The binning components used in this analysis
data : str
Path to file containing dataset definition
"""
model_dir = os.path.join('analysis', 'model_%s' % modelkey)
hpx_order = kwargs.get('hpx_order', 9)
self._name_factory.update_base_dict(data)
try:
model_info = self._models[modelkey]
except KeyError:
model_info = self.make_model_info(modelkey)
model_info.make_model_rois(components, self._name_factory)
#source_names = model_info.component_names
master_xml_mdl = os.path.basename(
self._name_factory.master_srcmdl_xml(modelkey=modelkey))
master_data = dict(scfile=self._name_factory.ft2file(fullpath=True),
cacheft1=False)
master_binning = dict(projtype='HPX',
coordsys=kwargs.get('coordsys', 'GAL'),
roiwidth=180.,
binsperdec=8,
hpx_ordering_scheme="RING",
hpx_order=hpx_order,
hpx_ebin=True)
master_fileio = dict(outdir=model_dir,
logfile=os.path.join(model_dir, 'fermipy.log'))
master_gtlike = dict(irfs=self._name_factory.irfs(**kwargs),
edisp_disable=['isodiff', 'diffuse', 'limb'])
master_selection = dict(glat=0., glon=0., radius=180.)
master_model = dict(catalogs=[master_xml_mdl])
master = dict(data=master_data,
binning=master_binning,
fileio=master_fileio,
selection=master_selection,
gtlike=master_gtlike,
model=master_model,
components=[])
fermipy_dict = master
#comp_rois = {}
for comp in components:
zcut = "zmax%i" % comp.zmax
compkey = "%s_%s" % (zcut, comp.make_key(
'{ebin_name}_{evtype_name}'))
name_keys = dict(zcut=zcut,
modelkey=modelkey,
component=compkey,
fullpath=True)
comp_data = dict(ltcube=self._name_factory.ltcube(**name_keys))
comp_selection = dict(logemin=comp.log_emin,
logemax=comp.log_emax,
zmax=comp.zmax,
evtype=comp.evtype)
comp_binning = dict(enumbins=comp.enumbins,
hpx_order=min(comp.hpx_order, hpx_order))
comp_gtlike = dict(srcmap=self._name_factory.merged_srcmaps(**name_keys),
bexpmap=self._name_factory.bexpcube(**name_keys))
#comp_roi_source_info = {}
comp_xml_mdl = os.path.basename(self._name_factory.comp_srcmdl_xml(modelkey=modelkey,
component=compkey))
comp_model = dict(catalogs=[master_xml_mdl, comp_xml_mdl])
sub_dict = dict(data=comp_data,
binning=comp_binning,
selection=comp_selection,
gtlike=comp_gtlike,
model=comp_model)
fermipy_dict['components'].append(sub_dict)
outfile = os.path.join(model_dir, 'config.yaml')
print ("Writing fermipy config file %s"%outfile)
utils.write_yaml(fermipy_dict, outfile)
return fermipy_dict
@staticmethod
def get_sub_comp_info(source_info, comp):
"""Build and return information about a sub-component for a particular selection
"""
sub_comps = source_info.get('components', None)
if sub_comps is None:
return source_info.copy()
moving = source_info.get('moving', False)
selection_dependent = source_info.get('selection_dependent', False)
if selection_dependent:
key = comp.make_key('{ebin_name}_{evtype_name}')
elif moving:
key = "zmax%i" % comp.zmax
ret_dict = source_info.copy()
ret_dict.update(sub_comps[key])
return ret_dict