Пример #1
0
class DiffuseModelManager(object):
    """ Small helper class to keep track of diffuse component templates

    This keeps track of the 'diffuse component infomation' dictionary

    This keyed by: key = {source_name}_{source_ver}
    Where:
    {source_name} is something like 'loopI'
    {source_ver} is somthinng like v00

    The dictioary is
    diffuse_comp_info_dict[key] - > `model_component.ModelComponentInfo`

    Note that some components ( those that represent moving sources or are selection depedent )
    will have a sub-dictionary of diffuse_comp_info_dict object for each sub-component

    The compoents are defined in a file called
    config/diffuse_components.yaml
    """
    def __init__(self, **kwargs):
        """ C'tor

        Keyword arguments
        -----------------

        name_policy : str
            Name of yaml file contain file naming policy definitions
        basedir : str
            Top level directory for finding files
        """
        self._name_factory = NameFactory(basedir=kwargs.get('basedir'))
        self._diffuse_comp_info_dict = {}

    @staticmethod
    def read_diffuse_component_yaml(yamlfile):
        """ Read the yaml file for the diffuse components
        """
        diffuse_components = yaml.safe_load(open(yamlfile))
        return diffuse_components

    def sourcekeys(self):
        """Return the list of source keys"""
        return sorted(self._diffuse_comp_info_dict.keys())

    def diffuse_comp_info(self, sourcekey):
        """Return the Component info associated to a particular key
        """
        return self._diffuse_comp_info_dict[sourcekey]

    def make_template_name(self, model_type, sourcekey):
        """ Make the name of a template file for particular component

        Parameters
        ----------

        model_type : str
            Type of model to use for this component
        sourcekey : str
            Key to identify this component

        Returns filename or None if component does not require a template file
        """
        format_dict = self.__dict__.copy()
        format_dict['sourcekey'] = sourcekey
        if model_type == 'IsoSource':
            return self._name_factory.spectral_template(**format_dict)
        elif model_type == 'MapCubeSource':
            return self._name_factory.diffuse_template(**format_dict)
        else:
            raise ValueError("Unexpected model_type %s" % model_type)

    def make_xml_name(self, sourcekey):
        """ Make the name of an xml file for a model definition of a single component

        Parameters
        ----------

        sourcekey : str
            Key to identify this component
        """
        format_dict = self.__dict__.copy()
        format_dict['sourcekey'] = sourcekey
        return self._name_factory.srcmdl_xml(**format_dict)

    def make_diffuse_comp_info(self,
                               source_name,
                               source_ver,
                               diffuse_dict,
                               components=None,
                               comp_key=None):
        """ Make a dictionary mapping the merged component names to list of template files

        Parameters
        ----------

        source_name : str
           Name of the source
        source_ver : str
           Key identifying the version of the source
        diffuse_dict : dict
           Information about this component
        comp_key : str
           Used when we need to keep track of sub-components, i.e.,
           for moving and selection dependent sources.

        Returns `model_component.ModelComponentInfo` or
        `model_component.IsoComponentInfo`
        """
        model_type = diffuse_dict['model_type']
        sourcekey = '%s_%s' % (source_name, source_ver)
        if comp_key is None:
            template_name = self.make_template_name(model_type, sourcekey)
            srcmdl_name = self.make_xml_name(sourcekey)
        else:
            template_name = self.make_template_name(
                model_type, "%s_%s" % (sourcekey, comp_key))
            srcmdl_name = self.make_xml_name("%s_%s" % (sourcekey, comp_key))

        template_name = self._name_factory.fullpath(localpath=template_name)
        srcmdl_name = self._name_factory.fullpath(localpath=srcmdl_name)

        kwargs = dict(source_name=source_name,
                      source_ver=source_ver,
                      model_type=model_type,
                      srcmdl_name=srcmdl_name,
                      components=components,
                      comp_key=comp_key)
        kwargs.update(diffuse_dict)
        if model_type == 'IsoSource':
            kwargs['Spectral_Filename'] = template_name
            return IsoComponentInfo(**kwargs)
        elif model_type == 'MapCubeSource':
            kwargs['Spatial_Filename'] = template_name
            return MapCubeComponentInfo(**kwargs)
        else:
            raise ValueError("Unexpected model type %s" % model_type)

    def make_diffuse_comp_info_dict(self, diffuse_sources, components):
        """ Make a dictionary maping from diffuse component to information about that component

        Parameters
        ----------

        diffuse_sources : dict
            Dictionary with diffuse source defintions
        components : dict
            Dictionary with event selection defintions,
            needed for selection depenedent diffuse components

        Returns
        -------

        ret_dict : dict
            Dictionary mapping sourcekey to `model_component.ModelComponentInfo`
        """
        ret_dict = {}
        for key, value in diffuse_sources.items():
            model_type = value.get('model_type', 'mapcube')
            if model_type == 'galprop_rings':
                continue
            selection_dependent = value.get('selection_dependent', False)
            moving = value.get('moving', False)
            versions = value.get('versions', [])
            for version in versions:
                #sourcekey = self._name_factory.sourcekey(source_name=key,
                #                                         source_ver=version)
                comp_dict = None
                if selection_dependent:
                    # For selection dependent diffuse sources we need to split
                    # by binning component
                    comp_dict = {}
                    for comp in components:
                        comp_key = comp.make_key('{ebin_name}_{evtype_name}')
                        comp_dict[comp_key] = self.make_diffuse_comp_info(
                            key, version, value, None, comp_key)
                elif moving:
                    # For moving diffuse sources we need to split by zmax cut
                    comp_dict = {}
                    zmax_dict = {}
                    for comp in components:
                        zmax_dict[int(comp.zmax)] = True
                    zmax_list = sorted(zmax_dict.keys())
                    for zmax in zmax_list:
                        comp_key = "zmax%i" % (zmax)
                        comp_dict[comp_key] = self.make_diffuse_comp_info(
                            key, version, value, None, comp_key)

                comp_info = self.make_diffuse_comp_info(
                    key, version, value, comp_dict)
                ret_dict[comp_info.sourcekey] = comp_info

        self._diffuse_comp_info_dict.update(ret_dict)
        return ret_dict
Пример #2
0
class CatalogSourceManager(object):
    """ Small helper class to keep track of how we deal with catalog sources

    This keeps track of two dictionaries

    One of the dictionaries is keyed by catalog name, and contains information
    about complete catalogs
    catalog_comp_info_dicts[catalog_name] : `model_component.CatalogInfo`

    The other dictionary is keyed by [{catalog_name}_{split_ver}][{split_key}]
    Where:
    {catalog_name} is something like '3FGL'
    {split_ver} is somthing like 'v00' and specifes how to divide sources in the catalog
    {split_key} refers to a specific sub-selection of sources

    split_comp_info_dicts[splitkey] : `model_component.ModelComponentInfo`
    """
    def __init__(self, **kwargs):
        """ C'tor

        Keyword arguments
        -----------------

        basedir : str
            Top level directory for finding files
        """
        self._name_factory = NameFactory(**kwargs)
        self._catalog_comp_info_dicts = {}
        self._split_comp_info_dicts = {}

    def read_catalog_info_yaml(self, splitkey):
        """ Read the yaml file for a particular split key
        """
        catalog_info_yaml = self._name_factory.catalog_split_yaml(
            sourcekey=splitkey, fullpath=True)
        yaml_dict = yaml.safe_load(open(catalog_info_yaml))
        # resolve env vars
        yaml_dict['catalog_file'] = os.path.expandvars(
            yaml_dict['catalog_file'])
        yaml_dict['catalog_extdir'] = os.path.expandvars(
            yaml_dict['catalog_extdir'])
        return yaml_dict

    def build_catalog_info(self, catalog_info):
        """ Build a CatalogInfo object """
        cat = SourceFactory.build_catalog(**catalog_info)
        catalog_info['catalog'] = cat
        # catalog_info['catalog_table'] =
        #    Table.read(catalog_info['catalog_file'])
        catalog_info['catalog_table'] = cat.table
        catalog_info['roi_model'] =\
            SourceFactory.make_fermipy_roi_model_from_catalogs([cat])
        catalog_info['srcmdl_name'] =\
            self._name_factory.srcmdl_xml(sourcekey=catalog_info['catalog_name'])
        return CatalogInfo(**catalog_info)

    def catalogs(self):
        """ Return the list of full catalogs used """
        return sorted(self._catalog_comp_info_dicts.keys())

    def splitkeys(self):
        """ Return the list of catalog split keys used """
        return sorted(self._split_comp_info_dicts.keys())

    def catalog_comp_info_dict(self, catkey):
        """ Return the roi_model for an entire catalog """
        return self._catalog_comp_info_dicts[catkey]

    def split_comp_info_dict(self, catalog_name, split_ver):
        """ Return the information about a particular scheme for how to handle catalog sources """
        return self._split_comp_info_dicts["%s_%s" % (catalog_name, split_ver)]

    def catalog_components(self, catalog_name, split_ver):
        """ Return the set of merged components for a particular split key """
        return sorted(
            self._split_comp_info_dicts["%s_%s" %
                                        (catalog_name, split_ver)].keys())

    def split_comp_info(self, catalog_name, split_ver, split_key):
        """ Return the info for a particular split key """
        return self._split_comp_info_dicts["%s_%s" % (catalog_name,
                                                      split_ver)][split_key]

    def make_catalog_comp_info(self, full_cat_info, split_key, rule_key,
                               rule_val, sources):
        """ Make the information about a single merged component

        Parameters
        ----------

        full_cat_info : `_model_component.CatalogInfo`
            Information about the full catalog
        split_key : str
            Key identifying the version of the spliting used
        rule_key : str
            Key identifying the specific rule for this component
        rule_val : list
            List of the cuts used to define this component
        sources : list
            List of the names of the sources in this component

        Returns `CompositeSourceInfo` or `CatalogSourcesInfo`
        """
        merge = rule_val.get('merge', True)
        sourcekey = "%s_%s_%s" % (full_cat_info.catalog_name, split_key,
                                  rule_key)
        srcmdl_name = self._name_factory.srcmdl_xml(sourcekey=sourcekey)
        srcmdl_name = self._name_factory.fullpath(localpath=srcmdl_name)
        kwargs = dict(source_name="%s_%s" %
                      (full_cat_info.catalog_name, rule_key),
                      source_ver=split_key,
                      sourcekey=sourcekey,
                      srcmdl_name=srcmdl_name,
                      source_names=sources,
                      catalog_info=full_cat_info,
                      roi_model=SourceFactory.copy_selected_sources(
                          full_cat_info.roi_model, sources))
        if merge:
            return CompositeSourceInfo(**kwargs)
        return CatalogSourcesInfo(**kwargs)

    def make_catalog_comp_info_dict(self, catalog_sources):
        """ Make the information about the catalog components

        Parameters
        ----------

        catalog_sources : dict
            Dictionary with catalog source defintions

        Returns
        -------

        catalog_ret_dict : dict
            Dictionary mapping catalog_name to `model_component.CatalogInfo`
        split_ret_dict : dict
            Dictionary mapping sourcekey to `model_component.ModelComponentInfo`
        """
        catalog_ret_dict = {}
        split_ret_dict = {}
        for key, value in catalog_sources.items():
            if value is None:
                continue
            if value['model_type'] != 'catalog':
                continue
            versions = value['versions']
            for version in versions:
                ver_key = "%s_%s" % (key, version)
                source_dict = self.read_catalog_info_yaml(ver_key)
                try:
                    full_cat_info = catalog_ret_dict[key]
                except KeyError:
                    full_cat_info = self.build_catalog_info(source_dict)
                    catalog_ret_dict[key] = full_cat_info

                try:
                    all_sources = [
                        x.strip() for x in full_cat_info.
                        catalog_table['Source_Name'].astype(str).tolist()
                    ]
                except KeyError:
                    print(full_cat_info.catalog_table.colnames)
                used_sources = []
                rules_dict = source_dict['rules_dict']
                if rules_dict is None:
                    rules_dict = {}
                split_dict = {}
                for rule_key, rule_val in rules_dict.items():
                    # full_key =\
                    #    self._name_factory.merged_sourcekey(catalog=ver_key,
                    #                                        rulekey=rule_key)
                    sources = select_sources(full_cat_info.catalog_table,
                                             rule_val['cuts'])
                    used_sources.extend(sources)
                    split_dict[rule_key] = self.make_catalog_comp_info(
                        full_cat_info, version, rule_key, rule_val, sources)

                # Now deal with the remainder
                for source in used_sources:
                    try:
                        all_sources.remove(source)
                    except ValueError:
                        continue
                rule_val = dict(cuts=[],
                                merge=source_dict['remainder'].get(
                                    'merge', False))
                split_dict['remain'] = self.make_catalog_comp_info(
                    full_cat_info, version, 'remain', rule_val, all_sources)

                # Merge in the info for this version of splits
                split_ret_dict[ver_key] = split_dict

        self._catalog_comp_info_dicts.update(catalog_ret_dict)
        self._split_comp_info_dicts.update(split_ret_dict)
        return (catalog_ret_dict, split_ret_dict)
Пример #3
0
class ModelManager(object):
    """ Small helper class to create fitting models and manager XML files for fermipy

    This class contains a 'library', which is a dictionary of all the source components:

    specifically it maps:

    sourcekey : `model_component.ModelComponentInfo`

    """
    def __init__(self, **kwargs):
        """
        Keyword arguments
        -------------------
        basedir : str
            Top level directory for finding files
        """
        self._name_factory = NameFactory(**kwargs)
        self._dmm = kwargs.get('DiffuseModelManager',
                               DiffuseModelManager(**kwargs))
        self._gmm = kwargs.get('GalpropMapManager',
                               GalpropMapManager(**kwargs))
        self._csm = kwargs.get('CatalogSourceManager',
                               CatalogSourceManager(**kwargs))
        self._library = {}
        self._models = {}
        self._spec_lib = SpectralLibrary({})

    def read_model_yaml(self, modelkey):
        """ Read the yaml file for the diffuse components
        """
        model_yaml = self._name_factory.model_yaml(modelkey=modelkey,
                                                   fullpath=True)
        model = yaml.safe_load(open(model_yaml))
        return model

    @property
    def dmm(self):
        """ Return the DiffuseModelManager """
        return self._dmm

    @property
    def gmm(self):
        """ Return the GalpropMapManager """
        return self._gmm

    @property
    def csm(self):
        """ Return the CatalogSourceManager """
        return self._csm

    def make_library(self, diffuse_yaml, catalog_yaml, binning_yaml):
        """ Build up the library of all the components

        Parameters
        ----------

        diffuse_yaml : str
            Name of the yaml file with the library of diffuse component definitions
        catalog_yaml : str
            Name of the yaml file width the library of catalog split definitions
        binning_yaml : str
            Name of the yaml file with the binning definitions
        """
        ret_dict = {}
        #catalog_dict = yaml.safe_load(open(catalog_yaml))
        components_dict = Component.build_from_yamlfile(binning_yaml)
        diffuse_ret_dict = make_diffuse_comp_info_dict(
            GalpropMapManager=self._gmm,
            DiffuseModelManager=self._dmm,
            library=diffuse_yaml,
            components=components_dict)
        catalog_ret_dict = make_catalog_comp_dict(
            library=catalog_yaml, CatalogSourceManager=self._csm)
        ret_dict.update(diffuse_ret_dict['comp_info_dict'])
        ret_dict.update(catalog_ret_dict['comp_info_dict'])
        self._library.update(ret_dict)
        return ret_dict

    def make_model_info(self, modelkey):
        """ Build a dictionary with the information for a particular model.

        Parameters
        ----------

        modelkey : str
            Key used to identify this particular model

        Return `ModelInfo`
        """
        model = self.read_model_yaml(modelkey)
        sources = model['sources']
        components = OrderedDict()
        spec_model_yaml = self._name_factory.fullpath(
            localpath=model['spectral_models'])
        self._spec_lib.update(yaml.safe_load(open(spec_model_yaml)))
        for source, source_info in sources.items():
            model_type = source_info.get('model_type', None)
            par_overrides = source_info.get('par_overides', None)
            version = source_info['version']
            spec_type = source_info['SpectrumType']
            edisp_disable = source_info.get('edisp_disable', False)
            sourcekey = "%s_%s" % (source, version)
            if model_type == 'galprop_rings':
                comp_info_dict = self.gmm.diffuse_comp_info_dicts(version)
                def_spec_type = spec_type['default']
                for comp_key, comp_info in comp_info_dict.items():
                    model_comp = ModelComponent(info=comp_info,
                                                spectrum=\
                                                    self._spec_lib[spec_type.get(comp_key,
                                                                                 def_spec_type)],
                                                par_overrides=par_overrides,
                                                edisp_disable=edisp_disable)
                    components[comp_key] = model_comp
            elif model_type == 'Catalog':
                comp_info_dict = self.csm.split_comp_info_dict(source, version)
                def_spec_type = spec_type['default']
                for comp_key, comp_info in comp_info_dict.items():
                    model_comp = ModelComponent(info=comp_info,
                                                spectrum=\
                                                    self._spec_lib[spec_type.get(comp_key,
                                                                                 def_spec_type)],
                                                par_overrides=par_overrides,
                                                edisp_disable=edisp_disable)
                    components[comp_key] = model_comp
            else:
                comp_info = self.dmm.diffuse_comp_info(sourcekey)
                model_comp = ModelComponent(info=comp_info,
                                            spectrum=self._spec_lib[spec_type],
                                            par_overrides=par_overrides,
                                            edisp_disable=edisp_disable)
                components[sourcekey] = model_comp
        ret_val = ModelInfo(model_name=modelkey, model_components=components)
        self._models[modelkey] = ret_val
        return ret_val

    def make_srcmap_manifest(self, modelkey, components, data):
        """Build a yaml file that specfies how to make the srcmap files for a particular model

        Parameters
        ----------

        modelkey : str
            Key used to identify this particular model
        components : list
            The binning components used in this analysis
        data : str
            Path to file containing dataset definition
        """
        try:
            model_info = self._models[modelkey]
        except KeyError:
            model_info = self.make_model_info(modelkey)
        self._name_factory.update_base_dict(data)
        outfile = os.path.join('analysis', 'model_%s' % modelkey,
                               'srcmap_manifest_%s.yaml' % modelkey)
        manifest = model_info.make_srcmap_manifest(components,
                                                   self._name_factory)

        outdir = os.path.dirname(outfile)
        try:
            os.makedirs(outdir)
        except OSError:
            pass
        utils.write_yaml(manifest, outfile)

    def make_fermipy_config_yaml(self, modelkey, components, data, **kwargs):
        """Build a fermipy top-level yaml configuration file

        Parameters
        ----------

        modelkey : str
            Key used to identify this particular model
        components : list
            The binning components used in this analysis
        data : str
            Path to file containing dataset definition
        """
        model_dir = os.path.join('analysis', 'model_%s' % modelkey)
        hpx_order = kwargs.get('hpx_order', 9)
        self._name_factory.update_base_dict(data)
        try:
            model_info = self._models[modelkey]
        except KeyError:
            model_info = self.make_model_info(modelkey)

        model_rois = model_info.make_model_rois(components, self._name_factory)

        #source_names = model_info.component_names

        master_xml_mdl = os.path.basename(
            self._name_factory.master_srcmdl_xml(modelkey=modelkey))

        master_data = dict(scfile=self._name_factory.ft2file(fullpath=True),
                           cacheft1=False)
        master_binning = dict(projtype='HPX',
                              roiwidth=360.,
                              binsperdec=4,
                              hpx_ordering_scheme="RING",
                              hpx_order=hpx_order,
                              hpx_ebin=True)
        # master_fileio = dict(outdir=model_dir,
        #                     logfile=os.path.join(model_dir, 'fermipy.log'))
        master_fileio = dict(logfile='fermipy.log')
        master_gtlike = dict(irfs=self._name_factory.irfs(**kwargs),
                             edisp_disable=model_info.edisp_disable_list(),
                             use_external_srcmap=True)
        master_selection = dict(glat=0., glon=0., radius=180.)
        master_model = dict(catalogs=[master_xml_mdl])
        master_plotting = dict(label_ts_threshold=1e9)

        master = dict(data=master_data,
                      binning=master_binning,
                      fileio=master_fileio,
                      selection=master_selection,
                      gtlike=master_gtlike,
                      model=master_model,
                      plotting=master_plotting,
                      components=[])

        fermipy_dict = master

        #comp_rois = {}

        for comp in components:
            zcut = "zmax%i" % comp.zmax
            compkey = "%s_%s" % (zcut,
                                 comp.make_key('{ebin_name}_{evtype_name}'))
            comp_roi = model_rois[compkey]
            name_keys = dict(zcut=zcut,
                             modelkey=modelkey,
                             component=compkey,
                             mktime='none',
                             coordsys=comp.coordsys,
                             fullpath=True)
            comp_data = dict(ltcube=self._name_factory.ltcube(**name_keys))
            comp_selection = dict(logemin=comp.log_emin,
                                  logemax=comp.log_emax,
                                  zmax=comp.zmax,
                                  evtype=comp.evtype)
            comp_binning = dict(enumbins=comp.enumbins,
                                hpx_order=min(comp.hpx_order, hpx_order),
                                coordsys=comp.coordsys)
            comp_gtlike = dict(
                srcmap=self._name_factory.merged_srcmaps(**name_keys),
                bexpmap=self._name_factory.bexpcube(**name_keys),
                use_external_srcmap=True)
            #comp_roi_source_info = {}
            diffuse_srcs = []
            for src in comp_roi.diffuse_sources:
                if isinstance(src, MapCubeSource):
                    diffuse_srcs.append(dict(name=src.name, file=src.mapcube))
                elif isinstance(src, IsoSource):
                    diffuse_srcs.append(
                        dict(name=src.name, file=src.fileFunction))
                else:
                    pass

            comp_model = dict(diffuse=diffuse_srcs)
            sub_dict = dict(data=comp_data,
                            binning=comp_binning,
                            selection=comp_selection,
                            gtlike=comp_gtlike,
                            model=comp_model)
            fermipy_dict['components'].append(sub_dict)

        # Add placeholder diffuse sources
        fermipy_dict['model']['diffuse'] = diffuse_srcs

        outfile = os.path.join(model_dir, 'config.yaml')
        print("Writing fermipy config file %s" % outfile)
        utils.write_yaml(fermipy_dict, outfile)
        return fermipy_dict

    @staticmethod
    def get_sub_comp_info(source_info, comp):
        """Build and return information about a sub-component for a particular selection
        """
        sub_comps = source_info.get('components', None)
        if sub_comps is None:
            return source_info.copy()
        moving = source_info.get('moving', False)
        selection_dependent = source_info.get('selection_dependent', False)
        if selection_dependent:
            key = comp.make_key('{ebin_name}_{evtype_name}')
        elif moving:
            key = "zmax%i" % comp.zmax
        ret_dict = source_info.copy()
        ret_dict.update(sub_comps[key])
        return ret_dict
Пример #4
0
class DiffuseModelManager(object):
    """ Small helper class to keep track of diffuse component templates

    This keeps track of the 'diffuse component infomation' dictionary

    This keyed by: key = {source_name}_{source_ver}
    Where:
    {source_name} is something like 'loopI'
    {source_ver} is somthinng like v00

    The dictioary is
    diffuse_comp_info_dict[key] - > `model_component.ModelComponentInfo`

    Note that some components ( those that represent moving sources or are selection depedent )
    will have a sub-dictionary of diffuse_comp_info_dict object for each sub-component

    The compoents are defined in a file called
    config/diffuse_components.yaml
    """

    def __init__(self, **kwargs):
        """ C'tor

        Keyword arguments
        -----------------

        name_policy : str
            Name of yaml file contain file naming policy definitions
        basedir : str
            Top level directory for finding files
        """
        self._name_factory = NameFactory(basedir=kwargs.get('basedir'))
        self._diffuse_comp_info_dict = {}

    @staticmethod
    def read_diffuse_component_yaml(yamlfile):
        """ Read the yaml file for the diffuse components
        """
        diffuse_components = yaml.safe_load(open(yamlfile))
        return diffuse_components

    def sourcekeys(self):
        """Return the list of source keys"""
        return sorted(self._diffuse_comp_info_dict.keys())

    def diffuse_comp_info(self, sourcekey):
        """Return the Component info associated to a particular key
        """
        return self._diffuse_comp_info_dict[sourcekey]

    def make_template_name(self, model_type, sourcekey):
        """ Make the name of a template file for particular component

        Parameters
        ----------

        model_type : str
            Type of model to use for this component
        sourcekey : str
            Key to identify this component

        Returns filename or None if component does not require a template file
        """
        format_dict = self.__dict__.copy()
        format_dict['sourcekey'] = sourcekey
        if model_type == 'IsoSource':
            return self._name_factory.spectral_template(**format_dict)
        elif model_type == 'MapCubeSource':
            return self._name_factory.diffuse_template(**format_dict)
        else:
            raise ValueError("Unexpected model_type %s" % model_type)

    def make_xml_name(self, sourcekey):
        """ Make the name of an xml file for a model definition of a single component

        Parameters
        ----------

        sourcekey : str
            Key to identify this component
        """
        format_dict = self.__dict__.copy()
        format_dict['sourcekey'] = sourcekey
        return self._name_factory.srcmdl_xml(**format_dict)

    def make_diffuse_comp_info(self, source_name, source_ver, diffuse_dict,
                               components=None, comp_key=None):
        """ Make a dictionary mapping the merged component names to list of template files

        Parameters
        ----------

        source_name : str
           Name of the source
        source_ver : str
           Key identifying the version of the source
        diffuse_dict : dict
           Information about this component
        comp_key : str
           Used when we need to keep track of sub-components, i.e.,
           for moving and selection dependent sources.

        Returns `model_component.ModelComponentInfo` or
        `model_component.IsoComponentInfo`
        """
        model_type = diffuse_dict['model_type']
        sourcekey = '%s_%s' % (source_name, source_ver)
        if comp_key is None:
            template_name = self.make_template_name(model_type, sourcekey)
            srcmdl_name = self.make_xml_name(sourcekey)
        else:
            template_name = self.make_template_name(
                model_type, "%s_%s" % (sourcekey, comp_key))
            srcmdl_name = self.make_xml_name("%s_%s" % (sourcekey, comp_key))

        template_name = self._name_factory.fullpath(localpath=template_name)
        srcmdl_name = self._name_factory.fullpath(localpath=srcmdl_name)

        kwargs = dict(source_name=source_name,
                      source_ver=source_ver,
                      model_type=model_type,
                      srcmdl_name=srcmdl_name,
                      components=components,
                      comp_key=comp_key)
        kwargs.update(diffuse_dict)
        if model_type == 'IsoSource':
            kwargs['Spectral_Filename'] = template_name
            return IsoComponentInfo(**kwargs)
        elif model_type == 'MapCubeSource':
            kwargs['Spatial_Filename'] = template_name
            return MapCubeComponentInfo(**kwargs)
        else:
            raise ValueError("Unexpected model type %s" % model_type)

    def make_diffuse_comp_info_dict(self, diffuse_sources, components):
        """ Make a dictionary maping from diffuse component to information about that component

        Parameters
        ----------

        diffuse_sources : dict
            Dictionary with diffuse source defintions
        components : dict
            Dictionary with event selection defintions,
            needed for selection depenedent diffuse components

        Returns
        -------

        ret_dict : dict
            Dictionary mapping sourcekey to `model_component.ModelComponentInfo`
        """
        ret_dict = {}
        for key, value in diffuse_sources.items():
            model_type = value.get('model_type', 'mapcube')
            if model_type == 'galprop_rings':
                continue
            selection_dependent = value.get('selection_dependent', False)
            moving = value.get('moving', False)
            versions = value.get('versions', [])
            for version in versions:
                #sourcekey = self._name_factory.sourcekey(source_name=key,
                #                                         source_ver=version)
                comp_dict = None
                if selection_dependent:
                    # For selection dependent diffuse sources we need to split
                    # by binning component
                    comp_dict = {}
                    for comp in components:
                        comp_key = comp.make_key('{ebin_name}_{evtype_name}')
                        comp_dict[comp_key] = self.make_diffuse_comp_info(
                            key, version, value, None, comp_key)
                elif moving:
                    # For moving diffuse sources we need to split by zmax cut
                    comp_dict = {}
                    zmax_dict = {}
                    for comp in components:
                        zmax_dict[int(comp.zmax)] = True
                    zmax_list = sorted(zmax_dict.keys())
                    for zmax in zmax_list:
                        comp_key = "zmax%i" % (zmax)
                        comp_dict[comp_key] = self.make_diffuse_comp_info(
                            key, version, value, None, comp_key)

                comp_info = self.make_diffuse_comp_info(
                    key, version, value, comp_dict)
                ret_dict[comp_info.sourcekey] = comp_info

        self._diffuse_comp_info_dict.update(ret_dict)
        return ret_dict
Пример #5
0
class CatalogSourceManager(object):
    """ Small helper class to keep track of how we deal with catalog sources

    This keeps track of two dictionaries

    One of the dictionaries is keyed by catalog name, and contains information
    about complete catalogs
    catalog_comp_info_dicts[catalog_name] : `model_component.CatalogInfo`

    The other dictionary is keyed by [{catalog_name}_{split_ver}][{split_key}]
    Where:
    {catalog_name} is something like '3FGL'
    {split_ver} is somthing like 'v00' and specifes how to divide sources in the catalog
    {split_key} refers to a specific sub-selection of sources

    split_comp_info_dicts[splitkey] : `model_component.ModelComponentInfo`
    """

    def __init__(self, **kwargs):
        """ C'tor

        Keyword arguments
        -----------------

        basedir : str
            Top level directory for finding files
        """
        self._name_factory = NameFactory(**kwargs)
        self._catalog_comp_info_dicts = {}
        self._split_comp_info_dicts = {}

    def read_catalog_info_yaml(self, splitkey):
        """ Read the yaml file for a particular split key
        """
        catalog_info_yaml = self._name_factory.catalog_split_yaml(sourcekey=splitkey,
                                                                  fullpath=True)
        yaml_dict = yaml.safe_load(open(catalog_info_yaml))
        # resolve env vars
        yaml_dict['catalog_file'] = os.path.expandvars(yaml_dict['catalog_file'])
        yaml_dict['catalog_extdir'] = os.path.expandvars(yaml_dict['catalog_extdir'])
        return yaml_dict

    def build_catalog_info(self, catalog_info):
        """ Build a CatalogInfo object """        
        cat = SourceFactory.build_catalog(**catalog_info)
        catalog_info['catalog'] = cat
        #catalog_info['catalog_table'] = 
        #    Table.read(catalog_info['catalog_file'])
        catalog_info['catalog_table'] = cat.table
        catalog_info['roi_model'] =\
            SourceFactory.make_fermipy_roi_model_from_catalogs([cat])
        catalog_info['srcmdl_name'] =\
            self._name_factory.srcmdl_xml(sourcekey=catalog_info['catalog_name'])
        return CatalogInfo(**catalog_info)

    def catalogs(self):
        """ Return the list of full catalogs used """
        return sorted(self._catalog_comp_info_dicts.keys())

    def splitkeys(self):
        """ Return the list of catalog split keys used """
        return sorted(self._split_comp_info_dicts.keys())

    def catalog_comp_info_dict(self, catkey):
        """ Return the roi_model for an entire catalog """
        return self._catalog_comp_info_dicts[catkey]

    def split_comp_info_dict(self, catalog_name, split_ver):
        """ Return the information about a particular scheme for how to handle catalog sources """
        return self._split_comp_info_dicts["%s_%s" % (catalog_name, split_ver)]

    def catalog_components(self, catalog_name, split_ver):
        """ Return the set of merged components for a particular split key """
        return sorted(self._split_comp_info_dicts["%s_%s" % (catalog_name, split_ver)].keys())

    def split_comp_info(self, catalog_name, split_ver, split_key):
        """ Return the info for a particular split key """
        return self._split_comp_info_dicts["%s_%s" % (catalog_name, split_ver)][split_key]

    def make_catalog_comp_info(self, full_cat_info, split_key, rule_key, rule_val, sources):
        """ Make the information about a single merged component

        Parameters
        ----------

        full_cat_info : `_model_component.CatalogInfo`
            Information about the full catalog
        split_key : str
            Key identifying the version of the spliting used
        rule_key : str
            Key identifying the specific rule for this component
        rule_val : list
            List of the cuts used to define this component
        sources : list
            List of the names of the sources in this component

        Returns `CompositeSourceInfo` or `CatalogSourcesInfo`
        """
        merge = rule_val.get('merge', True)
        sourcekey = "%s_%s_%s" % (
            full_cat_info.catalog_name, split_key, rule_key)
        srcmdl_name = self._name_factory.srcmdl_xml(sourcekey=sourcekey)
        srcmdl_name = self._name_factory.fullpath(localpath=srcmdl_name)
        kwargs = dict(source_name="%s_%s" % (full_cat_info.catalog_name, rule_key),
                      source_ver=split_key,
                      sourcekey=sourcekey,
                      srcmdl_name=srcmdl_name,
                      source_names=sources,
                      catalog_info=full_cat_info,
                      roi_model=\
                          SourceFactory.copy_selected_sources(full_cat_info.roi_model, sources))
        if merge:
            return CompositeSourceInfo(**kwargs)
        else:
            return CatalogSourcesInfo(**kwargs)

    def make_catalog_comp_info_dict(self, catalog_sources):
        """ Make the information about the catalog components

        Parameters
        ----------

        catalog_sources : dict
            Dictionary with catalog source defintions

        Returns
        -------

        catalog_ret_dict : dict
            Dictionary mapping catalog_name to `model_component.CatalogInfo`
        split_ret_dict : dict
            Dictionary mapping sourcekey to `model_component.ModelComponentInfo`
        """
        catalog_ret_dict = {}
        split_ret_dict = {}
        for key, value in catalog_sources.items():
            #model_type = value['model_type']
            versions = value['versions']
            for version in versions:
                ver_key = "%s_%s" % (key, version)
                source_dict = self.read_catalog_info_yaml(ver_key)
                try:
                    full_cat_info = catalog_ret_dict[key]
                except KeyError:
                    full_cat_info = self.build_catalog_info(source_dict)
                    catalog_ret_dict[key] = full_cat_info

                try:
                    all_sources = [x.strip() for x in full_cat_info.catalog_table[
                            'Source_Name'].tolist()]
                except KeyError:
                    print (full_cat_info.catalog_table.colnames)
                used_sources = []
                rules_dict = source_dict['rules_dict']
                split_dict = {}
                for rule_key, rule_val in rules_dict.items():
                    #full_key =\
                    #    self._name_factory.merged_sourcekey(catalog=ver_key,
                    #                                        rulekey=rule_key)
                    sources = select_sources(
                        full_cat_info.catalog_table, rule_val['cuts'])
                    used_sources.extend(sources)
                    split_dict[rule_key] = self.make_catalog_comp_info(
                        full_cat_info, version, rule_key, rule_val, sources)

                # Now deal with the remainder
                for source in used_sources:
                    try:
                        all_sources.remove(source)
                    except ValueError:
                        continue
                rule_val = dict(cuts=[],
                                merge=source_dict['remainder'].get('merge', False))
                split_dict['remain'] = self.make_catalog_comp_info(
                    full_cat_info, version, 'remain', rule_val, all_sources)

                # Merge in the info for this version of splits
                split_ret_dict[ver_key] = split_dict

        self._catalog_comp_info_dicts.update(catalog_ret_dict)
        self._split_comp_info_dicts.update(split_ret_dict)
        return (catalog_ret_dict, split_ret_dict)
Пример #6
0
class ModelManager(object):
    """ Small helper class to create fitting models and manager XML files for fermipy

    This class contains a 'library', which is a dictionary of all the source components:

    specifically it maps:

    sourcekey : `model_component.ModelComponentInfo`

    """

    def __init__(self, **kwargs):
        """
        Keyword arguments
        -------------------
        basedir : str
            Top level directory for finding files
        """
        self._name_factory = NameFactory(**kwargs)
        self._dmm = kwargs.get('DiffuseModelManager',
                               DiffuseModelManager(**kwargs))
        self._gmm = kwargs.get('GalpropMapManager',
                               GalpropMapManager(**kwargs))
        self._csm = kwargs.get('CatalogSourceManager',
                               CatalogSourceManager(**kwargs))
        self._library = {}
        self._models = {}
        self._spec_lib = SpectralLibrary({})

    def read_model_yaml(self, modelkey):
        """ Read the yaml file for the diffuse components
        """
        model_yaml = self._name_factory.model_yaml(modelkey=modelkey,
                                                   fullpath=True)
        model = yaml.safe_load(open(model_yaml))
        return model

    @property
    def dmm(self):
        """ Return the DiffuseModelManager """
        return self._dmm

    @property
    def gmm(self):
        """ Return the GalpropMapManager """
        return self._gmm

    @property
    def csm(self):
        """ Return the CatalogSourceManager """
        return self._csm

    def make_library(self, diffuse_yaml, catalog_yaml, binning_yaml):
        """ Build up the library of all the components

        Parameters
        ----------

        diffuse_yaml : str
            Name of the yaml file with the library of diffuse component definitions
        catalog_yaml : str
            Name of the yaml file with the library of catalog split definitions
        binning_yaml : str
            Name of the yaml file with the binning definitions
        """
        ret_dict = {}
        #catalog_dict = yaml.safe_load(open(catalog_yaml))
        components_dict = Component.build_from_yamlfile(binning_yaml)
        diffuse_ret_dict = make_diffuse_comp_info_dict(GalpropMapManager=self._gmm,
                                                       DiffuseModelManager=self._dmm,
                                                       diffuse=diffuse_yaml,
                                                       components=components_dict)
        catalog_ret_dict = make_catalog_comp_dict(sources=catalog_yaml,
                                                  CatalogSourceManager=self._csm)
        ret_dict.update(diffuse_ret_dict['comp_info_dict'])
        ret_dict.update(catalog_ret_dict['comp_info_dict'])
        self._library.update(ret_dict)
        return ret_dict

    def make_model_info(self, modelkey):
        """ Build a dictionary with the information for a particular model.

        Parameters
        ----------

        modelkey : str
            Key used to identify this particular model

        Return `ModelInfo`
        """
        model = self.read_model_yaml(modelkey)
        sources = model['sources']
        components = OrderedDict()
        spec_model_yaml = self._name_factory.fullpath(localpath=model['spectral_models'])
        self._spec_lib.update(yaml.safe_load(open(spec_model_yaml)))
        for source, source_info in sources.items():
            model_type = source_info.get('model_type', None)
            version = source_info['version']
            spec_type = source_info['SpectrumType']
            sourcekey = "%s_%s" % (source, version)
            if model_type == 'galprop_rings':
                comp_info_dict = self.gmm.diffuse_comp_info_dicts(version)
                def_spec_type = spec_type['default']
                for comp_key, comp_info in comp_info_dict.items():
                    model_comp = ModelComponent(info=comp_info,
                                                spectrum=\
                                                    self._spec_lib[spec_type.get(comp_key,
                                                                                 def_spec_type)])
                    components[comp_key] = model_comp
            elif model_type == 'Catalog':
                comp_info_dict = self.csm.split_comp_info_dict(source, version)
                def_spec_type = spec_type['default']
                for comp_key, comp_info in comp_info_dict.items():
                    model_comp = ModelComponent(info=comp_info,
                                                spectrum=\
                                                    self._spec_lib[spec_type.get(comp_key,
                                                                                 def_spec_type)])
                    components[comp_key] = model_comp
            else:
                comp_info = self.dmm.diffuse_comp_info(sourcekey)
                model_comp = ModelComponent(info=comp_info,
                                            spectrum=self._spec_lib[spec_type])
                components[sourcekey] = model_comp
        ret_val = ModelInfo(model_name=modelkey,
                            model_components=components)
        self._models[modelkey] = ret_val
        return ret_val

    def make_srcmap_manifest(self, modelkey, components, data):
        """Build a yaml file that specfies how to make the srcmap files for a particular model

        Parameters
        ----------

        modelkey : str
            Key used to identify this particular model
        components : list
            The binning components used in this analysis
        data : str
            Path to file containing dataset definition
        """
        try:
            model_info = self._models[modelkey]
        except KeyError:
            model_info = self.make_model_info(modelkey)
        self._name_factory.update_base_dict(data)
        outfile = os.path.join('analysis', 'model_%s' %
                               modelkey, 'srcmap_manifest_%s.yaml' % modelkey)
        manifest = model_info.make_srcmap_manifest(
            components, self._name_factory)

        outdir = os.path.dirname(outfile)
        try:
            os.makedirs(outdir)
        except OSError:
            pass
        utils.write_yaml(manifest, outfile)

    def make_fermipy_config_yaml(self, modelkey, components, data, **kwargs):
        """Build a fermipy top-level yaml configuration file

        Parameters
        ----------

        modelkey : str
            Key used to identify this particular model
        components : list
            The binning components used in this analysis
        data : str
            Path to file containing dataset definition
        """
        model_dir = os.path.join('analysis', 'model_%s' % modelkey)
        hpx_order = kwargs.get('hpx_order', 9)
        self._name_factory.update_base_dict(data)
        try:
            model_info = self._models[modelkey]
        except KeyError:
            model_info = self.make_model_info(modelkey)

        model_info.make_model_rois(components, self._name_factory)

        #source_names = model_info.component_names

        master_xml_mdl = os.path.basename(
            self._name_factory.master_srcmdl_xml(modelkey=modelkey))

        master_data = dict(scfile=self._name_factory.ft2file(fullpath=True),
                           cacheft1=False)
        master_binning = dict(projtype='HPX',
                              coordsys=kwargs.get('coordsys', 'GAL'),
                              roiwidth=180.,
                              binsperdec=8,
                              hpx_ordering_scheme="RING",
                              hpx_order=hpx_order,
                              hpx_ebin=True)
        master_fileio = dict(outdir=model_dir,
                             logfile=os.path.join(model_dir, 'fermipy.log'))
        master_gtlike = dict(irfs=self._name_factory.irfs(**kwargs),
                             edisp_disable=['isodiff', 'diffuse', 'limb'])
        master_selection = dict(glat=0., glon=0., radius=180.)
        master_model = dict(catalogs=[master_xml_mdl])

        master = dict(data=master_data,
                      binning=master_binning,
                      fileio=master_fileio,
                      selection=master_selection,
                      gtlike=master_gtlike,
                      model=master_model,
                      components=[])

        fermipy_dict = master

        #comp_rois = {}

        for comp in components:
            zcut = "zmax%i" % comp.zmax
            compkey = "%s_%s" % (zcut, comp.make_key(
                '{ebin_name}_{evtype_name}'))
            name_keys = dict(zcut=zcut,
                             modelkey=modelkey,
                             component=compkey,
                             fullpath=True)
            comp_data = dict(ltcube=self._name_factory.ltcube(**name_keys))
            comp_selection = dict(logemin=comp.log_emin,
                                  logemax=comp.log_emax,
                                  zmax=comp.zmax,
                                  evtype=comp.evtype)
            comp_binning = dict(enumbins=comp.enumbins,
                                hpx_order=min(comp.hpx_order, hpx_order))
            comp_gtlike = dict(srcmap=self._name_factory.merged_srcmaps(**name_keys),
                               bexpmap=self._name_factory.bexpcube(**name_keys))
            #comp_roi_source_info = {}

            comp_xml_mdl = os.path.basename(self._name_factory.comp_srcmdl_xml(modelkey=modelkey,
                                                                               component=compkey))
            comp_model = dict(catalogs=[master_xml_mdl, comp_xml_mdl])
            sub_dict = dict(data=comp_data,
                            binning=comp_binning,
                            selection=comp_selection,
                            gtlike=comp_gtlike,
                            model=comp_model)
            fermipy_dict['components'].append(sub_dict)

        outfile = os.path.join(model_dir, 'config.yaml')
        print ("Writing fermipy config file %s"%outfile)
        utils.write_yaml(fermipy_dict, outfile)
        return fermipy_dict

    @staticmethod
    def get_sub_comp_info(source_info, comp):
        """Build and return information about a sub-component for a particular selection
        """
        sub_comps = source_info.get('components', None)
        if sub_comps is None:
            return source_info.copy()
        moving = source_info.get('moving', False)
        selection_dependent = source_info.get('selection_dependent', False)
        if selection_dependent:
            key = comp.make_key('{ebin_name}_{evtype_name}')
        elif moving:
            key = "zmax%i" % comp.zmax
        ret_dict = source_info.copy()
        ret_dict.update(sub_comps[key])
        return ret_dict