def __init__(self, link, **kwargs):
"""C'tor
Keyword arguments
-----------------
interface : `SysInterface` subclass
Object used to interface with batch system
usage : str
Usage string for argument parser
description : str
Description string for argument parser
job_archive : `fermipy.job_archive.JobArchive` [optional]
Archive used to track jobs and associated data products
Defaults to None
scatter : `fermipy.chain.Link`
Link run for the scatter stage
Defaults to None
"""
kwargs.setdefault('interface', get_batch_job_interface(self.job_time))
self._scatter_link = link
self.default_options.update(self.default_options_base.copy())
Link.__init__(self, **kwargs)
# Override the value of job_check_sleep to avoid excess waiting
job_check_sleep = np.clip(int(self.job_time / 5), 60, 300)
self.args['job_check_sleep'] = job_check_sleep
self._job_configs = {}
def __init__(self, link, **kwargs):
"""C'tor
Keyword arguments
-----------------
interface : `SysInterface` subclass
Object used to interface with batch system
usage : str
Usage string for argument parser
description : str
Description string for argument parser
job_archive : `fermipy.job_archive.JobArchive` [optional]
Archive used to track jobs and associated data products
Defaults to None
scatter : `fermipy.chain.Link`
Link run for the scatter stage
Defaults to None
"""
kwargs.setdefault('interface', get_batch_job_interface(self.job_time))
self._scatter_link = link
self.default_options.update(self.default_options_base.copy())
Link.__init__(self, **kwargs)
# Override the value of job_check_sleep to avoid excess waiting
job_check_sleep = np.clip(int(self.job_time / 5), 60, 300)
self.args['job_check_sleep'] = job_check_sleep
self._job_configs = {}
class PlotStackSpectra(Link):
"""Small class to plot the stacked spectra from pre-computed tables.
"""
appname = 'stack-plot-stack-spectra'
linkname_default = 'plot-stack-spectra'
usage = '%s [options]' % (appname)
description = "Plot the stacked spectra stored in pre-computed tables"
default_options = dict(infile=defaults.generic['infile'],
outfile=defaults.generic['outfile'],
spec=defaults.common['spec'],
index=defaults.common['index'],
spec_type=defaults.common['spec_type'])
__doc__ += Link.construct_docstring(default_options)
def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
stack_spec_table = StackSpecTable.create_from_fits(args.infile)
stack_plot_by_index = plot_stack_spectra_by_index(
stack_spec_table, spec=args.spec, spec_type=args.spec_type)
stack_plot_by_spec = plot_stack_spectra_by_spec(
stack_spec_table, index=args.index, spec_type=args.spec_type)
if args.outfile:
stack_plot_by_index[0].savefig(
args.outfile.replace('.png', '_%s.png' % args.spec))
stack_plot_by_spec[0].savefig(
args.outfile.replace('.png', '_%1.FGeV.png' % args.index))
class PlotStack(Link):
"""Small class to plot the likelihood vs norm and spectral index
"""
appname = 'stack-plot-stack'
linkname_default = 'plot-stack'
usage = "%s [options]" % (appname)
description = "Plot the likelihood vs norm and index"
default_options = dict(infile=defaults.generic['infile'],
outfile=defaults.generic['outfile'],
spec=defaults.common['spec'],
global_min=defaults.common['global_min'])
__doc__ += Link.construct_docstring(default_options)
def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
exttype = splitext(args.infile)[-1]
if exttype in ['.fits']:
stack_castro = StackCastroData.create_from_fitsfile(
args.infile, args.spec, norm_type='stack')
else:
raise ValueError("Can not read file type %s for SED" % exttype)
stack_plot = plot_stack_castro(stack_castro,
global_min=args.global_min)
if args.outfile:
stack_plot[0].savefig(args.outfile)
return None
return stack_plot
class PlotDM(Link):
"""Small class to plot the likelihood vs <sigma v> and DM particle mass
"""
appname = 'dmpipe-plot-dm'
linkname_default = 'plot-dm'
usage = "%s [options]" % (appname)
description = "Plot the likelihood vs <sigma v> and DM particle mass"
default_options = dict(infile=defaults.generic['infile'],
outfile=defaults.generic['outfile'],
chan=defaults.common['chan'],
global_min=defaults.common['global_min'])
__doc__ += Link.construct_docstring(default_options)
def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
exttype = splitext(args.infile)[-1]
if exttype in ['.fits']:
dm_castro = DMCastroData.create_from_fitsfile(
args.infile, args.chan)
elif exttype in ['.yaml']:
dm_castro = DMCastroData.create_from_yamlfile(
args.infile, args.chan)
else:
raise ValueError("Can not read file type %s for SED" % exttype)
dm_plot = plot_dm_castro(dm_castro, global_min=args.global_min)
if args.outfile:
dm_plot[0].savefig(args.outfile)
return None
return dm_plot
class Gtlink_expcube2_wcs(Gtlink):
"""Small wrapper to run gtexpcube2 """
appname = 'gtexpcube2'
linkname_default = 'gtexpcube2'
usage = '%s [options]' % (appname)
description = "Link to run %s" % (appname)
default_options = dict(
irfs=diffuse_defaults.gtopts['irfs'],
evtype=diffuse_defaults.gtopts['evtype'],
cmap=('none', "Input counts cube template", str),
emin=(100., "Start energy (MeV) of first bin", float),
emax=(1000000., "Stop energy (MeV) of last bin", float),
enumbins=(12, "Number of logarithmically-spaced energy bins", int),
binsz=(0.25, "Image scale (in degrees/pixel)", float),
xref=(0., "First coordinate of image center in degrees (RA or GLON)",
float),
yref=(0., "Second coordinate of image center in degrees (DEC or GLAT)",
float),
axisrot=(0., "Rotation angle of image axis, in degrees", float),
proj=("CAR",
"Projection method e.g. AIT|ARC|CAR|GLS|MER|NCP|SIN|STG|TAN",
str),
nxpix=(1440, "Size of the X axis in pixels", int),
nypix=(720, "Size of the Y axis in pixels", int),
infile=(None, "Input livetime cube file", str),
outfile=diffuse_defaults.gtopts['outfile'],
coordsys=('GAL', "Coordinate system", str))
default_file_args = dict(infile=FileFlags.input_mask,
cmap=FileFlags.input_mask,
outfile=FileFlags.output_mask)
__doc__ += Link.construct_docstring(default_options)
class ResidualCRChain(Chain):
"""Chain to preform analysis of residual cosmic-ray contamination
This chain consists of:
split-and-mktime : `SplitAndMktimeChain`
Chain to bin up the data and make exposure cubes
residual-cr : `ResidualCR`
Residual CR analysis
"""
appname = 'fermipy-residual-cr-chain'
linkname_default = 'residual-cr-chain'
usage = '%s [options]' % (appname)
description = 'Run residual cosmic ray analysis'
default_options = dict(config=diffuse_defaults.diffuse['config'])
__doc__ += Link.construct_docstring(default_options)
def __init__(self, **kwargs):
"""C'tor
"""
super(ResidualCRChain, self).__init__(**kwargs)
self.comp_dict = None
def _map_arguments(self, args):
"""Map from the top-level arguments to the arguments provided to
the indiviudal links """
config_yaml = args['config']
config_dict = load_yaml(config_yaml)
data = config_dict.get('data')
comp = config_dict.get('comp')
dry_run = args.get('dry_run', False)
self._set_link('prepare', SplitAndMktimeChain,
comp=comp, data=data,
ft1file=config_dict['ft1file'],
ft2file=config_dict['ft2file'],
hpx_order_ccube=config_dict.get('hpx_order_ccube', 7),
hpx_order_expcube=config_dict.get('hpx_order_expcube', 7),
mktime=config_dict.get('mktimefitler', None),
do_ltsum=config_dict.get('do_ltsum', False),
scratch=config_dict.get('scratch', None),
dry_run=dry_run)
self._set_link('residual-cr', ResidualCR_SG,
comp=comp, data=data,
mktimefilter=config_dict.get('mktimefitler', None),
hpx_order=config_dict.get('hpx_order_fitting', 4),
clean=config_dict.get('clean_class', None),
dirty=config_dict.get('dirty_class', None),
select_factor=config_dict.get('select_factor', None),
mask_factor=config_dict.get('mask_factor', None),
sigma=config_dict.get('sigma', None),
full_output=config_dict.get('full_output', False),
dry_run=dry_run)
def update_args(self, override_args):
"""Update the argument used to invoke the application
See help for `chain.Link` for details
This calls the base class function then fills the parameters of the GtApp object
"""
Link.update_args(self, override_args)
dry_run = override_args.get('dry_run', False)
if self.__app is None:
self.__app = build_gtapp(self.appname, dry_run, **self.args)
#except:
# raise ValueError("Failed to build link %s %s %s" %
# (self.linkname, self.appname, self.args))
else:
update_gtapp(self.__app, **self.args)
class PlotDMSpectra(Link):
"""Small class to plot the DM spectra from pre-computed tables.
"""
appname = 'dmpipe-plot-dm-spectra'
linkname_default = 'plot-dm-spectra'
usage = '%s [options]' % (appname)
description = "Plot the DM spectra stored in pre-computed tables"
default_options = dict(infile=defaults.generic['infile'],
outfile=defaults.generic['outfile'],
chan=defaults.common['chan'],
mass=defaults.common['mass'],
spec_type=defaults.common['spec_type'])
__doc__ += Link.construct_docstring(default_options)
def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
dm_spec_table = DMSpecTable.create_from_fits(args.infile)
dm_plot_by_mass = plot_dm_spectra_by_mass(dm_spec_table,
chan=args.chan,
spec_type=args.spec_type)
dm_plot_by_chan = plot_dm_spectra_by_channel(dm_spec_table,
mass=args.mass,
spec_type=args.spec_type)
if args.outfile:
dm_plot_by_mass[0].savefig(
args.outfile.replace('.png', '_%s.png' % args.chan))
dm_plot_by_chan[0].savefig(
args.outfile.replace('.png', '_%1.FGeV.png' % args.mass))
class Gtexpcube2wcs_SG(ScatterGather):
"""Small class to generate configurations for gtexphpsun
"""
appname = 'fermipy-gtexpcube2wcs-sg'
usage = "%s [options]" % (appname)
description = "Submit gtexpcube2 jobs in parallel"
clientclass = Gtlink_expcube2_wcs
job_time = 300
default_options = dict(
comp=diffuse_defaults.diffuse['comp'],
data=diffuse_defaults.diffuse['data'],
mktimefilter=diffuse_defaults.diffuse['mktimefilter'],
binsz=(1.0, "Image scale (in degrees/pixel)", float),
nxpix=(360, "Size of the X axis in pixels", int),
nypix=(180, "Size of the Y axis in pixels", int))
__doc__ += Link.construct_docstring(default_options)
def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
components = Component.build_from_yamlfile(args['comp'])
NAME_FACTORY.update_base_dict(args['data'])
mktime = args['mktimefilter']
base_config = dict(nxpix=args['nxpix'],
nypix=args['nypix'],
binsz=args['binsz'])
for comp in components:
zcut = "zmax%i" % comp.zmax
key = comp.make_key('{ebin_name}_{evtype_name}')
name_keys = dict(zcut=zcut,
ebin=comp.ebin_name,
psftype=comp.evtype_name,
irf_ver=NAME_FACTORY.irf_ver(),
coordsys=comp.coordsys,
mktime=mktime,
fullpath=True)
outfile = NAME_FACTORY.bexpcube(**name_keys)
ltcube = NAME_FACTORY.ltcube(**name_keys)
full_config = base_config.copy()
full_config.update(
dict(infile=ltcube,
outfile=outfile,
irfs=NAME_FACTORY.irfs(**name_keys),
evtype=comp.evtype,
emin=comp.emin,
emax=comp.emax,
enumbins=comp.enumbins,
logfile=make_nfs_path(outfile.replace('.fits', '.log'))))
job_configs[key] = full_config
return job_configs
def update_args(self, override_args):
"""Update the argument used to invoke the application
See help for `chain.Link` for details
This calls the base class function then fills the parameters of the GtApp object
"""
Link.update_args(self, override_args)
dry_run = override_args.get('dry_run', False)
if self.__app is None:
self.__app = build_gtapp(self.appname, dry_run, **self.args)
#except:
# raise ValueError("Failed to build link %s %s %s" %
# (self.linkname, self.appname, self.args))
else:
update_gtapp(self.__app, **self.args)
class PlotCastro(Link):
"""Small class to plot an SED as a 'Castro' plot.
"""
appname = 'fermipy-plot-castro'
linkname_default = 'plot-castro'
usage = '%s [options]' % (appname)
description = "Plot likelihood v. flux normalization and energy"
default_options = dict(infile=defaults.generic['infile'],
outfile=defaults.generic['outfile'])
__doc__ += Link.construct_docstring(default_options)
def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
exttype = splitext(args.infile)[-1]
if exttype in ['.fits', '.npy']:
castro_data = CastroData.create_from_sedfile(args.infile)
elif exttype in ['.yaml']:
castro_data = CastroData.create_from_yamlfile(args.infile)
else:
raise ValueError("Can not read file type %s for SED" % extype)
ylims = [1e-8, 1e-5]
plot = plotCastro(castro_data, ylims)
if args.outfile:
plot[0].savefig(args.outfile)
class SplitAndBin_SG(ScatterGather):
"""Small class to generate configurations for SplitAndBin
"""
appname = 'fermipy-split-and-bin-sg'
usage = "%s [options]" % (appname)
description = "Prepare data for diffuse all-sky analysis"
clientclass = SplitAndBin
job_time = 1500
default_options = dict(
comp=diffuse_defaults.diffuse['comp'],
data=diffuse_defaults.diffuse['data'],
hpx_order_max=diffuse_defaults.diffuse['hpx_order_ccube'],
ft1file=(None, 'Input FT1 file', str),
scratch=(None, 'Path to scratch area', str))
__doc__ += Link.construct_docstring(default_options)
def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
comp_file = args.get('comp', None)
if comp_file is not None:
comp_dict = yaml.safe_load(open(comp_file))
coordsys = comp_dict.pop('coordsys')
for v in comp_dict.values():
v['coordsys'] = coordsys
else:
return job_configs
NAME_FACTORY.update_base_dict(args['data'])
inputfiles = create_inputlist(args['ft1file'])
outdir_base = os.path.join(NAME_FACTORY.base_dict['basedir'],
'counts_cubes')
for idx, infile in enumerate(inputfiles):
key = "%06i" % idx
output_dir = os.path.join(outdir_base, key)
try:
os.mkdir(output_dir)
except OSError:
pass
logfile = make_nfs_path(
os.path.join(output_dir, 'scatter_%s.log' % key))
#job_configs[key] = args.copy()
job_configs[key] = args.copy()
job_configs[key].update(
dict(ft1file=infile,
comp=args['comp'],
hpx_order_max=args['hpx_order_max'],
outdir=outdir_base,
outkey=key,
logfile=logfile,
pfiles=output_dir))
return job_configs
class PlotLimits(Link):
"""Small class to Plot Stack limits on norm v. index
"""
appname = 'stack-plot-limits'
linkname_default = 'plot-limits'
usage = '%s [options]' % (appname)
description = "Plot stacked limits on norm v. index"
default_options = dict(infile=defaults.generic['infile'],
outfile=defaults.generic['outfile'],
spec=defaults.common['spec'],
bands=defaults.collect['bands'],
sim=defaults.sims['sim'])
__doc__ += Link.construct_docstring(default_options)
def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
limit_col = 'ul_0.95'
ylims = (1.0e-2, 1.0e2)
if is_not_null(args.infile):
tab_m = Table.read(args.infile, hdu="indices")
tab_s = Table.read(args.infile, hdu=args.spec)
xvals = tab_m['indices'][0]
yvals = tab_s[limit_col][0]
ldict = dict(limits=(xvals, yvals))
else:
ldict = {}
if is_not_null(args.bands):
tab_b = Table.read(args.bands, hdu=args.spec)
tab_bm = Table.read(args.bands, hdu="indices")
bands = get_ul_bands(tab_b, limit_col)
bands['indices'] = tab_bm['indices'][0]
else:
bands = None
if is_not_null(args.sim):
sim_srcs = load_yaml(args.sim)
injected_src = sim_srcs.get('injected_source', None)
else:
injected_src = None
xlims = (1e1, 1e4)
stack_plot = plot_limits_from_arrays(ldict, xlims, ylims, bands)
if injected_src is not None:
mc_model = injected_src['source_model']
plot_mc_truth(stack_plot[1], mc_model)
if args.outfile:
stack_plot[0].savefig(args.outfile)
return None
return stack_plot
class PlotSEDSummaryBands_SG(ScatterGather):
"""Small class to generate configurations for the `PlotSEDLimits` class.
This loops over all the targets defined in the target list.
"""
appname = 'fermipy-plot-sed-summary-bands-sg'
usage = "%s [options]" % (appname)
description = "Make sed plots for set of targets"
clientclass = PlotSEDSummaryBands
job_time = 60
default_options = dict(ttype=defaults.common['ttype'],
targetlist=defaults.common['targetlist'],
nsims=defaults.sims['nsims'],
seed=defaults.sims['seed'],
band_sim=defaults.sims['band_sim'],
band_type=defaults.sims['band_type'])
__doc__ += Link.construct_docstring(default_options)
def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
ttype = args['ttype']
(targets_yaml, sim) = NAME_FACTORY.resolve_targetfile(args)
if targets_yaml is None:
return job_configs
targets = load_yaml(targets_yaml)
first = args['seed']
last = first + args['nsims'] - 1
for target_name, target_list in targets.items():
for targ_prof in target_list:
name_keys = dict(target_type=ttype,
target_name=target_name,
profile=targ_prof,
fullpath=True)
targ_key = "%s_%s" % (target_name, targ_prof)
input_path = NAME_FACTORY.sedfile(**name_keys)
output_path = input_path.replace('.fits', '.png')
logfile = make_nfs_path(input_path.replace('.fits', '.log'))
sed_file = NAME_FACTORY.sim_sedfile(sim_name=args['band_sim'],
**name_keys)
summaryfile = sed_file.replace(
'_SEED.fits', '_summary_%06i_%06i.fits' % (first, last))
job_config = dict(infile=input_path,
outfile=output_path,
summaryfile=summaryfile,
band_sim=args['band_sim'],
logfile=logfile)
job_configs[targ_key] = job_config
return job_configs
class PipelineData(Chain):
"""Chain together the steps of the dSphs pipeline
This chain consists of:
analyze-roi : `AnalyzeROI_SG`
Do the baseline analysis for each target in the target list.
analyze-sed : `AnalyzeSED_SG`
Extract the SED for each profile for each target in the target list.
Optional plotting modules includde
plot-castro : `PlotCastro`
Make 'Castro' plots of the SEDs for each profile for each target
in the target list.
"""
appname = 'fermipy-pipeline-data'
linkname_default = 'pipeline-data'
usage = '%s [options]' % (appname)
description = 'Data analysis pipeline'
default_options = dict(config=defaults.common['config'],
sim=defaults.sims['sim'],
dry_run=defaults.common['dry_run'])
__doc__ += Link.construct_docstring(default_options)
def _map_arguments(self, args):
"""Map from the top-level arguments to the arguments provided to
the indiviudal links """
config_yaml = args['config']
config_dict = load_yaml(config_yaml)
ttype = config_dict.get('ttype')
config_localpath = config_dict.get('config_localpath', None)
specfile = config_dict.get('specfile')
targetlist = config_dict.get('targetlist')
data_plotting = config_dict.get('data_plotting')
self._set_link('analyze-roi',
AnalyzeROI_SG,
ttype=ttype,
targetlist=targetlist,
config=config_localpath)
self._set_link('analyze-sed',
AnalyzeSED_SG,
ttype=ttype,
targetlist=targetlist,
config=config_localpath)
config_plot_castro = _get_plot_config(data_plotting, 'plot-castro')
if config_plot_castro is not None:
self._set_link('plot-castro-sg',
PlotCastro_SG,
ttype=ttype,
targetlist=targetlist,
**config_plot_castro)
class Gtsuntemp_SG(ScatterGather):
"""Small class to generate configurations for gtsuntemp
"""
appname = 'fermipy-gtsuntemp-sg'
usage = "%s [options]" % (appname)
description = "Submit gtsuntemp jobs in parallel"
clientclass = Gtlink_suntemp
job_time = 300
default_options = dict(
comp=diffuse_defaults.diffuse['comp'],
data=diffuse_defaults.diffuse['data'],
mktimefilter=diffuse_defaults.diffuse['mktimefilter'],
sourcekeys=diffuse_defaults.sun_moon['sourcekeys'])
__doc__ += Link.construct_docstring(default_options)
def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
components = Component.build_from_yamlfile(args['comp'])
NAME_FACTORY.update_base_dict(args['data'])
# FIXME
mktime = args['mktimefilter']
for comp in components:
for sourcekey in args['sourcekeys']:
zcut = "zmax%i" % comp.zmax
key = comp.make_key(
'{ebin_name}_{evtype_name}') + "_%s" % sourcekey
name_keys = dict(zcut=zcut,
ebin=comp.ebin_name,
psftype=comp.evtype_name,
irf_ver=NAME_FACTORY.irf_ver(),
sourcekey=sourcekey,
mktime=mktime,
coordsys=comp.coordsys,
fullpath=True)
outfile = NAME_FACTORY.template_sunmoon(**name_keys)
job_configs[key] = dict(
expsun=NAME_FACTORY.bexpcube_sun(**name_keys),
avgexp=NAME_FACTORY.bexpcube(**name_keys),
sunprof=NAME_FACTORY.angprofile(**name_keys),
cmap='none',
outfile=outfile,
irfs=NAME_FACTORY.irfs(**name_keys),
evtype=comp.evtype,
emin=comp.emin,
emax=comp.emax,
enumbins=comp.enumbins,
logfile=outfile.replace('.fits', '.log'))
return job_configs
class FitDiffuse_SG(ScatterGather):
"""Small class to generate configurations for this script
This loops over all the models defined in the model list.
"""
appname = 'fermipy-fit-diffuse-sg'
usage = "%s [options]" % (appname)
description = "Run analyses on a series of models"
clientclass = FitDiffuse
job_time = 6000
default_options = dict(models=defaults.diffuse['models'],
config=defaults.diffuse['config'],
roi_baseline=defaults.diffuse['roi_baseline'],
fit_strategy=defaults.diffuse['fit_strategy'],
input_pars=defaults.diffuse['input_pars'],
load_baseline=defaults.diffuse['load_baseline'],
make_plots=defaults.diffuse['make_plots'])
__doc__ += Link.construct_docstring(default_options)
def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
# Tweak the batch job args
try:
self._interface._lsf_args.update(dict(n=2))
self._interface._lsf_args.update(
dict(R='\"select[rhel60&&!fell] -R span[hosts=1]\"'))
except AttributeError:
pass
models = load_yaml(args['models'])
base_config = dict(fit_strategy=args['fit_strategy'],
input_pars=args['input_pars'],
load_baseline=args['load_baseline'],
make_plots=args['make_plots'])
for modelkey in models:
config_file = os.path.join('analysis', 'model_%s' % modelkey,
args['config'])
#roi_baseline = os.path.join('analysis', 'model_%s' % modelkey,
# args['roi_baseline'])
roi_baseline = args['roi_baseline']
logfile = os.path.join('analysis', 'model_%s' % modelkey,
'fit_%s.log' % modelkey)
job_config = base_config.copy()
job_config.update(
dict(config=config_file,
roi_baseline=roi_baseline,
logfile=logfile))
job_configs[modelkey] = job_config
return job_configs
class RandomDirGen_SG(ScatterGather):
"""Small class to generate configurations for this script
This adds the following arguments:
"""
appname = 'fermipy-random-dir-gen-sg'
usage = "%s [options]" % (appname)
description = "Run analyses on a series of ROIs"
clientclass = RandomDirGen
job_time = 60
default_options = dict(ttype=defaults.common['ttype'],
targetlist=defaults.common['targetlist'],
config=defaults.common['config'],
rand_config=defaults.sims['rand_config'],
sim=defaults.sims['sim'])
__doc__ += Link.construct_docstring(default_options)
def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
ttype = args['ttype']
(targets_yaml, sim) = NAME_FACTORY.resolve_targetfile(args)
if targets_yaml is None:
return job_configs
config_yaml = 'config.yaml'
config_override = args.get('config')
if is_not_null(config_override):
config_yaml = config_override
rand_yaml = NAME_FACTORY.resolve_randconfig(args)
targets = load_yaml(targets_yaml)
base_config = dict(rand_config=rand_yaml)
for target_name in targets.keys():
name_keys = dict(target_type=ttype,
target_name=target_name,
sim_name=sim,
fullpath=True)
simdir = NAME_FACTORY.sim_targetdir(**name_keys)
config_path = os.path.join(simdir, config_yaml)
outfile = os.path.join(simdir, 'skydirs.yaml')
logfile = make_nfs_path(outfile.replace('yaml', 'log'))
job_config = base_config.copy()
job_config.update(
dict(config=config_path, outfile=outfile, logfile=logfile))
job_configs[target_name] = job_config
return job_configs
class CollectLimits(Link):
"""Small class to collect limit results from a series of simulations.
"""
appname = 'stack-collect-limits'
linkname_default = 'collect-limits'
usage = '%s [options]' % (appname)
description = "Collect Limits from simulations"
default_options = dict(limitfile=defaults.generic['limitfile'],
specconfig=defaults.common['specconfig'],
outfile=defaults.generic['outfile'],
summaryfile=defaults.generic['summaryfile'],
nsims=defaults.sims['nsims'],
seed=defaults.sims['seed'],
dry_run=defaults.common['dry_run'])
__doc__ += Link.construct_docstring(default_options)
def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
limitfile = args.limitfile
first = args.seed
last = first + args.nsims
flist = [limitfile.replace("_SEED.fits", "_%06i.fits" % seed)\
for seed in range(first, last)]
spec_config = load_yaml(args.specconfig)
specs = spec_config['specs']
sum_specs = specs.copy()
outfile = args.outfile
summaryfile = args.summaryfile
hdus = sum_specs + ['INDICES']
out_tables, out_names = vstack_tables(flist, hdus)
if is_not_null(outfile):
fits_utils.write_tables_to_fits(outfile,
out_tables,
namelist=out_names)
if is_not_null(summaryfile):
summary_tables = []
for ot in out_tables[0:-1]:
summary_table = summarize_limits_results(ot)
summary_tables.append(summary_table)
summary_tables.append(Table(out_tables[-1][0]))
fits_utils.write_tables_to_fits(summaryfile,
summary_tables,
namelist=out_names)
class AnalyzeROI_SG(ScatterGather):
"""Small class to generate configurations for the `AnalyzeROI` class.
This loops over all the targets defined in the target list.
"""
appname = 'fermipy-analyze-roi-sg'
usage = "%s [options]" % (appname)
description = "Run analyses on a series of ROIs"
clientclass = AnalyzeROI
job_time = 1500
default_options = dict(ttype=defaults.common['ttype'],
targetlist=defaults.common['targetlist'],
config=defaults.common['config'],
roi_baseline=defaults.common['roi_baseline'],
make_plots=defaults.common['make_plots'])
__doc__ += Link.construct_docstring(default_options)
def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
ttype = args['ttype']
(targets_yaml, sim) = NAME_FACTORY.resolve_targetfile(args)
if sim is not None:
raise ValueError("Found 'sim' argument on AnalyzeROI_SG config.")
if targets_yaml is None:
return job_configs
config_yaml = 'config.yaml'
config_override = args.get('config')
if is_not_null(config_override):
config_yaml = config_override
targets = load_yaml(targets_yaml)
base_config = dict(roi_baseline=args['roi_baseline'],
make_plots=args['make_plots'])
for target_name in targets.keys():
name_keys = dict(target_type=ttype,
target_name=target_name,
fullpath=True)
target_dir = NAME_FACTORY.targetdir(**name_keys)
config_path = os.path.join(target_dir, config_yaml)
logfile = make_nfs_path(
os.path.join(target_dir,
"%s_%s.log" % (self.linkname, target_name)))
job_config = base_config.copy()
job_config.update(dict(config=config_path, logfile=logfile))
job_configs[target_name] = job_config
return job_configs
class SplitAndBinChain(Chain):
"""Chain to run split and bin and then make exposure cubes
This chain consists of:
split-and-bin : `SplitAndBin_SG`
Chain to make the binned counts maps for each input file
coadd-split : `CoaddSplit_SG`
Link to co-add the binnec counts maps files
expcube2 : `Gtexpcube2_SG`
Link to make the corresponding binned exposure maps
"""
appname = 'fermipy-split-and-bin-chain'
linkname_default = 'split-and-bin-chain'
usage = '%s [options]' % (appname)
description = 'Run split-and-bin, coadd-split and exposure'
default_options = dict(data=diffuse_defaults.diffuse['data'],
comp=diffuse_defaults.diffuse['comp'],
ft1file=diffuse_defaults.diffuse['ft1file'],
hpx_order_ccube=diffuse_defaults.diffuse['hpx_order_ccube'],
hpx_order_expcube=diffuse_defaults.diffuse['hpx_order_expcube'],
scratch=diffuse_defaults.diffuse['scratch'],
dry_run=diffuse_defaults.diffuse['dry_run'])
__doc__ += Link.construct_docstring(default_options)
def _map_arguments(self, args):
"""Map from the top-level arguments to the arguments provided to
the indiviudal links """
data = args.get('data')
comp = args.get('comp')
ft1file = args.get('ft1file')
scratch = args.get('scratch', None)
dry_run = args.get('dry_run', None)
self._set_link('split-and-bin', SplitAndBin_SG,
comp=comp, data=data,
hpx_order_max=args.get('hpx_order_ccube', 9),
ft1file=ft1file,
scratch=scratch,
dry_run=dry_run)
self._set_link('coadd-split', CoaddSplit_SG,
comp=comp, data=data,
ft1file=ft1file)
self._set_link('expcube2', Gtexpcube2_SG,
comp=comp, data=data,
hpx_order_max=args.get('hpx_order_expcube', 5),
dry_run=dry_run)
class SpecTable(Link):
"""Small class to build a table with all the stack spectra for this analysis
"""
appname = 'stack-spec-table'
linkname_default = 'spec-table'
usage = '%s [options]' % (appname)
description = "Build a table with the spectra for stacked analysis"
default_options = dict(ttype=defaults.common['ttype'],
config=defaults.common['config'],
specconfig=defaults.common['specconfig'],
specfile=defaults.common['specfile'],
clobber=defaults.common['clobber'])
__doc__ += Link.construct_docstring(default_options)
def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
if args.ttype is not None:
name_keys = dict(target_type=args.ttype, fullpath=True)
config_file = NAME_FACTORY.ttypeconfig(**name_keys)
spec_config = NAME_FACTORY.specconfig(**name_keys)
spec_file = NAME_FACTORY.specfile(**name_keys)
else:
config_file = None
spec_config = None
spec_file = None
if is_not_null(args.config):
config_file = args.config
if is_not_null(args.specconfig):
spec_config = args.specconfig
if is_not_null(args.specfile):
spec_file = args.specfile
if config_file is None:
sys.stderr.write('No input configuration file is specified')
return -1
if spec_config is None:
sys.stderr.write('No input spectra configurate file is specified')
return -1
if spec_file is None:
sys.stderr.write('No output spectra file is specified')
return -1
stack_spec_table = StackSpecTable.create_from_config(
config_file, spec_file)
stack_spec_table.write_fits(spec_file, args.clobber)
return 0
class AnalyzeROI(Link):
"""Small class that wraps an analysis script.
This particular script does baseline fitting of an ROI.
"""
appname = 'fermipy-analyze-roi'
linkname_default = 'analyze-roi'
usage = '%s [options]' % (appname)
description = "Run analysis of a single ROI"
default_options = dict(config=defaults.common['config'],
roi_baseline=defaults.common['roi_baseline'],
make_plots=defaults.common['make_plots'])
__doc__ += Link.construct_docstring(default_options)
def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
if not HAVE_ST:
raise RuntimeError(
"Trying to run fermipy analysis, but don't have ST")
gta = GTAnalysis(args.config,
logging={'verbosity': 3},
fileio={'workdir_regex': '\.xml$|\.npy$'})
gta.setup(overwrite=False)
baseline_roi_fit(gta,
make_plots=args.make_plots,
minmax_npred=[1e3, np.inf])
localize_sources(gta,
nstep=5,
dtheta_max=0.5,
update=True,
prefix='base',
make_plots=args.make_plots)
gta.find_sources(sqrt_ts_threshold=5.0,
search_skydir=gta.roi.skydir,
search_minmax_radius=[1.0, np.nan])
gta.optimize()
gta.print_roi()
gta.print_params()
gta.free_sources(skydir=gta.roi.skydir, distance=1.0, pars='norm')
gta.fit(covar=True)
gta.print_roi()
gta.print_params()
gta.write_roi(args.roi_baseline, make_plots=args.make_plots)
def print_summary(self, stream=sys.stdout, indent="", recurse_level=2):
"""Print a summary of the activity done by this `Link`.
Parameters
----------
stream : `file`
Stream to print to
indent : str
Indentation at start of line
recurse_level : int
Number of recursion levels to print
"""
Link.print_summary(self, stream, indent, recurse_level)
if recurse_level > 0:
recurse_level -= 1
indent += " "
stream.write("\n")
self._scatter_link.print_summary(stream, indent, recurse_level)
def print_summary(self, stream=sys.stdout, indent="", recurse_level=2):
"""Print a summary of the activity done by this `Link`.
Parameters
----------
stream : `file`
Stream to print to
indent : str
Indentation at start of line
recurse_level : int
Number of recursion levels to print
"""
Link.print_summary(self, stream, indent, recurse_level)
if recurse_level > 0:
recurse_level -= 1
indent += " "
stream.write("\n")
self._scatter_link.print_summary(stream, indent, recurse_level)
class GatherSrcmaps_SG(ScatterGather):
"""Small class to generate configurations for `Link_FermipyGatherSrcmaps`
"""
appname = 'fermipy-gather-srcmaps-sg'
usage = "%s [options]" % (appname)
description = "Submit fermipy-gather-srcmaps jobs in parallel"
clientclass = Link_FermipyGatherSrcmaps
job_time = 300
default_options = dict(comp=diffuse_defaults.diffuse['comp'],
data=diffuse_defaults.diffuse['data'],
library=diffuse_defaults.diffuse['library'])
__doc__ += Link.construct_docstring(default_options)
def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
components = Component.build_from_yamlfile(args['comp'])
NAME_FACTORY.update_base_dict(args['data'])
ret_dict = make_catalog_comp_dict(
library=args['library'], basedir=NAME_FACTORY.base_dict['basedir'])
catalog_info_dict = ret_dict['catalog_info_dict']
for catalog_name in catalog_info_dict:
for comp in components:
zcut = "zmax%i" % comp.zmax
key = comp.make_key('{ebin_name}_{evtype_name}')
name_keys = dict(zcut=zcut,
sourcekey=catalog_name,
ebin=comp.ebin_name,
psftype=comp.evtype_name,
coordsys=comp.coordsys,
irf_ver=NAME_FACTORY.irf_ver(),
mktime='none',
fullpath=True)
outfile = NAME_FACTORY.srcmaps(**name_keys)
outfile_tokens = os.path.splitext(outfile)
infile_regexp = "%s_*.fits" % outfile_tokens[0]
logfile = make_nfs_path(outfile.replace('.fits', '.log'))
job_configs[key] = dict(output=outfile,
args=infile_regexp,
logfile=logfile)
return job_configs
def print_summary(self, stream=sys.stdout, indent="", recurse_level=2):
"""Print a summary of the activity done by this `Chain`.
Parameters
-----------
stream : `file`
Stream to print to, must have 'write' method.
indent : str
Indentation at start of line
recurse_level : int
Number of recursion levels to print
"""
Link.print_summary(self, stream, indent, recurse_level)
if recurse_level > 0:
recurse_level -= 1
indent += " "
for link in self._links.values():
stream.write("\n")
link.print_summary(stream, indent, recurse_level)
class Link_FermipyCoadd(AppLink):
"""Small wrapper to run fermipy-coadd """
appname = 'fermipy-coadd'
linkname_default = 'coadd'
usage = '%s [options]' % (appname)
description = "Link to run %s" % (appname)
default_options = dict(args=([], "List of input files", list),
output=(None, "Output file", str))
default_file_args = dict(args=FileFlags.input_mask,
output=FileFlags.output_mask)
__doc__ += Link.construct_docstring(default_options)
class Gtexphpsun_SG(ScatterGather):
"""Small class to generate configurations for gtexphpsun
"""
appname = 'fermipy-gtexphpsun-sg'
usage = "%s [options]" % (appname)
description = "Submit gtexphpsun jobs in parallel"
clientclass = Gtlink_exphpsun
job_time = 300
default_options = dict(
comp=diffuse_defaults.diffuse['comp'],
data=diffuse_defaults.diffuse['data'],
mktimefilter=diffuse_defaults.diffuse['mktimefilter'])
__doc__ += Link.construct_docstring(default_options)
def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
components = Component.build_from_yamlfile(args['comp'])
NAME_FACTORY.update_base_dict(args['data'])
mktime = args['mktimefilter']
for comp in components:
zcut = "zmax%i" % comp.zmax
key = comp.make_key('{ebin_name}_{evtype_name}')
name_keys = dict(zcut=zcut,
ebin=comp.ebin_name,
psftype=comp.evtype_name,
irf_ver=NAME_FACTORY.irf_ver(),
mktime=mktime,
fullpath=True)
outfile = NAME_FACTORY.bexpcube_sun(**name_keys)
ltcube_sun = NAME_FACTORY.ltcube_sun(**name_keys)
job_configs[key] = dict(
infile=NAME_FACTORY.ltcube_sun(**name_keys),
outfile=outfile,
irfs=NAME_FACTORY.irfs(**name_keys),
evtype=comp.evtype,
emin=comp.emin,
emax=comp.emax,
enumbins=comp.enumbins,
logfile=make_nfs_path(outfile.replace('.fits', '.log')))
return job_configs
def print_summary(self, stream=sys.stdout, indent="", recurse_level=2):
"""Print a summary of the activity done by this `Chain`.
Parameters
-----------
stream : `file`
Stream to print to, must have 'write' method.
indent : str
Indentation at start of line
recurse_level : int
Number of recursion levels to print
"""
Link.print_summary(self, stream, indent, recurse_level)
if recurse_level > 0:
recurse_level -= 1
indent += " "
for link in self._links.values():
stream.write("\n")
link.print_summary(stream, indent, recurse_level)
class Gtlink_ltsum(Gtlink):
"""Small wrapper to run gtltsum """
appname = 'gtltsum'
linkname_default = 'gtltsum'
usage = '%s [options]' % (appname)
description = "Link to run %s" % (appname)
default_options = dict(infile1=(None, "Livetime cube 1 or list of files",
str),
infile2=("none", "Livetime cube 2", str),
outfile=(None, "Output file", str))
default_file_args = dict(infile1=FileFlags.input_mask,
outfile=FileFlags.output_mask)
__doc__ += Link.construct_docstring(default_options)
class Link_FermipyHealview(AppLink):
"""Small wrapper to run fermipy-healview """
appname = 'fermipy-healview'
linkname_default = 'fermipy-healview'
usage = '%s [options]' % (appname)
description = "Link to run %s" % (appname)
default_options = dict(input=(None, "Input file", str),
output=(None, "Output file name", str),
extension=(None, "FITS HDU with HEALPix map", str),
zscale=("log", "Scaling for color scale", str))
default_file_args = dict(args=FileFlags.input_mask,
output=FileFlags.output_mask)
__doc__ += Link.construct_docstring(default_options)
class Link_FermipyGatherSrcmaps(AppLink):
"""Small wrapper to run fermipy-gather-srcmaps """
appname = 'fermipy-gather-srcmaps'
linkname_default = 'gather-srcmaps'
usage = '%s [options]' % (appname)
description = "Link to run %s" % (appname)
default_options = dict(output=(None, "Output file name", str),
args=([], "List of input files", list),
gzip=(False, "Compress output", bool),
rm=(False, "Remove input files", bool),
clobber=(False, "Overwrite output", bool))
default_file_args = dict(args=FileFlags.input_mask,
output=FileFlags.output_mask)
__doc__ += Link.construct_docstring(default_options)