def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
limitfile = args.limitfile
first = args.seed
last = first + args.nsims
flist = [limitfile.replace("_SEED.fits", "_%06i.fits" % seed)\
for seed in range(first, last)]
spec_config = load_yaml(args.specconfig)
specs = spec_config['specs']
sum_specs = specs.copy()
outfile = args.outfile
summaryfile = args.summaryfile
hdus = sum_specs + ['INDICES']
out_tables, out_names = vstack_tables(flist, hdus)
if is_not_null(outfile):
fits_utils.write_tables_to_fits(outfile,
out_tables,
namelist=out_names)
if is_not_null(summaryfile):
summary_tables = []
for ot in out_tables[0:-1]:
summary_table = summarize_limits_results(ot)
summary_tables.append(summary_table)
summary_tables.append(Table(out_tables[-1][0]))
fits_utils.write_tables_to_fits(summaryfile,
summary_tables,
namelist=out_names)
def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
sedfile = args.sed_file
if is_not_null(args.config):
configfile = os.path.join(os.path.dirname(sedfile), args.config)
else:
configfile = os.path.join(os.path.dirname(sedfile), 'config.yaml')
nbins = _get_enum_bins(configfile)
first = args.seed
last = first + args.nsims
flist = [sedfile.replace("_SEED.fits", "_%06i.fits" % seed)
for seed in range(first, last)]
outfile = args.outfile
summaryfile = args.summaryfile
outtable = fill_output_table(
flist, "SED", CollectSED.collist, nbins=nbins)
if is_not_null(outfile):
outtable.write(outfile)
if is_not_null(summaryfile):
summary = summarize_sed_results(outtable)
summary.write(summaryfile)
def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
if args.chan.find('_decay') >= 0:
decay = True
limit_col = 'll_0.95'
ylims = (1e+22, 1e+28)
else:
decay = False
limit_col = 'ul_0.95'
ylims = (1e-28, 1e-22)
if is_not_null(args.infile):
tab_m = Table.read(args.infile, hdu="masses")
tab_s = Table.read(args.infile, hdu=args.chan)
xvals = tab_m['masses'][0]
yvals = tab_s[limit_col][0]
ldict = dict(limits=(xvals, yvals))
else:
ldict = {}
if is_not_null(args.bands):
tab_b = Table.read(args.bands, hdu=args.chan)
tab_bm = Table.read(args.bands, hdu="masses")
bands = get_ul_bands(tab_b, limit_col)
bands['masses'] = tab_bm['masses'][0]
else:
bands = None
if is_not_null(args.sim):
sim_srcs = load_yaml(args.sim)
injected_src = sim_srcs.get('injected_source', None)
else:
injected_src = None
xlims = (1e1, 1e4)
dm_plot = plot_limits_from_arrays(ldict,
xlims,
ylims,
bands,
decay=decay)
if injected_src is not None:
mc_model = injected_src['source_model']
plot_mc_truth(dm_plot[1], mc_model)
if args.outfile:
dm_plot[0].savefig(args.outfile)
return None
return dm_plot
def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
ttype = args['ttype']
(targets_yaml, sim) = NAME_FACTORY.resolve_targetfile(args)
if sim is not None:
raise ValueError("Found 'sim' argument on AnalyzeROI_SG config.")
if targets_yaml is None:
return job_configs
config_yaml = 'config.yaml'
config_override = args.get('config')
if is_not_null(config_override):
config_yaml = config_override
targets = load_yaml(targets_yaml)
base_config = dict(roi_baseline=args['roi_baseline'],
make_plots=args['make_plots'])
for target_name in targets.keys():
name_keys = dict(target_type=ttype,
target_name=target_name,
fullpath=True)
target_dir = NAME_FACTORY.targetdir(**name_keys)
config_path = os.path.join(target_dir, config_yaml)
logfile = make_nfs_path(os.path.join(
target_dir, "%s_%s.log" % (self.linkname, target_name)))
job_config = base_config.copy()
job_config.update(dict(config=config_path,
logfile=logfile))
job_configs[target_name] = job_config
return job_configs
def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
if not args.rosters:
raise RuntimeError("You must specify at least one target roster")
if is_null(args.ttype):
raise RuntimeError("You must specify a target type")
if is_null(args.sims):
sims = []
else:
sims = args.sims
if is_null(args.alias_dict):
aliases = None
else:
aliases = load_yaml(args.alias_dict)
name_keys = dict(target_type=args.ttype,
fullpath=True)
config_file = NAME_FACTORY.ttypeconfig(**name_keys)
if is_not_null(args.config):
config_file = args.config
roster_dict = {}
for roster in args.rosters:
a_roster = load_yaml(roster)
roster_dict.update(a_roster)
base_config = load_yaml(config_file)
self._write_target_dirs(args.ttype, roster_dict, base_config,
sims, args.spatial_models, aliases)
def resolve_targetfile(self, args, require_sim_name=False): # x
"""Get the name of the targetfile based on the job arguments"""
ttype = args.get('ttype')
if is_null(ttype):
sys.stderr.write('Target type must be specified')
return (None, None)
sim = args.get('sim')
if is_null(sim):
if require_sim_name:
sys.stderr.write('Simulation scenario must be specified')
return (None, None)
else:
sim = None
name_keys = dict(target_type=ttype,
targetlist='target_list.yaml',
sim_name=sim,
fullpath=True)
if sim is None:
targetfile = self.targetfile(**name_keys)
else:
targetfile = self.sim_targetfile(**name_keys)
targets_override = args.get('targetfile')
if is_not_null(targets_override):
targetfile = targets_override
return (targetfile, sim)
def resolve_targetfile(self, args, require_sim_name=False): # x
"""Get the name of the targetfile based on the job arguments"""
ttype = args.get('ttype')
if is_null(ttype):
sys.stderr.write('Target type must be specified')
return (None, None)
sim = args.get('sim')
if is_null(sim):
if require_sim_name:
sys.stderr.write('Simulation scenario must be specified')
return (None, None)
else:
sim = None
name_keys = dict(target_type=ttype,
targetlist='target_list.yaml',
sim_name=sim,
fullpath=True)
if sim is None:
targetfile = self.targetfile(**name_keys)
else:
targetfile = self.sim_targetfile(**name_keys)
targets_override = args.get('targetfile')
if is_not_null(targets_override):
targetfile = targets_override
return (targetfile, sim)
def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
ttype = args['ttype']
(targets_yaml, sim) = NAME_FACTORY.resolve_targetfile(args)
if sim is not None:
raise ValueError("Found 'sim' argument on AnalyzeROI_SG config.")
if targets_yaml is None:
return job_configs
config_yaml = 'config.yaml'
config_override = args.get('config')
if is_not_null(config_override):
config_yaml = config_override
targets = load_yaml(targets_yaml)
base_config = dict(roi_baseline=args['roi_baseline'],
make_plots=args['make_plots'])
for target_name in targets.keys():
name_keys = dict(target_type=ttype,
target_name=target_name,
fullpath=True)
target_dir = NAME_FACTORY.targetdir(**name_keys)
config_path = os.path.join(target_dir, config_yaml)
logfile = make_nfs_path(
os.path.join(target_dir,
"%s_%s.log" % (self.linkname, target_name)))
job_config = base_config.copy()
job_config.update(dict(config=config_path, logfile=logfile))
job_configs[target_name] = job_config
return job_configs
def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
if args.ttype is not None:
name_keys = dict(target_type=args.ttype, fullpath=True)
config_file = NAME_FACTORY.ttypeconfig(**name_keys)
spec_config = NAME_FACTORY.specconfig(**name_keys)
spec_file = NAME_FACTORY.specfile(**name_keys)
else:
config_file = None
spec_config = None
spec_file = None
if is_not_null(args.config):
config_file = args.config
if is_not_null(args.specconfig):
spec_config = args.specconfig
if is_not_null(args.specfile):
spec_file = args.specfile
if config_file is None:
sys.stderr.write('No input configuration file is specified')
return -1
if spec_config is None:
sys.stderr.write('No input spectra configurate file is specified')
return -1
if spec_file is None:
sys.stderr.write('No output spectra file is specified')
return -1
spec_config = load_yaml(spec_config)
channels = spec_config['channels']
if isinstance(spec_config['masses'], dict):
masses = np.logspace(np.log10(spec_config['masses']['mass_min']),
np.log10(spec_config['masses']['mass_max']),
spec_config['masses']['mass_nstep'])
elif isinstance(spec_config['masses'], list):
masses = spec_config['masses']
dm_spec_table = DMSpecTable.create_from_config(config_file, channels,
masses)
dm_spec_table.write_fits(spec_file, args.clobber)
return 0
def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
if args.ttype is None:
raise RuntimeError('Target type must be specified')
name_keys = dict(target_type=args.ttype,
rosterlist='roster_list.yaml',
sim_name=args.sim,
fullpath=True)
spec_config = NAME_FACTORY.specconfig(**name_keys)
if is_not_null(args.specconfig):
spec_config = args.specconfig
spec_config = load_yaml(spec_config)
channels = spec_config['channels']
if is_not_null(args.sim):
roster_file = NAME_FACTORY.sim_rosterfile(**name_keys)
sim_name = args.sim
is_sim = True
else:
roster_file = NAME_FACTORY.rosterfile(**name_keys)
is_sim = False
sim_name = None
if is_not_null(args.rosterlist):
roster_file = args.rosterlist
roster_dict = load_yaml(roster_file)
if is_sim:
seedlist = list(range(args.seed, args.seed + args.nsims))
else:
seedlist = [0]
astro_prior = args.astro_prior
if is_null(astro_prior):
astro_prior = 'none'
for seed in seedlist:
StackLikelihood.stack_rosters(roster_dict, args.ttype, channels,
astro_prior, sim_name, seed,
args.clobber)
def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
ttype = args['ttype']
(targets_yaml, sim) = NAME_FACTORY.resolve_targetfile(args)
if targets_yaml is None:
return job_configs
config_yaml = 'config.yaml'
config_override = args.get('config')
if is_not_null(config_override):
config_yaml = config_override
targets = load_yaml(targets_yaml)
nsims_job = args['nsims_job']
first_seed = args['seed']
nsims = args['nsims']
last_seed = first_seed + nsims
base_config = dict(sim_profile=args['sim_profile'],
roi_baseline=args['roi_baseline'],
non_null_src=args['non_null_src'],
do_find_src=args['do_find_src'],
sim=sim)
for target_name, target_list in targets.items():
name_keys = dict(target_type=ttype,
target_name=target_name,
sim_name=sim,
fullpath=True)
simdir = NAME_FACTORY.sim_targetdir(**name_keys)
config_path = os.path.join(simdir, config_yaml)
job_config = base_config.copy()
job_config.update(dict(config=config_path, profiles=target_list))
current_seed = first_seed
while current_seed < last_seed:
fullkey = "%s_%06i" % (target_name, current_seed)
logfile = make_nfs_path(
os.path.join(
simdir, "%s_%s_%06i.log" %
(self.linkname, target_name, current_seed)))
if nsims_job <= 0 or current_seed + nsims_job >= last_seed:
nsims_current = last_seed - current_seed
else:
nsims_current = nsims_job
job_config.update(
dict(seed=current_seed,
nsims=nsims_current,
logfile=logfile))
job_configs[fullkey] = job_config.copy()
current_seed += nsims_current
return job_configs
def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
limit_col = 'ul_0.95'
ylims = (1e-2, 1e+2)
if is_not_null(args.infile):
tab_m = Table.read(args.infile, hdu="indices")
tab_s = Table.read(args.infile, hdu=args.spec)
xvals = tab_m['indices'][0]
yvals = tab_s[limit_col][0]
ldict = dict(limits=(xvals, yvals))
else:
ldict = {}
if is_not_null(args.bands):
tab_b = Table.read(args.bands, hdu=args.spec)
tab_bm = Table.read(args.bands, hdu="indices")
bands = get_ul_bands(tab_b, 'mles')
bands['indices'] = tab_bm['indices'][0]
else:
bands = None
if is_not_null(args.sim):
sim_srcs = load_yaml(args.sim)
injected_src = sim_srcs.get('injected_source', None)
else:
injected_src = None
xlims = (1e1, 1e4)
stack_plot = plot_limits_from_arrays(ldict, xlims, ylims, bands)
if injected_src is not None:
mc_model = injected_src['source_model']
plot_mc_truth(stack_plot[1], mc_model)
if args.outfile:
stack_plot[0].savefig(args.outfile)
return None
return stack_plot
def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
ttype = args['ttype']
(targets_yaml, sim) = NAME_FACTORY.resolve_targetfile(args)
if targets_yaml is None:
return job_configs
config_yaml = 'config.yaml'
config_override = args.get('config')
if is_not_null(config_override):
config_yaml = config_override
targets = load_yaml(targets_yaml)
nsims_job = args['nsims_job']
first_seed = args['seed']
nsims = args['nsims']
last_seed = first_seed + nsims
base_config = dict(sim_profile=args['sim_profile'],
roi_baseline=args['roi_baseline'],
non_null_src=args['non_null_src'],
sim=sim)
for target_name, target_list in targets.items():
name_keys = dict(target_type=ttype,
target_name=target_name,
sim_name=sim,
fullpath=True)
simdir = NAME_FACTORY.sim_targetdir(**name_keys)
config_path = os.path.join(simdir, config_yaml)
job_config = base_config.copy()
job_config.update(dict(config=config_path,
profiles=target_list))
current_seed = first_seed
while current_seed < last_seed:
fullkey = "%s_%06i" % (target_name, current_seed)
logfile = make_nfs_path(os.path.join(simdir, "%s_%s_%06i.log" % (self.linkname,
target_name, current_seed)))
if nsims_job <= 0 or current_seed + nsims_job >= last_seed:
nsims_current = last_seed - current_seed
else:
nsims_current = nsims_job
job_config.update(dict(seed=current_seed,
nsims=nsims_current,
logfile=logfile))
job_configs[fullkey] = job_config.copy()
current_seed += nsims_current
return job_configs
def write_stacked(ttype, roster_name, stacked_dict, astro_prior_key, sim,
seed, clobber): #pylint: disable=too-many-arguments
""" Write the stacked DMCastroData object to a FITS file
Parameters
----------
ttype : str
Type of target, used for bookkeeping and file names
roster_name : str
Name of the roster, used for bookkeeping and file names
stacked_dict : dict
Dictionary of `DMCastroData` objects, keyed by channel
astro_prior_key : str
String that identifies the type of prior on the J-factor
sim : str
String that specifies the simulation scenario
seed : int or None
Key for the simulation instance, used for bookkeeping and file names
clobber : bool
Flag to overwrite existing files.
"""
name_keys = dict(target_type=ttype,
target_name="stacked",
fullpath=True,
roster_name=roster_name,
sim_name=sim,
seed="%06i" % seed,
astro_prior=astro_prior_key)
if is_not_null(sim):
outdir = NAME_FACTORY.sim_targetdir(**name_keys)
outpath = NAME_FACTORY.sim_resultsfile(**name_keys)
else:
outdir = NAME_FACTORY.targetdir(**name_keys)
outpath = NAME_FACTORY.resultsfile(**name_keys)
try:
os.makedirs(outdir)
except OSError:
pass
limitfile = outpath.replace('results', 'limits')
print("Writing stacked results %s" % outpath)
StackLikelihood.write_fits_files(stacked_dict, outpath, limitfile,
clobber)
def resolve_randconfig(self, args):
"""Get the name of the specturm file based on the job arguments"""
ttype = args.get('ttype')
if is_null(ttype):
sys.stderr.write('Target type must be specified')
return None
name_keys = dict(target_type=ttype, fullpath=True)
randconfig = self.randconfig(**name_keys)
rand_override = args.get('rand_config')
if is_not_null(rand_override):
randconfig = rand_override
return randconfig
def resolve_specfile(self, args):
"""Get the name of the specturm file based on the job arguments"""
ttype = args.get('ttype')
if is_null(ttype):
sys.stderr.write('Target type must be specified')
return None
name_keys = dict(target_type=ttype, fullpath=True)
specfile = self.specfile(**name_keys)
spec_override = args.get('specfile')
if is_not_null(spec_override):
specfile = spec_override
return specfile
def resolve_randconfig(self, args):
"""Get the name of the specturm file based on the job arguments"""
ttype = args.get('ttype')
if is_null(ttype):
sys.stderr.write('Target type must be specified')
return None
name_keys = dict(target_type=ttype,
fullpath=True)
randconfig = self.randconfig(**name_keys)
rand_override = args.get('rand_config')
if is_not_null(rand_override):
randconfig = rand_override
return randconfig
def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
if args.ttype is not None:
name_keys = dict(target_type=args.ttype, fullpath=True)
config_file = NAME_FACTORY.ttypeconfig(**name_keys)
spec_config = NAME_FACTORY.specconfig(**name_keys)
spec_file = NAME_FACTORY.specfile(**name_keys)
else:
config_file = None
spec_config = None
spec_file = None
if is_not_null(args.config):
config_file = args.config
if is_not_null(args.specconfig):
spec_config = args.specconfig
if is_not_null(args.specfile):
spec_file = args.specfile
if config_file is None:
sys.stderr.write('No input configuration file is specified')
return -1
if spec_config is None:
sys.stderr.write('No input spectra configurate file is specified')
return -1
if spec_file is None:
sys.stderr.write('No output spectra file is specified')
return -1
stack_spec_table = StackSpecTable.create_from_config(
config_file, spec_file)
stack_spec_table.write_fits(spec_file, args.clobber)
return 0
def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
limitfile = args.limitfile
first = args.seed
last = first + args.nsims
flist = [limitfile.replace("_SEED.fits", "_%06i.fits" % seed)\
for seed in range(first, last)]
spec_config = load_yaml(args.specconfig)
channels = spec_config['channels']
sum_chans = CollectLimits.select_channels(channels, limitfile)
outfile = args.outfile
summaryfile = args.summaryfile
hdus = sum_chans + ['MASSES']
out_tables, out_names = vstack_tables(flist, hdus)
if is_not_null(outfile):
fits_utils.write_tables_to_fits(outfile,
out_tables,
namelist=out_names)
if is_not_null(summaryfile):
summary_tables = []
for ot, chan in zip(out_tables[0:-1], sum_chans):
decay = chan.find('_decay') >= 0
summary_table = summarize_limits_results(ot, decay)
summary_tables.append(summary_table)
summary_tables.append(Table(out_tables[-1][0]))
fits_utils.write_tables_to_fits(summaryfile,
summary_tables,
namelist=out_names)
def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
ttype = args['ttype']
(targets_yaml, sim) = NAME_FACTORY.resolve_targetfile(args)
if sim is not None:
raise ValueError("Found 'sim' argument on AnalyzeSED_SG config.")
if targets_yaml is None:
return job_configs
targets = load_yaml(targets_yaml)
config_yaml = 'config.yaml'
if is_not_null(args['skydirs']):
skydirs = args['skydirs']
else:
skydirs = None
base_config = dict(roi_baseline=args['roi_baseline'],
make_plots=args['make_plots'],
non_null_src=args['non_null_src'])
for target_name, target_list in targets.items():
name_keys = dict(target_type=ttype,
target_name=target_name,
sim_name='random',
fullpath=True)
if skydirs is None:
target_dir = NAME_FACTORY.targetdir(**name_keys)
skydir_path = None
else:
target_dir = NAME_FACTORY.sim_targetdir(**name_keys)
skydir_path = os.path.join(target_dir, skydirs)
config_path = os.path.join(target_dir, config_yaml)
logfile = make_nfs_path(
os.path.join(target_dir,
"%s_%s.log" % (self.linkname, target_name)))
job_config = base_config.copy()
job_config.update(
dict(config=config_path,
profiles=target_list,
skydirs=skydir_path,
logfile=logfile))
job_configs[target_name] = job_config
return job_configs
def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
astro_priors = args['astro_priors']
clobber = args['clobber']
sim = args['sim']
if is_not_null(sim):
is_sim = True
nsims = args['nsims']
seed = args['seed']
else:
is_sim = False
nsims = -1
seed = -1
base_config = dict(ttype=args['ttype'],
specconfig=args['specconfig'],
rosterlist=args['rosterlist'],
sim=sim,
nsims=nsims,
seed=seed,
clobber=clobber)
for astro_prior in astro_priors:
name_keys = dict(target_type=args['ttype'],
target_name='stacked',
astro_prior=astro_prior,
sim_name=sim,
fullpath=True)
if is_sim:
target_dir = NAME_FACTORY.sim_targetdir(**name_keys)
full_key = "%s:%s" % (astro_prior, sim)
else:
target_dir = NAME_FACTORY.targetdir(**name_keys)
full_key = astro_prior
logfile = os.path.join(target_dir, 'stack_%s.log' % astro_prior)
job_config = base_config.copy()
job_config.update(dict(astro_prior=astro_prior, logfile=logfile))
job_configs[full_key] = job_config
return job_configs
def update_args(self, override_args):
"""Update the argument used to invoke the application
Note that this will also update the dictionary of input and output files.
Parameters
-----------
override_args : dict
Dictionary of arguments to override the current values
"""
self.args = extract_arguments(override_args, self.args)
self._latch_file_info()
scratch_dir = self.args.get('scratch', None)
if is_not_null(scratch_dir):
self._file_stage = FileStageManager(scratch_dir, '.')
def update_args(self, override_args):
"""Update the argument used to invoke the application
Note that this will also update the dictionary of input and output files.
Parameters
-----------
override_args : dict
Dictionary of arguments to override the current values
"""
self.args = extract_arguments(override_args, self.args)
self._latch_file_info()
scratch_dir = self.args.get('scratch', None)
if is_not_null(scratch_dir):
self._file_stage = FileStageManager(scratch_dir, '.')
def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
if is_null(args.config):
raise ValueError("Config yaml file must be specified")
if is_null(args.rand_config):
raise ValueError(
"Random direction config yaml file must be specified")
config = load_yaml(args.config)
rand_config = load_yaml(args.rand_config)
wcsgeom = self._make_wcsgeom_from_config(config)
dir_dict = self._build_skydir_dict(wcsgeom, rand_config)
if is_not_null(args.outfile):
write_yaml(dir_dict, args.outfile)
def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
ttype = args['ttype']
(targets_yaml, sim) = NAME_FACTORY.resolve_targetfile(args)
if sim is not None:
raise ValueError("Found 'sim' argument on AnalyzeSED_SG config.")
if targets_yaml is None:
return job_configs
targets = load_yaml(targets_yaml)
config_yaml = 'config.yaml'
if is_not_null(args['skydirs']):
skydirs = args['skydirs']
else:
skydirs = None
base_config = dict(roi_baseline=args['roi_baseline'],
make_plots=args['make_plots'],
non_null_src=args['non_null_src'])
for target_name, target_list in targets.items():
name_keys = dict(target_type=ttype,
target_name=target_name,
sim_name='random',
fullpath=True)
if skydirs is None:
target_dir = NAME_FACTORY.targetdir(**name_keys)
skydir_path = None
else:
target_dir = NAME_FACTORY.sim_targetdir(**name_keys)
skydir_path = os.path.join(target_dir, skydirs)
config_path = os.path.join(target_dir, config_yaml)
logfile = make_nfs_path(os.path.join(
target_dir, "%s_%s.log" % (self.linkname, target_name)))
job_config = base_config.copy()
job_config.update(dict(config=config_path,
profiles=target_list,
skydirs=skydir_path,
logfile=logfile))
job_configs[target_name] = job_config
return job_configs
def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
if is_null(args.config):
raise ValueError("Config yaml file must be specified")
if is_null(args.rand_config):
raise ValueError(
"Random direction config yaml file must be specified")
config = load_yaml(args.config)
rand_config = load_yaml(args.rand_config)
wcsgeom = self._make_wcsgeom_from_config(config)
dir_dict = self._build_skydir_dict(wcsgeom, rand_config)
if is_not_null(args.outfile):
write_yaml(dir_dict, args.outfile)
def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
if not HAVE_ST:
raise RuntimeError(
"Trying to run fermipy analysis, but don't have ST")
if is_not_null(args.roi_baseline):
gta = GTAnalysis.create(args.roi_baseline, args.config)
else:
gta = GTAnalysis(args.config,
logging={'verbosity': 3},
fileio={'workdir_regex': '\.xml$|\.npy$'})
gta.print_roi()
test_source = args.target
gta.sed(test_source, outfile='sed_%s.fits' % 'FL8Y', make_plots=True)
gta.extension(test_source, make_plots=True)
return gta
def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
if DMSKY_ROSTER_LIB:
roster_lib = RosterLibrary()
roster_dict = {}
else:
raise RuntimeError(
"Can't load roster library, probably b/c old version of yaml is not compatible with dmsky"
)
if not args.rosters:
raise RuntimeError("You must specify at least one target roster")
if is_null(args.ttype):
raise RuntimeError("You must specify a target type")
if is_null(args.sims):
sims = []
else:
sims = args.sims
if is_null(args.alias_dict):
aliases = None
else:
aliases = load_yaml(args.alias_dict)
name_keys = dict(target_type=args.ttype, fullpath=True)
config_file = NAME_FACTORY.ttypeconfig(**name_keys)
if is_not_null(args.config):
config_file = args.config
for roster in args.rosters:
rost = roster_lib.create_roster(roster)
roster_dict[roster] = rost
base_config = load_yaml(config_file)
self._write_target_dirs(args.ttype, roster_dict, base_config, sims,
args.spatial_models, aliases)
def update_args(self, override_args):
"""Update the argument used to invoke the application
Note that this will also update the dictionary of input
and output files.
Parameters
-----------
override_args : dict
dictionary passed to the links
"""
self.args = extract_arguments(override_args, self.args)
self._map_arguments(self.args)
scratch_dir = self.args.get('scratch', None)
if is_not_null(scratch_dir):
self._file_stage = FileStageManager(scratch_dir, '.')
for link in self._links.values():
link._set_file_stage(self._file_stage)
self._latch_file_info()
def update_args(self, override_args):
"""Update the argument used to invoke the application
Note that this will also update the dictionary of input
and output files.
Parameters
-----------
override_args : dict
dictionary passed to the links
"""
self.args = extract_arguments(override_args, self.args)
self._map_arguments(self.args)
scratch_dir = self.args.get('scratch', None)
if is_not_null(scratch_dir):
self._file_stage = FileStageManager(scratch_dir, '.')
for link in self._links.values():
link._set_file_stage(self._file_stage)
self._latch_file_info()
def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
ttype = args['ttype']
(targets_yaml, sim) = NAME_FACTORY.resolve_targetfile(args)
if targets_yaml is None:
return job_configs
config_yaml = 'config.yaml'
config_override = args.get('config')
if is_not_null(config_override):
config_yaml = config_override
targets = load_yaml(targets_yaml)
base_config = dict(sim_profile=args['sim_profile'],
roi_baseline=args['roi_baseline'],
sim=sim,
nsims=args['nsims'],
seed=args['seed'])
for target_name, target_list in targets.items():
name_keys = dict(target_type=ttype,
target_name=target_name,
sim_name=sim,
fullpath=True)
simdir = NAME_FACTORY.sim_targetdir(**name_keys)
config_path = os.path.join(simdir, config_yaml)
logfile = make_nfs_path(
os.path.join(simdir,
"%s_%s.log" % (self.linkname, target_name)))
job_config = base_config.copy()
job_config.update(
dict(config=config_path, logfile=logfile,
profiles=target_list))
job_configs[target_name] = job_config
return job_configs
def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
ttype = args['ttype']
(targets_yaml, sim) = NAME_FACTORY.resolve_targetfile(args)
if targets_yaml is None:
return job_configs
config_yaml = 'config.yaml'
config_override = args.get('config')
if is_not_null(config_override):
config_yaml = config_override
rand_yaml = NAME_FACTORY.resolve_randconfig(args)
targets = load_yaml(targets_yaml)
base_config = dict(rand_config=rand_yaml)
for target_name in targets.keys():
name_keys = dict(target_type=ttype,
target_name=target_name,
sim_name=sim,
fullpath=True)
simdir = NAME_FACTORY.sim_targetdir(**name_keys)
config_path = os.path.join(simdir, config_yaml)
outfile = os.path.join(simdir, 'skydirs.yaml')
logfile = make_nfs_path(outfile.replace('yaml', 'log'))
job_config = base_config.copy()
job_config.update(dict(config=config_path,
outfile=outfile,
logfile=logfile))
job_configs[target_name] = job_config
return job_configs
def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
ttype = args['ttype']
(targets_yaml, sim) = NAME_FACTORY.resolve_targetfile(args)
if targets_yaml is None:
return job_configs
config_yaml = 'config.yaml'
config_override = args.get('config')
if is_not_null(config_override):
config_yaml = config_override
rand_yaml = NAME_FACTORY.resolve_randconfig(args)
targets = load_yaml(targets_yaml)
base_config = dict(rand_config=rand_yaml)
for target_name in targets.keys():
name_keys = dict(target_type=ttype,
target_name=target_name,
sim_name=sim,
fullpath=True)
simdir = NAME_FACTORY.sim_targetdir(**name_keys)
config_path = os.path.join(simdir, config_yaml)
outfile = os.path.join(simdir, 'skydirs.yaml')
logfile = make_nfs_path(outfile.replace('yaml', 'log'))
job_config = base_config.copy()
job_config.update(
dict(config=config_path, outfile=outfile, logfile=logfile))
job_configs[target_name] = job_config
return job_configs
def stack_roster(rost, ttype, channels, astro_prior_key, sim, seed): #pylint: disable=too-many-arguments
""" Stack all of the DMCastroData in a roster
Parameters
----------
rost : list
List of the targets
ttype : str
Type of target, used for bookkeeping and file names
channels : list
List of the channels to convert
j_prior_key : str
String that identifies the type of prior on the J-factor
sim : str
String that specifies the simulation scenario
seed : int or None
Key for the simulation instance, used for bookkeeping and file names
Returns
-------
output : dict
Dictionary of `DMCastroData` objects, keyed by channel
"""
component_dict = {}
out_dict = {}
for chan in channels:
component_dict[chan] = []
for target_key in rost:
tokens = target_key.split(':')
name_keys = dict(target_type=ttype,
target_name=tokens[0],
profile=tokens[1],
fullpath=True,
sim_name=sim,
seed="%06i" % seed,
astro_prior=astro_prior_key)
if is_not_null(sim):
dmlike_path = NAME_FACTORY.sim_dmlikefile(**name_keys)
else:
dmlike_path = NAME_FACTORY.dmlikefile(**name_keys)
tab_m = Table.read(dmlike_path, hdu="MASSES")
for chan in channels:
if chan.find('_decay') >= 0:
decay = True
norm_type = 'tau'
else:
decay = False
norm_type = 'sigmav'
try:
tab_s = Table.read(dmlike_path, hdu=chan)
except KeyError:
continue
dm_castro = DMCastroData.create_from_tables(
tab_s, tab_m, chan, norm_type, decay)
component_dict[chan].append(dm_castro)
for chan, comps in list(component_dict.items()):
if not comps:
continue
decay = chan.find('_decay') >= 0
stacked = DMCastroData.create_from_stack(comps, decay=decay)
out_dict[chan] = stacked
return out_dict
def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
if not HAVE_ST:
raise RuntimeError(
"Trying to run fermipy analysis, but don't have ST")
if args.load_baseline:
gta = GTAnalysis.create(args.roi_baseline, args.config)
else:
gta = GTAnalysis(args.config,
logging={'verbosity': 3},
fileio={'workdir_regex': '\.xml$|\.npy$'})
gta.setup()
if is_not_null(args.input_pars):
gta.load_parameters_from_yaml(args.input_pars)
gta.write_roi(args.roi_baseline,
save_model_map=True,
save_weight_map=True,
make_plots=args.make_plots)
src_list = get_src_names(gta)
plotter = plotting.AnalysisPlotter(gta.config['plotting'],
fileio=gta.config['fileio'],
logging=gta.config['logging'])
if is_null(args.fit_strategy):
return
fit_strategy = load_yaml(args.fit_strategy)
npred_current = None
npred_prev = None
plots_only = False
for fit_stage in fit_strategy:
mask = fit_stage.get('mask', None)
npred_threshold = fit_stage.get('npred_threshold', 1.0e4)
frac_threshold = fit_stage.get('frac_threshold', 0.5)
npred_frac = fit_stage.get('npred_frac', 0.9999)
if plots_only:
gta.load_roi("%s.npy" % fit_stage['key'])
npred_current = set_wts_get_npred_wt(gta, mask)
skip_list_region = get_unchanged(src_list,
npred_current,
npred_prev,
frac_threshold=frac_threshold)
else:
npred_current = set_wts_get_npred_wt(gta, mask)
skip_list_region = get_unchanged(src_list,
npred_current,
npred_prev,
frac_threshold=frac_threshold)
gta.optimize(npred_frac=npred_frac,
npred_threshold=npred_threshold,
skip=skip_list_region)
snapshot(gta,
plotter,
fit_stage['key'],
make_plots=args.make_plots)
npred_prev = npred_current
npred_current = build_srcdict(gta, 'npred_wt')
def compute_castro_data(castro_data, spec_vals, xscan_vals, **kwargs):
""" Convert CastroData object, i.e., Likelihood as a function of
flux and energy flux, to a StackCastroData object, i.e., Likelihood as
a function of stacking normalization and index
Parameters
----------
castro_data : `CastroData`
Input data
spec_vals : `numpy.array`
Input spectra
xscan_vals : `numpy.array`
Values for scan variables
Returns
-------
output : `StackCastroData`
The stacking-space likelihood curves
"""
ref_stack = kwargs.get('ref_stack', 1.)
norm_factor = kwargs.pop('norm_factor', 1.)
spec_name = kwargs.pop('spec_name')
astro_prior = kwargs.pop('astro_prior')
n_xval = len(xscan_vals)
n_yval = kwargs.get('nystep', 200)
norm_limits = castro_data.getLimits(1e-5)
# This puts the spectrum in units of the reference spectrum
# This means that the scan values will be expressed
# In units of the reference spectra as well
spec_vals /= norm_factor
norm_vals = np.ndarray((n_xval, n_yval))
dll_vals = np.ndarray((n_xval, n_yval))
mle_vals = np.ndarray((n_xval))
nll_offsets = np.ndarray((n_xval))
scan_mask = np.ones((n_xval), bool)
# for i, index in enumerate(xscan_vals):
for i in range(n_xval):
max_ratio = 1. / ((spec_vals[i] / norm_limits).max())
log_max_ratio = np.log10(max_ratio)
norm_vals[i][0] = 10**(log_max_ratio - 5)
norm_vals[i][1:] = np.logspace(log_max_ratio - 4,
log_max_ratio + 4, n_yval - 1)
test_vals = (np.expand_dims(spec_vals[i], 1) *
(np.expand_dims(norm_vals[i], 1).T))
dll_vals[i, 0:] = castro_data(test_vals)
mle_vals[i] = norm_vals[i][dll_vals[i].argmin()]
nll_offsets[i] = dll_vals[i].min()
dll_vals[i] -= nll_offsets[i]
msk = np.isfinite(dll_vals[i])
if not msk.any():
print("Skipping scan value %0.2e for spec %s" %
(xscan_vals[i], spec_name))
scan_mask[i] = False
continue
if is_not_null(astro_prior):
try:
lnlfn = castro.LnLFn(norm_vals[i], dll_vals[i], 'dummy')
lnlfn_prior = LnLFn_norm_prior(lnlfn, astro_prior)
dll_vals[i, 0:] = lnlfn_prior(norm_vals[i])
nll_offsets[i] = dll_vals[i].min()
except ValueError:
print("Skipping index %0.2e for spec %s" %
(xscan_vals[i], spec_name))
scan_mask[i] = False
dll_vals[i, 0:] = np.nan * np.ones((n_yval))
nll_offsets[i] = np.nan
kwcopy = kwargs.copy()
kwcopy['xscan_vals'] = xscan_vals[scan_mask]
kwcopy['astro_prior'] = astro_prior
# Here we convert the normalization values to standard units
norm_vals *= ref_stack
stack_castro = StackCastroData(norm_vals[scan_mask],
dll_vals[scan_mask],
nll_offsets[scan_mask], **kwcopy)
return stack_castro
def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
if not HAVE_ST:
raise RuntimeError(
"Trying to run fermipy analysis, but don't have ST")
if args.load_baseline:
gta = GTAnalysis.create(args.roi_baseline,
args.config)
else:
gta = GTAnalysis(args.config,
logging={'verbosity': 3},
fileio={'workdir_regex': '\.xml$|\.npy$'})
gta.setup()
if is_not_null(args.input_pars):
gta.load_parameters_from_yaml(args.input_pars)
gta.write_roi(args.roi_baseline,
save_model_map=True,
save_weight_map=True,
make_plots=args.make_plots)
src_list = get_src_names(gta)
plotter = plotting.AnalysisPlotter(gta.config['plotting'],
fileio=gta.config['fileio'],
logging=gta.config['logging'])
if is_null(args.fit_strategy):
return
fit_strategy = load_yaml(args.fit_strategy)
npred_current = None
npred_prev = None
plots_only = False
for fit_stage in fit_strategy:
mask = fit_stage.get('mask', None)
npred_threshold = fit_stage.get('npred_threshold', 1.0e4)
frac_threshold = fit_stage.get('frac_threshold', 0.5)
npred_frac = fit_stage.get('npred_frac', 0.9999)
if plots_only:
gta.load_roi("%s.npy" % fit_stage['key'])
npred_current = set_wts_get_npred_wt(gta, mask)
skip_list_region = get_unchanged(src_list,
npred_current,
npred_prev,
frac_threshold=frac_threshold)
else:
npred_current = set_wts_get_npred_wt(gta, mask)
skip_list_region = get_unchanged(src_list,
npred_current,
npred_prev,
frac_threshold=frac_threshold)
gta.optimize(npred_frac=npred_frac,
npred_threshold=npred_threshold,
skip=skip_list_region)
snapshot(gta, plotter, fit_stage['key'], make_plots=args.make_plots)
npred_prev = npred_current
npred_current = build_srcdict(gta, 'npred_wt')
def convert_sed(spec_table, channels, sed_file, outfile, limitfile,
**kwargs):
"""Convert a single SED to DM space.
Parameters
----------
spec_table : `DMSpecTable`
Object with all the DM spectra
channels : list
List of the channels to convert
sed_file : str
Path to the SED file
outfile : str
Path to write the output `DMCastroData` object to
limitfile : str
Path to write the output limits to.
Keyword arguments
-----------------
norm_type : str
Normalization type to use
j_factor : dict
Dictionary with information about the J-factor
d_factor : dict
Dictionary with information about the J-factor
clobber : bool
Flag to overwrite existing files.
"""
norm_type = kwargs.get('norm_type', 'eflux')
j_factor = kwargs.get('j_factor', None)
d_factor = kwargs.get('d_factor', None)
clobber = kwargs.get('clobber', False)
use_chans = ConvertCastro.select_channels(channels, sed_file)
exttype = os.path.splitext(sed_file)[-1]
if exttype in ['.fits', '.npy']:
sed = CastroData.create_from_sedfile(sed_file, norm_type)
elif exttype in ['.yaml']:
sed = CastroData.create_from_yamlfile(sed_file)
else:
raise ValueError("Can not read file type %s for SED" % exttype)
c_list, t_list, n_list = ConvertCastro.convert_sed_to_dm(
spec_table,
sed,
use_chans,
norm_type=norm_type,
j_val=j_factor,
d_val=d_factor)
if is_not_null(outfile):
fits_utils.write_tables_to_fits(outfile,
t_list,
clobber=clobber,
namelist=n_list)
if is_not_null(limitfile):
mass_table = t_list[-1]
_, t_list_lim, n_list_lim = ConvertCastro.extract_dm_limits(
c_list, use_chans, [0.68, 0.95], mass_table)
fits_utils.write_tables_to_fits(limitfile,
t_list_lim,
clobber=clobber,
namelist=n_list_lim)
def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
norm_type = 'eflux'
spec_table = DMSpecTable.create_from_fits(args.specfile)
profile = load_yaml(args.astro_value_file)
channels = spec_table.channel_names
j_factor = None
d_factor = None
j_value = profile.get('j_integ', None)
j_sigma = profile.get('j_sigma', None)
if is_null(j_value):
j_factor = None
elif is_null(args.astro_prior) or is_null(j_sigma) or j_sigma == 0.0:
j_factor = j_value
else:
j_factor = dict(functype=args.astro_prior,
j_value=j_value,
mu=j_value,
sigma=j_sigma)
d_value = profile.get('d_integ', None)
d_sigma = profile.get('d_sigma', None)
if is_null(d_value):
d_factor = None
elif is_null(args.astro_prior) or is_null(d_sigma) or d_sigma == 0.0:
d_factor = d_value
else:
d_factor = dict(functype=args.astro_prior,
d_value=d_value,
mu=d_value,
sigma=d_sigma)
if args.nsims < 0:
seedlist = [None]
else:
seedlist = list(range(args.seed, args.seed + args.nsims))
for seed in seedlist:
sedfile = args.sed_file
outfile = args.outfile
limitfile = args.limitfile
if seed is not None:
sedfile = sedfile.replace('_SEED.fits', '_%06i.fits' % seed)
if is_not_null(outfile):
outfile = outfile.replace('_SEED.fits',
'_%06i.fits' % seed)
if is_not_null(limitfile):
limitfile = limitfile.replace('_SEED.fits',
'_%06i.fits' % seed)
self.convert_sed(spec_table,
channels,
sedfile,
outfile,
limitfile,
norm_type=norm_type,
j_factor=j_factor,
d_factor=d_factor,
clober=args.clobber)
def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
ttype = args['ttype']
(targets_yaml, sim) = NAME_FACTORY.resolve_targetfile(args)
if targets_yaml is None:
return job_configs
specfile = NAME_FACTORY.resolve_specfile(args)
targets = load_yaml(targets_yaml)
astro_priors = args['astro_priors']
clobber = args['clobber']
if is_not_null(sim):
is_sim = True
nsims = args['nsims']
seed = args['seed']
else:
is_sim = False
nsims = -1
seed = -1
base_config = dict(specfile=specfile,
nsims=nsims,
seed=seed,
clobber=clobber)
for target_name, profile_list in list(targets.items()):
for profile in profile_list:
for astro_prior in astro_priors:
full_key = "%s:%s:%s" % (target_name, profile, astro_prior)
target_version = profile.split('_')[0]
name_keys = dict(target_type=ttype,
target_name=target_name,
target_version=target_version,
profile=profile,
astro_prior=astro_prior,
fullpath=True)
if is_sim:
name_keys['sim_name'] = sim
sed_file = NAME_FACTORY.sim_sedfile(**name_keys)
astro_value_yaml = NAME_FACTORY.sim_astro_valuefile(
**name_keys)
outfile = NAME_FACTORY.sim_dmlikefile(**name_keys)
limitfile = NAME_FACTORY.sim_dmlimitsfile(**name_keys)
full_key += ":%s" % sim
else:
sed_file = NAME_FACTORY.sedfile(**name_keys)
astro_value_yaml = NAME_FACTORY.astro_valuefile(
**name_keys)
outfile = NAME_FACTORY.dmlikefile(**name_keys)
limitfile = NAME_FACTORY.dmlimitsfile(**name_keys)
logfile = make_nfs_path(outfile.replace('.fits', '.log'))
job_config = base_config.copy()
job_config.update(
dict(sed_file=sed_file,
astro_value_file=astro_value_yaml,
astro_prior=astro_prior,
outfile=outfile,
limitfile=limitfile,
logfile=logfile))
job_configs[full_key] = job_config
return job_configs