def _write_data_target_config(cls, base_config, target, target_dir):
""" Write a fermipy configuration file for one target.
Parameters
----------
base_config : dict
Baseline configuration
target : `dict`
Specific target
target_dir : str
Directory to write to
Returns
-------
output : dict
The configuration for this specific target
"""
target_config_path = os.path.join(target_dir, 'config.yaml')
target_config = base_config.copy()
target_config['selection']['ra'] = target['ra']
target_config['selection']['dec'] = target['dec']
write_yaml(target_config, target_config_path)
return target_config
def _write_astro_value_yaml(cls, target, astro_val_path):
"""Write a yaml file describing the J-factor and D-factor of one target.
Parameters
----------
target : `dmsky.targets.Target`
Specific target
astro_val_path : str
Path for the output file
Returns
-------
output : dict
The description of the target J-factor
"""
astro_profile_data = target.profile.copy()
astro_profile_data['j_integ'] = target.j_integ
astro_profile_data['j_sigma'] = target.j_sigma
# Put the D-factors in a try-expect block
try:
astro_profile_data['d_integ'] = target.d_integ
astro_profile_data['d_sigma'] = target.d_sigma
except Exception:
sys.stdout.write(
"WARNING, could not compute D-Factors for target %s\n" %
target.name)
write_yaml(astro_profile_data, astro_val_path)
return astro_profile_data
def main():
usage = "combine_dnm.py [options]"
description = "add the pos and neg dnm components"
parser = argparse.ArgumentParser(usage=usage, description=description)
parser.add_argument('-i',
'--input',
type=str,
default=None,
help='Input file prefix')
parser.add_argument('-o',
'--output',
type=str,
default=None,
help='Output ')
parser.add_argument('-f',
'--fact',
default=False,
action='store_true',
help='Use factors')
args = parser.parse_args(sys.argv[1:])
out_dict = utils.load_yaml(args.input)
combine_dnm_maps(out_dict, args.fact)
utils.write_yaml(out_dict, args.output)
def make_srcmap_manifest(self, modelkey, components, data):
"""Build a yaml file that specfies how to make the srcmap files for a particular model
Parameters
----------
modelkey : str
Key used to identify this particular model
components : list
The binning components used in this analysis
data : str
Path to file containing dataset definition
"""
try:
model_info = self._models[modelkey]
except KeyError:
model_info = self.make_model_info(modelkey)
self._name_factory.update_base_dict(data)
outfile = os.path.join('analysis', 'model_%s' % modelkey,
'srcmap_manifest_%s.yaml' % modelkey)
manifest = model_info.make_srcmap_manifest(components,
self._name_factory)
outdir = os.path.dirname(outfile)
try:
os.makedirs(outdir)
except OSError:
pass
utils.write_yaml(manifest, outfile)
def main():
usage = "npred_txt2yaml.py [options]"
description = "Convert Gardian text files to yaml dictionary"
parser = argparse.ArgumentParser(usage=usage, description=description)
parser.add_argument('-i',
'--input',
type=str,
default=None,
help='Input file')
parser.add_argument('-o',
'--output',
type=str,
default=None,
help='Output ')
parser.add_argument('-d',
'--conv_dict',
type=str,
default=None,
help='Output ')
args = parser.parse_args(sys.argv[1:])
conv_dict = load_yaml(args.conv_dict)
npred_dict = make_npred_dict(args.input, conv_dict)
write_yaml(npred_dict, args.output)
def make_srcmap_manifest(self, modelkey, components, data):
"""Build a yaml file that specfies how to make the srcmap files for a particular model
Parameters
----------
modelkey : str
Key used to identify this particular model
components : list
The binning components used in this analysis
data : str
Path to file containing dataset definition
"""
try:
model_info = self._models[modelkey]
except KeyError:
model_info = self.make_model_info(modelkey)
self._name_factory.update_base_dict(data)
outfile = os.path.join('analysis', 'model_%s' %
modelkey, 'srcmap_manifest_%s.yaml' % modelkey)
manifest = model_info.make_srcmap_manifest(
components, self._name_factory)
outdir = os.path.dirname(outfile)
try:
os.makedirs(outdir)
except OSError:
pass
utils.write_yaml(manifest, outfile)
def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
if not HAVE_ST:
raise RuntimeError(
"Trying to run fermipy analysis, but don't have ST")
workdir = os.path.dirname(args.config)
_config_file = self._clone_config_and_srcmaps(args.config, args.seed)
gta = GTAnalysis(_config_file,
logging={'verbosity': 3},
fileio={'workdir_regex': '\.xml$|\.npy$'})
gta.load_roi(args.roi_baseline)
simfile = os.path.join(workdir,
'sim_%s_%s.yaml' % (args.sim, args.sim_profile))
mcube_file = "%s_%s_%06i" % (args.sim, args.sim_profile, args.seed)
sim_config = utils.load_yaml(simfile)
injected_source = sim_config.get('injected_source', None)
if injected_source is not None:
src_dict = injected_source['source_model']
src_dict['ra'] = gta.config['selection']['ra']
src_dict['dec'] = gta.config['selection']['dec']
injected_name = injected_source['name']
gta.add_source(injected_name, src_dict)
gta.write_model_map(mcube_file)
mc_spec_dict = dict(
true_counts=gta.model_counts_spectrum(injected_name),
energies=gta.energies,
model=src_dict)
mcspec_file = os.path.join(
workdir, "mcspec_%s_%06i.yaml" % (mcube_file, args.seed))
utils.write_yaml(mc_spec_dict, mcspec_file)
else:
injected_name = None
gta.write_roi('sim_baseline_%06i' % args.seed)
test_sources = {}
for profile in args.profiles:
profile_path = os.path.join(workdir, 'profile_%s.yaml' % profile)
test_source = load_yaml(profile_path)
test_sources[profile] = test_source
first = args.seed
last = first + args.nsims
for seed in range(first, last):
self._run_simulation(gta,
args.roi_baseline,
injected_name,
test_sources,
first,
seed,
non_null_src=args.non_null_src,
do_find_src=args.do_find_src)
def _run_simulation(gta,
roi_baseline,
injected_name,
test_sources,
current_seed,
seed,
non_null_src,
do_find_src=False):
"""Simulate a realization of this analysis"""
gta.load_roi('sim_baseline_%06i.npy' % current_seed)
gta.set_random_seed(seed)
gta.simulate_roi()
skip_srcs = []
if injected_name:
gta.zero_source(injected_name)
skip_srcs.append(injected_name)
gta.optimize(skip=skip_srcs)
if do_find_src:
gta.find_sources(sqrt_ts_threshold=5.0,
search_skydir=gta.roi.skydir,
search_minmax_radius=[1.0, np.nan])
gta.optimize(skip=skip_srcs)
gta.free_sources(skydir=gta.roi.skydir, distance=1.0, pars='norm')
if injected_name:
gta.free_source(injected_name, False)
gta.fit(covar=True)
gta.write_roi('sim_refit_%06i' % current_seed)
for pkey, test_source in test_sources.items():
test_source_name = test_source['name']
sedfile = "sed_%s_%06i.fits" % (test_source_name, seed)
correl_dict, test_src_name = add_source_get_correlated(
gta,
test_source_name,
test_source['source_model'],
correl_thresh=0.25,
non_null_src=non_null_src)
# Write the list of correlated sources
correl_yaml = os.path.join(
gta.workdir, "correl_%s_%06i.yaml" % (test_source_name, seed))
write_yaml(correl_dict, correl_yaml)
gta.free_sources(False)
for src_name in correl_dict.keys():
gta.free_source(src_name, pars='norm')
if injected_name:
gta.free_source(injected_name, False)
gta.sed(test_source_name, outfile=sedfile)
# Set things back to how they were
gta.delete_source(test_source_name)
gta.load_xml('sim_refit_%06i' % current_seed)
def main():
usage = "npred_mapSums.py [options]"
description = "Convert Gardian text files to yaml dictionary"
parser = argparse.ArgumentParser(usage=usage, description=description)
parser.add_argument('-i',
'--input',
type=str,
default=None,
help='Input file')
parser.add_argument('-e',
'--exposure',
type=str,
default=None,
help='Exposure file')
parser.add_argument('-o',
'--output',
type=str,
default=None,
help='Output ')
args = parser.parse_args(sys.argv[1:])
erange_0_edges = np.logspace(np.log10(30), 2, 3)
erange_1_edges = np.logspace(2, np.log10(300), 3)
erange_2_edges = np.logspace(np.log10(300), 3, 4)
erange_3_edges = np.logspace(3, 6, 10)
erange_0 = np.sqrt(erange_0_edges[0:-1] * erange_0_edges[1:])
erange_1 = np.sqrt(erange_1_edges[0:-1] * erange_1_edges[1:])
erange_2 = np.sqrt(erange_2_edges[0:-1] * erange_2_edges[1:])
erange_3 = np.sqrt(erange_3_edges[0:-1] * erange_3_edges[1:])
energies = [erange_0, erange_1, erange_2, erange_3]
ewidths = [
erange_0_edges[1:] - erange_0_edges[0:-1],
erange_1_edges[1:] - erange_1_edges[0:-1],
erange_2_edges[1:] - erange_2_edges[0:-1],
erange_3_edges[1:] - erange_3_edges[0:-1]
]
l = []
factor = 12 * 32 * 32
for evals, ew, comp in zip(energies, ewidths, COMPONENTS):
exppath = os.path.join(args.exposure,
"%s_expcube_moreplanes_ring_P8R3.fits" % (comp))
egbpath = args.input
exp_vals = get_expmap_sums(exppath, evals)
egb_vals = get_int_egb(egbpath, evals)
o_vals = exp_vals * egb_vals * ew / factor
l.append(o_vals)
o = dict(iso_map=l)
write_yaml(o, args.output)
def _write_profile_yaml(cls, target, profile_path, targ_ver, spatial):
""" Write a yaml file describing the spatial profile of the target.
Parameters
----------
target : dict
Specific target
profile_path : str
Path for the output file
targ_ver : str
Version of the target, used for bookkeeping
spatial : str
Spatial model, one of ['point', 'radial', 'map']
Returns
-------
output : dict
The description of the target spatial profile
"""
source_model = dict(SpectrumType='PowerLaw',
RA=target['ra'],
DEC=target['dec'])
if spatial in [None, 'point']:
source_model.update(dict(SpatialModel='PointSource'))
elif spatial in ['map']:
if target.j_map_file is None:
j_map_file = profile_path.replace('.yaml', '.fits')
target.write_jmap_wcs(j_map_file)
source_model.update(
dict(SpatialModel='DiffuseSource',
SpatialType='SpatialMap',
Spatial_Filename=target['map']))
elif spatial in ['radial']:
target.j_rad_file = profile_path.replace('.yaml', '.dat')
target.write_j_rad_file()
source_model.update(
dict(SpatialModel='DiffuseSource',
SpatialType='RadialProfile',
radialprofile=target['radial']))
else:
raise ValueError('Did not recognize spatial type %s' % spatial)
ver_name = "%s_%s" % (targ_ver, spatial)
profile_dict = dict(name=ver_name, source_model=source_model)
write_yaml(profile_dict, profile_path)
return profile_dict
def main():
usage = "fermitool-gardian_srcmap_sums.py [options]"
description = "sum source maps files over all pixels"
parser = argparse.ArgumentParser(usage=usage, description=description)
parser.add_argument('-i',
'--input',
type=str,
default=None,
help='Input file prefix')
parser.add_argument('-o',
'--output',
type=str,
default=None,
help='Output ')
parser.add_argument('-d',
'--conv_dict',
type=str,
default=None,
help='Conversion dict ')
args = parser.parse_args(sys.argv[1:])
out_dict = {}
conv_dict = load_yaml(args.conv_dict)
comps = conv_dict['COMPONENTS']
src_dict = conv_dict['SRC_NAME_DICT']
sel_dep_comps = conv_dict['SEL_DEP_COMPS']
cache_dirs = conv_dict['CACHE_DIR_DICT']
sys.stdout.write("Reading %i components\n" % (len(SRC_NAME_DICT)))
for k, v in sorted(src_dict.items()):
sys.stdout.write('.')
sys.stdout.flush()
cache_dir = cache_dirs[k]
for c in comps:
if k in sel_dep_comps:
fp = os.path.join(args.input, cache_dir,
"%s_%s%s_cache.fits.gz" % (c, k, c[0:2]))
else:
fp = os.path.join(args.input, cache_dir,
"%s_%s_cache.fits.gz" % (c, k))
map_in = read_map_from_fits(fp)
sums = map_in.counts.sum(1)
if not out_dict.has_key(v):
out_dict[v] = []
out_dict[v].append(sums)
sys.stdout.write('!\n')
utils.write_yaml(out_dict, args.output)
def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
if not HAVE_ST:
raise RuntimeError(
"Trying to run fermipy analysis, but don't have ST")
workdir = os.path.dirname(args.config)
_config_file = self._clone_config_and_srcmaps(args.config, args.seed)
gta = GTAnalysis(_config_file, logging={'verbosity': 3},
fileio={'workdir_regex': '\.xml$|\.npy$'})
gta.load_roi(args.roi_baseline)
simfile = os.path.join(workdir, 'sim_%s_%s.yaml' %
(args.sim, args.sim_profile))
mcube_file = "%s_%s_%06i" % (args.sim, args.sim_profile, args.seed)
sim_config = utils.load_yaml(simfile)
injected_source = sim_config.get('injected_source', None)
if injected_source is not None:
src_dict = injected_source['source_model']
src_dict['ra'] = gta.config['selection']['ra']
src_dict['dec'] = gta.config['selection']['dec']
injected_name = injected_source['name']
gta.add_source(injected_name, src_dict)
gta.write_model_map(mcube_file)
mc_spec_dict = dict(true_counts=gta.model_counts_spectrum(injected_name),
energies=gta.energies,
model=src_dict)
mcspec_file = os.path.join(workdir,
"mcspec_%s_%06i.yaml" % (mcube_file, args.seed))
utils.write_yaml(mc_spec_dict, mcspec_file)
else:
injected_name = None
gta.write_roi('sim_baseline_%06i' % args.seed)
test_sources = []
for profile in args.profiles:
profile_path = os.path.join(workdir, 'profile_%s.yaml' % profile)
test_source = load_yaml(profile_path)
test_sources.append(test_source)
first = args.seed
last = first + args.nsims
for seed in range(first, last):
self._run_simulation(gta, args.roi_baseline,
injected_name, test_sources, first, seed,
non_null_src=args.non_null_src)
def _write_sim_target_config(cls, target_config, target_dir,
sim_target_dir):
""" Write a fermipy configurate file for one target
for simulated analysis.
This largely copies the configuration for flight data.
It does make a few changes to point some of the input
files (like the exposure map) to the flight data verions
to avoid having to re-compute them.
Parameters
----------
target_config : dict
Configuration for flight data analysis for this target
target_dir : str
Analysis directory for flight data for this target
sim_target_dir : str
Directory to write to
Returns
-------
output : dict
The configuration for this specific target
"""
sim_target_config_path = os.path.join(sim_target_dir, 'config.yaml')
sim_target_config = copy.deepcopy(target_config)
comps = sim_target_config.get('components', [sim_target_config])
for i, comp in enumerate(comps):
comp_name = "%02i" % i
if 'gtlike' not in comp:
comp['gtlike'] = {}
comp['gtlike']['bexpmap'] = os.path.abspath(
os.path.join(target_dir, 'bexpmap_%s.fits' % comp_name))
comp['gtlike']['srcmap'] = os.path.abspath(
os.path.join(sim_target_dir, 'srcmap_%s.fits' % comp_name))
comp['gtlike']['use_external_srcmap'] = True
write_yaml(sim_target_config, sim_target_config_path)
return sim_target_config
def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
if is_null(args.config):
raise ValueError("Config yaml file must be specified")
if is_null(args.rand_config):
raise ValueError(
"Random direction config yaml file must be specified")
config = load_yaml(args.config)
rand_config = load_yaml(args.rand_config)
wcsgeom = self._make_wcsgeom_from_config(config)
dir_dict = self._build_skydir_dict(wcsgeom, rand_config)
if is_not_null(args.outfile):
write_yaml(dir_dict, args.outfile)
def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
if is_null(args.config):
raise ValueError("Config yaml file must be specified")
if is_null(args.rand_config):
raise ValueError(
"Random direction config yaml file must be specified")
config = load_yaml(args.config)
rand_config = load_yaml(args.rand_config)
wcsgeom = self._make_wcsgeom_from_config(config)
dir_dict = self._build_skydir_dict(wcsgeom, rand_config)
if is_not_null(args.outfile):
write_yaml(dir_dict, args.outfile)
def _clone_config_and_srcmaps(config_path, seed):
"""Clone the configuration"""
workdir = os.path.dirname(config_path)
new_config_path = config_path.replace('.yaml', '_%06i.yaml' % seed)
config = load_yaml(config_path)
comps = config.get('components', [config])
for i, comp in enumerate(comps):
comp_name = "%02i" % i
if 'gtlike' not in comp:
comp['gtlike'] = {}
orig_srcmap = os.path.abspath(os.path.join(workdir, 'srcmap_%s.fits' % (comp_name)))
new_srcmap = os.path.abspath(os.path.join(workdir, 'srcmap_%06i_%s.fits' % (seed, comp_name)))
comp['gtlike']['srcmap'] = os.path.abspath(os.path.join(workdir, 'srcmap_%06i_%s.fits' % (seed, comp_name)))
comp['gtlike']['use_external_srcmap'] = True
copyfile(orig_srcmap, new_srcmap)
write_yaml(config, new_config_path)
return new_config_path
def write_target_dirs(basedir, roster_dict, base_config):
""" Create and populate directoris for target analysis
"""
target_dict = {}
target_info_dict = {}
roster_info_dict = {}
try:
os.makedirs(basedir)
except OSError:
pass
for roster_name, rost in roster_dict.items():
tlist = []
for target_name, target in rost.items():
target_key = "%s:%s" % (target_name, target.version)
print("Writing %s" % (target_key))
tlist.append(target_key)
if target_info_dict.has_key(target_name):
target_info_dict[target_name].append(target.version)
else:
target_info_dict[target_name] = [target.version]
target_dir = os.path.join(basedir, target_name)
target_config_path = os.path.join(target_dir, 'config_baseline.yaml')
jmap_path = os.path.join(target_dir, 'profile_%s.fits' % target.version)
profile_path = os.path.join(target_dir, 'profile_%s.yaml' % target.version)
if target_dict.has_key(target_name):
# Already made the config for this target
target_config = target_dict[target_name]
else:
# Make the config for this target
try:
os.makedirs(target_dir)
except OSError:
pass
target_config = base_config.copy()
target_config['selection']['ra'] = target.ra
target_config['selection']['dec'] = target.dec
target_dict[target_name] = target_config
write_yaml(target_config, target_config_path)
profile_data = target.profile.copy()
target.write_jmap_wcs(jmap_path, clobber=True)
profile_data['j_integ'] = target.j_integ
profile_data['j_sigma'] = target.j_sigma
profile_data['j_map_file'] = jmap_path
write_yaml(profile_data, profile_path)
roster_info_dict[roster_name] = tlist
write_yaml(roster_info_dict, os.path.join(basedir, 'roster_list.yaml'))
write_yaml(target_info_dict, os.path.join(basedir, 'target_list.yaml'))
def main():
usage = "srcmap_sums.py [options]"
description = "sum source maps files over all pixels"
parser = argparse.ArgumentParser(usage=usage, description=description)
parser.add_argument('-i',
'--input',
type=str,
default=None,
help='Input file prefix')
parser.add_argument('-o',
'--output',
type=str,
default=None,
help='Output ')
parser.add_argument('-d',
'--conv_dict',
type=str,
default=None,
help='Conversion dict ')
args = parser.parse_args(sys.argv[1:])
out_dict = {}
conv_dict = utils.load_yaml(args.conv_dict)
comps = conv_dict['COMPONENTS']
src_dict = conv_dict['SRC_NAME_DICT']
for c in comps:
fp = "%s_%s_GAL_V2.fits" % (args.input, c)
fin = fits.open(fp)
ebins = fits_utils.find_and_read_ebins(fin)
for hdu in fin[4:]:
if hdu.name in ['__weights__']:
continue
if hdu.name[0:5] == 'FL8Y ':
continue
sums = get_sums(hdu, ebins)
src_name = src_dict[hdu.name]
if not out_dict.has_key(src_name):
out_dict[src_name] = []
out_dict[src_name].append(sums)
utils.write_yaml(out_dict, args.output)
def main():
usage = "npred_mapSums.py [options]"
description = "Convert Gardian text files to yaml dictionary"
parser = argparse.ArgumentParser(usage=usage, description=description)
parser.add_argument('-i', '--input', type=str, default=None,
help='Input file')
parser.add_argument('-o', '--output', type=str, default=None,
help='Output ')
parser.add_argument('-d', '--conv_dict', type=str, default=None,
help='Conversion dict ')
args = parser.parse_args(sys.argv[1:])
conv_dict = load_yaml(args.conv_dict)
comps = conv_dict['COMPONENTS']
src_dict = conv_dict['SRC_NAME_DICT']
sel_dep_comps = conv_dict['SEL_DEP_COMPS']
sum_dict = make_sum_dict(args.input, comps, src_dict, sel_dep_comps)
write_yaml(sum_dict, args.output)
def _write_profile_yaml(cls, target, profile_path, targ_ver, spatial):
""" Write a yaml file describing the spatial profile of the target.
Parameters
----------
target : dict
Specific target
profile_path : str
Path for the output file
targ_ver : str
Version of the target, used for bookkeeping
spatial : str
Spatial model, one of ['point', 'radial', 'map']
Returns
-------
output : dict
The description of the target spatial profile
"""
source_model = dict(SpectrumType='PowerLaw',
RA=target['ra'],
DEC=target['dec'])
if spatial in [None, 'point']:
source_model.update(dict(SpatialModel='PointSource'))
else:
raise ValueError('Did not recognize spatial type %s' % spatial)
ver_name = "%s_%s" % (targ_ver, spatial)
profile_dict = dict(name=ver_name,
source_model=source_model)
write_yaml(profile_dict, profile_path)
return profile_dict
def _write_astro_value_yaml(cls, target, astro_val_path):
"""Write a yaml file describing the target
Parameters
----------
target : dict
Specific target
astro_val_path : str
Path for the output file
Returns
-------
output : dict
The description of the targe
"""
astro_profile_data = target.copy()
write_yaml(astro_profile_data, astro_val_path)
return astro_profile_data
def _run_simulation(gta, roi_baseline,
injected_name, test_sources, current_seed, seed, non_null_src):
"""Simulate a realization of this analysis"""
gta.load_roi('sim_baseline_%06i.npy' % current_seed)
gta.set_random_seed(seed)
gta.simulate_roi()
if injected_name:
gta.zero_source(injected_name)
gta.optimize()
gta.find_sources(sqrt_ts_threshold=5.0, search_skydir=gta.roi.skydir,
search_minmax_radius=[1.0, np.nan])
gta.optimize()
gta.free_sources(skydir=gta.roi.skydir, distance=1.0, pars='norm')
gta.fit(covar=True)
gta.write_roi('sim_refit_%06i' % current_seed)
for test_source in test_sources:
test_source_name = test_source['name']
sedfile = "sed_%s_%06i.fits" % (test_source_name, seed)
correl_dict, test_src_name = add_source_get_correlated(gta, test_source_name,
test_source['source_model'],
correl_thresh=0.25,
non_null_src=non_null_src)
# Write the list of correlated sources
correl_yaml = os.path.join(gta.workdir,
"correl_%s_%06i.yaml" % (test_source_name, seed))
write_yaml(correl_dict, correl_yaml)
gta.free_sources(False)
for src_name in correl_dict.keys():
gta.free_source(src_name, pars='norm')
gta.sed(test_source_name, prefix=pkey, outfile=sedfile)
# Set things back to how they were
gta.delete_source(test_source_name)
gta.load_xml('sim_refit_%06i' % current_seed)
def _clone_config_and_srcmaps(config_path, seed):
"""Clone the configuration"""
workdir = os.path.dirname(config_path)
new_config_path = config_path.replace('.yaml', '_%06i.yaml' % seed)
config = load_yaml(config_path)
comps = config.get('components', [config])
for i, comp in enumerate(comps):
comp_name = "%02i" % i
if 'gtlike' not in comp:
comp['gtlike'] = {}
orig_srcmap = os.path.abspath(
os.path.join(workdir, 'srcmap_%s.fits' % (comp_name)))
new_srcmap = os.path.abspath(
os.path.join(workdir,
'srcmap_%06i_%s.fits' % (seed, comp_name)))
comp['gtlike']['srcmap'] = os.path.abspath(
os.path.join(workdir,
'srcmap_%06i_%s.fits' % (seed, comp_name)))
comp['gtlike']['use_external_srcmap'] = True
copyfile(orig_srcmap, new_srcmap)
write_yaml(config, new_config_path)
return new_config_path
def main():
usage = "npred_txt2yaml.py [options]"
description = "Convert Gardian text files to yaml dictionary"
parser = argparse.ArgumentParser(usage=usage, description=description)
parser.add_argument('-i',
'--input',
type=str,
default=None,
help='Input file')
parser.add_argument('-o',
'--output',
type=str,
default=None,
help='Output ')
args = parser.parse_args(sys.argv[1:])
src_dict = load_npy(args.input)
npred_dict = make_npred_dict(src_dict['sources'])
npred_dict['Energies'] = get_energies(src_dict['roi'])
write_yaml(npred_dict, args.output)
def _write_sim_yaml(cls, target, sim, sim_target_dir, target_key):
"""Write a yaml file describing the 'True' target parameters
for simulated data.
Parameters
----------
target : dict
Specific target
sim : str
Name of the simulation scenario.
sim_target_dir : str
Directory to write to
target_key : str
Version of the target to write
Returns
-------
output : dict
The description of the 'True' target parameters
"""
sim_profile_yaml = os.path.join('config', 'sim_%s.yaml' % sim)
sim_profile = load_yaml(sim_profile_yaml)
injected_source = sim_profile.get('injected_source', None)
#if injected_source is not None:
# sim_profile['injected_source']['source_model']['norm']['value'] = target['norm']
sim_out_path = os.path.join(
sim_target_dir, 'sim_%s_%s.yaml' %
(sim, target_key))
write_yaml(sim_profile, sim_out_path)
return sim_profile
def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
if not HAVE_ST:
raise RuntimeError(
"Trying to run fermipy analysis, but don't have ST")
if is_null(args.skydirs):
skydir_dict = None
else:
skydir_dict = load_yaml(args.skydirs)
gta = GTAnalysis(args.config,
logging={'verbosity': 3},
fileio={'workdir_regex': '\.xml$|\.npy$'})
#gta.setup(overwrite=False)
gta.load_roi(args.roi_baseline)
gta.print_roi()
basedir = os.path.dirname(args.config)
# This should be a no-op, b/c it was done in the baseline analysis
for profile in args.profiles:
if skydir_dict is None:
skydir_keys = [None]
else:
skydir_keys = sorted(skydir_dict.keys())
for skydir_key in skydir_keys:
if skydir_key is None:
pkey, pdict = AnalyzeSED._build_profile_dict(
basedir, profile)
else:
skydir_val = skydir_dict[skydir_key]
pkey, pdict = AnalyzeSED._build_profile_dict(
basedir, profile)
pdict['ra'] = skydir_val['ra']
pdict['dec'] = skydir_val['dec']
pkey += "_%06i" % skydir_key
outfile = "sed_%s.fits" % pkey
# Add the source and get the list of correlated soruces
correl_dict = add_source_get_correlated(gta,
pkey,
pdict,
correl_thresh=0.25)
# Write the list of correlated sources
correl_yaml = os.path.join(basedir, "correl_%s.yaml" % pkey)
write_yaml(correl_dict, correl_yaml)
gta.free_sources(False)
for src_name in correl_dict.keys():
gta.free_source(src_name, pars='norm')
# build the SED
gta.sed(pkey, outfile=outfile, make_plots=args.make_plots)
# remove the source
gta.delete_source(pkey)
# put the ROI back to how it was
gta.load_xml(args.roi_baseline)
return gta
def run_analysis(self, argv):
"""Run this analysis"""
args = self._parser.parse_args(argv)
if not HAVE_ST:
raise RuntimeError(
"Trying to run fermipy analysis, but don't have ST")
if is_null(args.skydirs):
skydir_dict = None
else:
skydir_dict = load_yaml(args.skydirs)
gta = GTAnalysis(args.config,
logging={'verbosity': 3},
fileio={'workdir_regex': '\.xml$|\.npy$'})
#gta.setup(overwrite=False)
gta.load_roi(args.roi_baseline)
gta.print_roi()
basedir = os.path.dirname(args.config)
# This should be a no-op, b/c it was done in the baseline analysis
for profile in args.profiles:
if skydir_dict is None:
skydir_keys = [None]
else:
skydir_keys = sorted(skydir_dict.keys())
for skydir_key in skydir_keys:
if skydir_key is None:
pkey, psrc_name, pdict = build_profile_dict(basedir, profile)
else:
skydir_val = skydir_dict[skydir_key]
pkey, psrc_name, pdict = build_profile_dict(basedir, profile)
pdict['ra'] = skydir_val['ra']
pdict['dec'] = skydir_val['dec']
pkey += "_%06i" % skydir_key
outfile = "sed_%s.fits" % pkey
# Add the source and get the list of correlated soruces
correl_dict, test_src_name = add_source_get_correlated(gta, psrc_name,
pdict, correl_thresh=0.25,
non_null_src=args.non_null_src)
# Write the list of correlated sources
correl_yaml = os.path.join(basedir, "correl_%s.yaml" % pkey)
write_yaml(correl_dict, correl_yaml)
gta.free_sources(False)
for src_name in correl_dict.keys():
gta.free_source(src_name, pars='norm')
# build the SED
if args.non_null_src:
gta.update_source(test_src_name, reoptimize=True)
gta.write_roi("base_%s"% pkey, make_plots=False)
gta.sed(test_src_name, prefix=pkey, outfile=outfile, make_plots=args.make_plots)
# remove the source
gta.delete_source(test_src_name)
# put the ROI back to how it was
gta.load_xml(args.roi_baseline)
return gta
def make_fermipy_config_yaml(self, modelkey, components, data, **kwargs):
"""Build a fermipy top-level yaml configuration file
Parameters
----------
modelkey : str
Key used to identify this particular model
components : list
The binning components used in this analysis
data : str
Path to file containing dataset definition
"""
model_dir = os.path.join('analysis', 'model_%s' % modelkey)
hpx_order = kwargs.get('hpx_order', 9)
self._name_factory.update_base_dict(data)
try:
model_info = self._models[modelkey]
except KeyError:
model_info = self.make_model_info(modelkey)
model_info.make_model_rois(components, self._name_factory)
#source_names = model_info.component_names
master_xml_mdl = os.path.basename(
self._name_factory.master_srcmdl_xml(modelkey=modelkey))
master_data = dict(scfile=self._name_factory.ft2file(fullpath=True),
cacheft1=False)
master_binning = dict(projtype='HPX',
coordsys=kwargs.get('coordsys', 'GAL'),
roiwidth=180.,
binsperdec=8,
hpx_ordering_scheme="RING",
hpx_order=hpx_order,
hpx_ebin=True)
master_fileio = dict(outdir=model_dir,
logfile=os.path.join(model_dir, 'fermipy.log'))
master_gtlike = dict(irfs=self._name_factory.irfs(**kwargs),
edisp_disable=['isodiff', 'diffuse', 'limb'])
master_selection = dict(glat=0., glon=0., radius=180.)
master_model = dict(catalogs=[master_xml_mdl])
master = dict(data=master_data,
binning=master_binning,
fileio=master_fileio,
selection=master_selection,
gtlike=master_gtlike,
model=master_model,
components=[])
fermipy_dict = master
#comp_rois = {}
for comp in components:
zcut = "zmax%i" % comp.zmax
compkey = "%s_%s" % (zcut,
comp.make_key('{ebin_name}_{evtype_name}'))
name_keys = dict(zcut=zcut,
modelkey=modelkey,
component=compkey,
fullpath=True)
comp_data = dict(ltcube=self._name_factory.ltcube(**name_keys))
comp_selection = dict(logemin=comp.log_emin,
logemax=comp.log_emax,
zmax=comp.zmax,
evtype=comp.evtype)
comp_binning = dict(enumbins=comp.enumbins,
hpx_order=min(comp.hpx_order, hpx_order))
comp_gtlike = dict(
srcmap=self._name_factory.merged_srcmaps(**name_keys),
bexpmap=self._name_factory.bexpcube(**name_keys))
#comp_roi_source_info = {}
comp_xml_mdl = os.path.basename(
self._name_factory.comp_srcmdl_xml(modelkey=modelkey,
component=compkey))
comp_model = dict(catalogs=[master_xml_mdl, comp_xml_mdl])
sub_dict = dict(data=comp_data,
binning=comp_binning,
selection=comp_selection,
gtlike=comp_gtlike,
model=comp_model)
fermipy_dict['components'].append(sub_dict)
outfile = os.path.join(model_dir, 'config.yaml')
print("Writing fermipy config file %s" % outfile)
utils.write_yaml(fermipy_dict, outfile)
return fermipy_dict
def write_config(self, outfile):
"""Write the configuration dictionary to an output file."""
utils.write_yaml(self.config, outfile, default_flow_style=False)
def make_fermipy_config_yaml(self, modelkey, components, data, **kwargs):
"""Build a fermipy top-level yaml configuration file
Parameters
----------
modelkey : str
Key used to identify this particular model
components : list
The binning components used in this analysis
data : str
Path to file containing dataset definition
"""
model_dir = os.path.join('analysis', 'model_%s' % modelkey)
hpx_order = kwargs.get('hpx_order', 9)
self._name_factory.update_base_dict(data)
try:
model_info = self._models[modelkey]
except KeyError:
model_info = self.make_model_info(modelkey)
model_info.make_model_rois(components, self._name_factory)
#source_names = model_info.component_names
master_xml_mdl = os.path.basename(
self._name_factory.master_srcmdl_xml(modelkey=modelkey))
master_data = dict(scfile=self._name_factory.ft2file(fullpath=True),
cacheft1=False)
master_binning = dict(projtype='HPX',
coordsys=kwargs.get('coordsys', 'GAL'),
roiwidth=180.,
binsperdec=8,
hpx_ordering_scheme="RING",
hpx_order=hpx_order,
hpx_ebin=True)
master_fileio = dict(outdir=model_dir,
logfile=os.path.join(model_dir, 'fermipy.log'))
master_gtlike = dict(irfs=self._name_factory.irfs(**kwargs),
edisp_disable=['isodiff', 'diffuse', 'limb'])
master_selection = dict(glat=0., glon=0., radius=180.)
master_model = dict(catalogs=[master_xml_mdl])
master = dict(data=master_data,
binning=master_binning,
fileio=master_fileio,
selection=master_selection,
gtlike=master_gtlike,
model=master_model,
components=[])
fermipy_dict = master
#comp_rois = {}
for comp in components:
zcut = "zmax%i" % comp.zmax
compkey = "%s_%s" % (zcut, comp.make_key(
'{ebin_name}_{evtype_name}'))
name_keys = dict(zcut=zcut,
modelkey=modelkey,
component=compkey,
fullpath=True)
comp_data = dict(ltcube=self._name_factory.ltcube(**name_keys))
comp_selection = dict(logemin=comp.log_emin,
logemax=comp.log_emax,
zmax=comp.zmax,
evtype=comp.evtype)
comp_binning = dict(enumbins=comp.enumbins,
hpx_order=min(comp.hpx_order, hpx_order))
comp_gtlike = dict(srcmap=self._name_factory.merged_srcmaps(**name_keys),
bexpmap=self._name_factory.bexpcube(**name_keys))
#comp_roi_source_info = {}
comp_xml_mdl = os.path.basename(self._name_factory.comp_srcmdl_xml(modelkey=modelkey,
component=compkey))
comp_model = dict(catalogs=[master_xml_mdl, comp_xml_mdl])
sub_dict = dict(data=comp_data,
binning=comp_binning,
selection=comp_selection,
gtlike=comp_gtlike,
model=comp_model)
fermipy_dict['components'].append(sub_dict)
outfile = os.path.join(model_dir, 'config.yaml')
print ("Writing fermipy config file %s"%outfile)
utils.write_yaml(fermipy_dict, outfile)
return fermipy_dict
def make_fermipy_config_yaml(self, modelkey, components, data, **kwargs):
"""Build a fermipy top-level yaml configuration file
Parameters
----------
modelkey : str
Key used to identify this particular model
components : list
The binning components used in this analysis
data : str
Path to file containing dataset definition
"""
model_dir = os.path.join('analysis', 'model_%s' % modelkey)
hpx_order = kwargs.get('hpx_order', 9)
self._name_factory.update_base_dict(data)
try:
model_info = self._models[modelkey]
except KeyError:
model_info = self.make_model_info(modelkey)
model_rois = model_info.make_model_rois(components, self._name_factory)
#source_names = model_info.component_names
master_xml_mdl = os.path.basename(
self._name_factory.master_srcmdl_xml(modelkey=modelkey))
master_data = dict(scfile=self._name_factory.ft2file(fullpath=True),
cacheft1=False)
master_binning = dict(projtype='HPX',
roiwidth=360.,
binsperdec=4,
hpx_ordering_scheme="RING",
hpx_order=hpx_order,
hpx_ebin=True)
# master_fileio = dict(outdir=model_dir,
# logfile=os.path.join(model_dir, 'fermipy.log'))
master_fileio = dict(logfile='fermipy.log')
master_gtlike = dict(irfs=self._name_factory.irfs(**kwargs),
edisp_disable=model_info.edisp_disable_list(),
use_external_srcmap=True)
master_selection = dict(glat=0., glon=0., radius=180.)
master_model = dict(catalogs=[master_xml_mdl])
master_plotting = dict(label_ts_threshold=1e9)
master = dict(data=master_data,
binning=master_binning,
fileio=master_fileio,
selection=master_selection,
gtlike=master_gtlike,
model=master_model,
plotting=master_plotting,
components=[])
fermipy_dict = master
#comp_rois = {}
for comp in components:
zcut = "zmax%i" % comp.zmax
compkey = "%s_%s" % (zcut,
comp.make_key('{ebin_name}_{evtype_name}'))
comp_roi = model_rois[compkey]
name_keys = dict(zcut=zcut,
modelkey=modelkey,
component=compkey,
mktime='none',
coordsys=comp.coordsys,
fullpath=True)
comp_data = dict(ltcube=self._name_factory.ltcube(**name_keys))
comp_selection = dict(logemin=comp.log_emin,
logemax=comp.log_emax,
zmax=comp.zmax,
evtype=comp.evtype)
comp_binning = dict(enumbins=comp.enumbins,
hpx_order=min(comp.hpx_order, hpx_order),
coordsys=comp.coordsys)
comp_gtlike = dict(
srcmap=self._name_factory.merged_srcmaps(**name_keys),
bexpmap=self._name_factory.bexpcube(**name_keys),
use_external_srcmap=True)
#comp_roi_source_info = {}
diffuse_srcs = []
for src in comp_roi.diffuse_sources:
if isinstance(src, MapCubeSource):
diffuse_srcs.append(dict(name=src.name, file=src.mapcube))
elif isinstance(src, IsoSource):
diffuse_srcs.append(
dict(name=src.name, file=src.fileFunction))
else:
pass
comp_model = dict(diffuse=diffuse_srcs)
sub_dict = dict(data=comp_data,
binning=comp_binning,
selection=comp_selection,
gtlike=comp_gtlike,
model=comp_model)
fermipy_dict['components'].append(sub_dict)
# Add placeholder diffuse sources
fermipy_dict['model']['diffuse'] = diffuse_srcs
outfile = os.path.join(model_dir, 'config.yaml')
print("Writing fermipy config file %s" % outfile)
utils.write_yaml(fermipy_dict, outfile)
return fermipy_dict
def _write_target_dirs(cls, ttype, roster_dict,
base_config, sims, spatial_models,
aliases):
""" Create and populate directoris for target analysis.
Parameters
----------
ttype : str
Target type, used for bookeeping and directory naming
roster_dict : dict
Dictionary of dict objects with the analysis targets
base_config : dict
Baseline configuration
sims : list
List of names of simulation scenarios
spatial_models : dict
Dictionary types of spatial models to use in analysis
aliases : dict
Optional dictionary to remap target verion keys
"""
target_dict = {}
target_info_dict = {}
roster_info_dict = {}
for roster_name, rost in roster_dict.items():
for target_name, target in rost.items():
print(roster_name, target_name, target)
if aliases is not None:
try:
ver_key = aliases[target.get("version", "default")]
except KeyError:
ver_key = target.get("version", "default")
else:
ver_key = target.get("version", "default")
target_key = "%s:%s" % (target_name, ver_key)
print("Writing %s" % (target_key))
name_keys = dict(target_type=ttype,
target_name=target_name,
target_version=ver_key,
fullpath=True)
astro_val_path = NAME_FACTORY.astro_valuefile(**name_keys)
target_dir = NAME_FACTORY.targetdir(**name_keys)
try:
os.makedirs(target_dir)
except OSError:
pass
cls._write_astro_value_yaml(target, astro_val_path)
for sim in sims:
name_keys['sim_name'] = sim
sim_target_dir = NAME_FACTORY.sim_targetdir(**name_keys)
sim_astro_val_path = NAME_FACTORY.sim_astro_valuefile(**name_keys)
try:
os.makedirs(sim_target_dir)
except OSError:
pass
cls._write_astro_value_yaml(target, sim_astro_val_path)
cls._write_sim_yaml(target, sim, sim_target_dir, ver_key)
name_keys.pop('sim_name')
write_config = False
if target_name in target_dict:
# Already made the config for this target
target_config = target_dict[target_name].copy()
else:
# Make the config for this target
target_config = cls._write_data_target_config(base_config,
target, target_dir)
target_dict[target_name] = target_config
write_config = True
write_sim_config = write_config
for spatial in spatial_models:
ver_string = "%s_%s" % (ver_key, spatial)
roster_key = "%s_%s" % (roster_name, spatial)
full_key = "%s:%s" % (target_name, ver_string)
name_keys['profile'] = ver_string
profile_path = NAME_FACTORY.profilefile(**name_keys)
if target_name in target_info_dict:
target_info_dict[target_name].append(ver_string)
else:
target_info_dict[target_name] = [ver_string]
cls._write_profile_yaml(target, profile_path,
ver_key, spatial)
if roster_key in roster_info_dict:
roster_info_dict[roster_key].append(full_key)
else:
roster_info_dict[roster_key] = [full_key]
for sim in sims:
name_keys['sim_name'] = sim
sim_target_dir = NAME_FACTORY.sim_targetdir(**name_keys)
sim_profile_path = NAME_FACTORY.sim_profilefile(**name_keys)
if write_sim_config:
cls._write_sim_target_config(target_config,
target_dir, sim_target_dir)
cls._write_profile_yaml(target, sim_profile_path,
ver_key, spatial)
write_sim_config = False
roster_file = os.path.join(ttype, 'roster_list.yaml')
target_file = os.path.join(ttype, 'target_list.yaml')
write_yaml(roster_info_dict, roster_file)
write_yaml(target_info_dict, target_file)
for sim in sims:
sim_dir = os.path.join("%s_sim" % ttype, "sim_%s" % sim)
sim_roster_file = os.path.join(sim_dir, 'roster_list.yaml')
sim_target_file = os.path.join(sim_dir, 'target_list.yaml')
try:
os.makedirs(sim_dir)
except OSError:
pass
copyfile(roster_file, sim_roster_file)
copyfile(target_file, sim_target_file)
