def clone_configs(basedir, base_configs, opt_configs, scripts, args=''):
"""
"""
config = {}
for c in base_configs:
config = utils.merge_dict(config,
yaml.load(open(c)),
add_new_keys=True)
scriptdir = os.path.abspath(os.path.join(basedir, 'scripts'))
utils.mkdir(scriptdir)
bash_scripts = []
for script_in in scripts:
bash_script = """
cat $0
{scriptexe} --config={config} {args}
"""
if os.path.isfile(script_in):
script = os.path.basename(script_in)
scriptpath = os.path.join(scriptdir, script)
scriptexe = 'python ' + scriptpath
os.system('cp %s %s' % (script_in, scriptdir))
elif cmd_exists(script_in):
scriptexe = script_in
script = script_in
else:
raise Exception('Could not find script: %s' % script_in)
bash_scripts.append((script, scriptexe, bash_script))
for name, vdict in opt_configs.items():
dirname = os.path.abspath(os.path.join(basedir, name))
utils.mkdir(dirname)
cfgfile = os.path.join(dirname, 'config.yaml')
for script_in, bash_script in zip(scripts, bash_scripts):
runscript = os.path.splitext(bash_script[0])[0] + '.sh'
runscript = os.path.join(dirname, runscript)
with open(os.path.join(runscript), 'wt') as f:
f.write(bash_script[2].format(source=name,
scriptexe=bash_script[1],
config=cfgfile,
args=args))
if not config:
continue
c = copy.deepcopy(config)
c = utils.merge_dict(c, vdict, add_new_keys=True)
yaml.dump(utils.tolist(c),
open(cfgfile, 'w'),
default_flow_style=False)
def create_pars_dict(name, pars_dict=None):
"""Create a dictionary for the parameters of a function.
Parameters
----------
name : str
Name of the function.
pars_dict : dict
Existing parameter dict that will be merged with the
default dictionary created by this method.
"""
default_pars_dict = get_function_defaults(name)
if pars_dict is None:
pars_dict = {}
else:
pars_dict = copy.deepcopy(pars_dict)
for k, v in pars_dict.items():
if not k in default_pars_dict:
continue
if not isinstance(v, dict):
pars_dict[k] = {'name': k, 'value': v}
pars_dict = utils.merge_dict(default_pars_dict, pars_dict)
for k, v in pars_dict.items():
pars_dict[k] = make_parameter_dict(v)
return pars_dict
def create_config(args):
config = dict(
fileio=dict(
outdir=None,
),
data=dict(
evfile=None,
ltcube=None,
scfile=None,
),
binning=dict(
roiwidth=10.0,
binsz=0.1,
binsperdec=8,
coordsys='GAL',
),
selection=dict(
emin=None,
emax=None,
logemin=None,
logemax=None,
tmin=239557417,
tmax=512994417,
zmax=100,
evclass=128,
evtype=3,
target=None,
ra=None,
dec=None,
glon=None,
glat=None,
),
gtlike=dict(
edisp=True,
irfs='P8R2_SOURCE_V6',
edisp_disable=['isodiff', 'galdiff']
),
model=dict(
src_roiwidth=15.0,
galdiff='$FERMI_DIFFUSE_DIR/v5r0/gll_iem_v06.fits',
isodiff='iso_P8R2_SOURCE_V6_v06.txt',
catalogs=['3FGL'],
),
)
if args['config'] is not None:
config = utils.merge_dict(config, yaml.load(open(args['config'])))
if args['outdir'] is not None:
config['fileio']['outdir'] = os.path.abspath(args['outdir'])
for s in ['data', 'selection']:
for k in config[s].keys():
if k in args and args[k] is not None:
config[s][k] = args[k]
return config
def create_config(self, config=None, validate=True, **kwargs):
config = {} if config is None else config
o = create_default_config(self)
config = utils.merge_dict(config, kwargs, add_new_keys=True)
cast_config(config, self)
if validate:
validate_from_schema(config, self)
o = update_from_schema(o, config, self)
return o
def create(cls, configfile):
"""Create a configuration dictionary from a yaml config file.
This function will first populate the dictionary with defaults
taken from pre-defined configuration files. The configuration
dictionary is then updated with the user-defined configuration
file. Any settings defined by the user will take precedence
over the default settings."""
# populate config dictionary with an initial set of values
# config_logging = ConfigManager.load('logging.yaml')
config = {}
if config['fileio']['outdir'] is None:
config['fileio']['outdir'] = os.path.abspath(
os.path.dirname(configfile))
user_config = cls.load(configfile)
config = utils.merge_dict(config, user_config, True)
config['fileio']['outdir'] = os.path.abspath(
config['fileio']['outdir'])
return config
def update_from_schema(cfg, cfgin, schema):
"""Update configuration dictionary ``cfg`` with the contents of
``cfgin`` using the ``schema`` dictionary to determine the valid
input keys.
Parameters
----------
cfg : dict
Configuration dictionary to be updated.
cfgin : dict
New configuration dictionary that will be merged with ``cfg``.
schema : dict
Configuration schema defining the valid configuration keys and
their types.
Returns
-------
cfgout : dict
"""
cfgout = copy.deepcopy(cfg)
for k, v in schema.items():
if k not in cfgin:
continue
if isinstance(v, dict):
cfgout.setdefault(k, {})
cfgout[k] = update_from_schema(cfg[k], cfgin[k], v)
elif v[2] is dict:
cfgout[k] = utils.merge_dict(cfg[k], cfgin[k], add_new_keys=True)
else:
cfgout[k] = cfgin[k]
return cfgout
def _make_lc(self, name, **kwargs):
# make array of time values in MET
if kwargs['time_bins']:
times = np.array(kwargs['time_bins'])
elif kwargs['nbins']:
times = np.linspace(self.tmin, self.tmax,
kwargs['nbins'] + 1)
else:
times = np.arange(self.tmin, self.tmax,
kwargs['binsz'])
diff_sources = [s.name for s in self.roi.sources if s.diffuse]
skydir = self.roi[name].skydir
if kwargs.get('free_radius', None) is not None:
kwargs['free_sources'] += [
s.name for s in self.roi.get_sources(skydir=skydir,
distance=kwargs['free_radius'],
exclude=diff_sources)]
# save params from full time fit
spectrum = self.like[name].src.spectrum()
specname = spectrum.genericName()
const_spectrum = (specname, gtutils.get_function_pars_dict(spectrum))
# Create Configurations
lck_params = copy.deepcopy(self._lck_params)
config = copy.deepcopy(self.config)
config['ltcube']['use_local_ltcube'] = kwargs['use_local_ltcube']
config['gtlike']['use_scaled_srcmap'] = kwargs['use_scaled_srcmap']
config['model']['diffuse_dir'] = [self.workdir]
config['selection']['filter'] = None
if config['components'] is None:
config['components'] = []
for j, c in enumerate(self.components):
if len(config['components']) <= j:
config['components'] += [{}]
data_cfg = {'evfile': c.files['ft1'],
'scfile': c.data_files['scfile'],
'ltcube': None}
gtlike_cfg = {}
if config['gtlike']['use_scaled_srcmap']:
gtlike_cfg['bexpmap_base'] = c.files['bexpmap']
gtlike_cfg['bexpmap_roi_base'] = c.files['bexpmap_roi']
gtlike_cfg['srcmap_base'] = c.files['srcmap']
config['components'][j] = \
utils.merge_dict(config['components'][j],
{'data': data_cfg, 'gtlike': gtlike_cfg},
add_new_keys=True)
config.setdefault('selection', {})
config['selection']['filter'] = None
outdir = kwargs.get('outdir', None)
basedir = outdir + '/' if outdir is not None else ''
wrap = partial(_process_lc_bin, name=name, config=config,
basedir=basedir, workdir=self.workdir, diff_sources=diff_sources,
const_spectrum=const_spectrum, roi=self.roi, lck_params=lck_params, **kwargs)
itimes = enumerate(zip(times[:-1], times[1:]))
if kwargs.get('multithread', False):
p = Pool(processes=kwargs.get('nthread', None))
mapo = p.map(wrap, itimes)
p.close()
else:
mapo = map(wrap, itimes)
if not kwargs.get('save_bin_data', False):
for m in mapo:
shutil.rmtree(m['config']['fileio']['outdir'])
o = self._create_lc_dict(name, times)
o['config'] = kwargs
itimes = enumerate(zip(times[:-1], times[1:]))
for i, time in itimes:
if not mapo[i]['fit_success']:
self.logger.error(
'Fit failed in bin %d in range %i %i.' % (i, time[0], time[1]))
continue
for k in o.keys():
if k == 'config':
continue
if not k in mapo[i]:
continue
# if (isinstance(o[k], np.ndarray) and
# o[k][i].shape != mapo[i][k].shape):
# gta.logger.warning('Incompatible shape for column %s', k)
# continue
try:
o[k][i] = mapo[i][k]
except:
pass
systematic = kwargs.get('systematic', 0.02)
o['ts_var'] = calcTS_var(loglike=o['loglike_fixed'],
loglike_const=o['loglike_const'],
flux_err=o['flux_err_fixed'],
flux_const=mapo[0]['flux_const'],
systematic=systematic,
fit_success=o['fit_success_fixed'])
return o
def __init__(self, options=None, **kwargs):
self._options = {} if options is None else options
self._options = utils.merge_dict(self._options, kwargs,
add_new_keys=True)
def residmap(self, prefix='', **kwargs):
"""Generate 2-D spatial residual maps using the current ROI
model and the convolution kernel defined with the `model`
argument.
Parameters
----------
prefix : str
String that will be prefixed to the output residual map files.
model : dict
Dictionary defining the properties of the convolution kernel.
exclude : str or list of str
Source or sources that will be removed from the model when
computing the residual map.
loge_bounds : list
Restrict the analysis to an energy range (emin,emax) in
log10(E/MeV) that is a subset of the analysis energy range.
By default the full analysis energy range will be used. If
either emin/emax are None then only an upper/lower bound on
the energy range wil be applied.
make_plots : bool
Generate plots.
write_fits : bool
Write the output to a FITS file.
write_npy : bool
Write the output dictionary to a numpy file.
Returns
-------
maps : dict
A dictionary containing the `~fermipy.utils.Map` objects
for the residual significance and amplitude.
"""
self.logger.info('Generating residual maps')
config = copy.deepcopy(self.config['residmap'])
config = utils.merge_dict(config, kwargs, add_new_keys=True)
# Defining default properties of test source model
config['model'].setdefault('Index', 2.0)
config['model'].setdefault('SpectrumType', 'PowerLaw')
config['model'].setdefault('SpatialModel', 'PointSource')
config['model'].setdefault('Prefactor', 1E-13)
make_plots = kwargs.get('make_plots', False)
maps = self._make_residual_map(prefix, config, **kwargs)
if make_plots:
plotter = plotting.AnalysisPlotter(self.config['plotting'],
fileio=self.config['fileio'],
logging=self.config['logging'])
plotter.make_residmap_plots(maps, self.roi)
self.logger.info('Finished residual maps')
return maps
