def _make_extension_fits(self, ext, filename, **kwargs):
maps = {
'EXT_TOT_MAP': ext['ext_tot_map'],
'EXT_SRC_MAP': ext['ext_src_map'],
'EXT_BKG_MAP': ext['ext_bkg_map'],
'PTSRC_TOT_MAP': ext['ptsrc_tot_map'],
'PTSRC_SRC_MAP': ext['ptsrc_src_map'],
'PTSRC_BKG_MAP': ext['ptsrc_bkg_map']
}
hdu_images = []
for k, v in sorted(maps.items()):
if v is None:
continue
hdu_images += [v.make_hdu(k)]
tab = fits_utils.dict_to_table(ext)
hdu_data = fits.table_to_hdu(tab)
hdu_data.name = 'EXT_DATA'
if ext.get('tsmap'):
hdus = [ext['tsmap'].make_hdu(hdu='PRIMARY')]
else:
hdus = [fits.PrimaryHDU()]
hdus += [hdu_data] + hdu_images
hdus[0].header['CONFIG'] = json.dumps(utils.tolist(ext['config']))
hdus[1].header['CONFIG'] = json.dumps(utils.tolist(ext['config']))
fits_utils.write_hdus(hdus,
filename,
keywords={'SRCNAME': ext['name']})
def _make_extension_fits(self, ext, filename, **kwargs):
maps = {'EXT_TOT_MAP': ext['ext_tot_map'],
'EXT_SRC_MAP': ext['ext_src_map'],
'EXT_BKG_MAP': ext['ext_bkg_map'],
'PTSRC_TOT_MAP': ext['ptsrc_tot_map'],
'PTSRC_SRC_MAP': ext['ptsrc_src_map'],
'PTSRC_BKG_MAP': ext['ptsrc_bkg_map']}
hdu_images = []
for k, v in sorted(maps.items()):
if v is None:
continue
hdu_images += [v.make_hdu(k)]
tab = fits_utils.dict_to_table(ext)
hdu_data = fits.table_to_hdu(tab)
hdu_data.name = 'EXT_DATA'
if ext.get('tsmap'):
hdus = [ext['tsmap'].make_hdu(hdu='PRIMARY')]
else:
hdus = [fits.PrimaryHDU()]
hdus += [hdu_data] + hdu_images
hdus[0].header['CONFIG'] = json.dumps(utils.tolist(ext['config']))
hdus[1].header['CONFIG'] = json.dumps(utils.tolist(ext['config']))
fits_utils.write_hdus(hdus, filename,
keywords={'SRCNAME': ext['name']})
def clone_configs(basedir, base_configs, opt_configs, scripts, args=''):
"""
"""
config = {}
for c in base_configs:
config = utils.merge_dict(config,
yaml.load(open(c)),
add_new_keys=True)
scriptdir = os.path.abspath(os.path.join(basedir, 'scripts'))
utils.mkdir(scriptdir)
bash_scripts = []
for script_in in scripts:
bash_script = """
cat $0
{scriptexe} --config={config} {args}
"""
if os.path.isfile(script_in):
script = os.path.basename(script_in)
scriptpath = os.path.join(scriptdir, script)
scriptexe = 'python ' + scriptpath
os.system('cp %s %s' % (script_in, scriptdir))
elif cmd_exists(script_in):
scriptexe = script_in
script = script_in
else:
raise Exception('Could not find script: %s' % script_in)
bash_scripts.append((script, scriptexe, bash_script))
for name, vdict in opt_configs.items():
dirname = os.path.abspath(os.path.join(basedir, name))
utils.mkdir(dirname)
cfgfile = os.path.join(dirname, 'config.yaml')
for script_in, bash_script in zip(scripts, bash_scripts):
runscript = os.path.splitext(bash_script[0])[0] + '.sh'
runscript = os.path.join(dirname, runscript)
with open(os.path.join(runscript), 'wt') as f:
f.write(bash_script[2].format(source=name,
scriptexe=bash_script[1],
config=cfgfile,
args=args))
if not config:
continue
c = copy.deepcopy(config)
c = utils.merge_dict(c, vdict, add_new_keys=True)
yaml.dump(utils.tolist(c),
open(cfgfile, 'w'),
default_flow_style=False)
def clone_configs(basedir, base_configs, opt_configs, scripts, args=''):
"""
"""
config = {}
for c in base_configs:
config = utils.merge_dict(config, yaml.load(open(c)),
add_new_keys=True)
scriptdir = os.path.abspath(os.path.join(basedir, 'scripts'))
utils.mkdir(scriptdir)
bash_scripts = []
for script_in in scripts:
bash_script = """
cat $0
{scriptexe} --config={config} {args}
"""
if os.path.isfile(script_in):
script = os.path.basename(script_in)
scriptpath = os.path.join(scriptdir, script)
scriptexe = 'python ' + scriptpath
os.system('cp %s %s' % (script_in, scriptdir))
elif cmd_exists(script_in):
scriptexe = script_in
script = script_in
else:
raise Exception('Could not find script: %s' % script_in)
bash_scripts.append((script, scriptexe, bash_script))
for name, vdict in opt_configs.items():
dirname = os.path.abspath(os.path.join(basedir, name))
utils.mkdir(dirname)
cfgfile = os.path.join(dirname, 'config.yaml')
for script_in, bash_script in zip(scripts, bash_scripts):
runscript = os.path.splitext(bash_script[0])[0] + '.sh'
runscript = os.path.join(dirname, runscript)
with open(os.path.join(runscript), 'wt') as f:
f.write(bash_script[2].format(source=name,
scriptexe=bash_script[1],
config=cfgfile,
args=args))
if not config:
continue
c = copy.deepcopy(config)
c = utils.merge_dict(c, vdict, add_new_keys=True)
yaml.dump(utils.tolist(c), open(cfgfile, 'w'),
default_flow_style=False)
def _process_lc_bin(itime, name, config, basedir, workdir, diff_sources, const_spectrum, roi, lck_params,
**kwargs):
i, time = itime
roi = copy.deepcopy(roi)
config = copy.deepcopy(config)
config['selection']['tmin'] = time[0]
config['selection']['tmax'] = time[1]
# create output directories labeled in MET vals
outdir = basedir + 'lightcurve_%.0f_%.0f' % (time[0], time[1])
config['fileio']['outdir'] = os.path.join(workdir, outdir)
config['logging']['prefix'] = 'lightcurve_%.0f_%.0f ' % (time[0], time[1])
config['fileio']['logfile'] = os.path.join(config['fileio']['outdir'],
'fermipy.log')
utils.mkdir(config['fileio']['outdir'])
yaml.dump(utils.tolist(config),
open(os.path.join(config['fileio']['outdir'],
'config.yaml'), 'w'))
xmlfile = os.path.join(config['fileio']['outdir'], 'base.xml')
try:
from fermipy.gtanalysis import GTAnalysis
gta = GTAnalysis(config, roi, loglevel=logging.DEBUG)
gta.logger.info('Fitting time range %i %i' % (time[0], time[1]))
gta.setup()
except:
print('Analysis failed in time range %i %i' %
(time[0], time[1]))
print(sys.exc_info()[0])
raise
return {}
gta._lck_params = lck_params
# Recompute source map for source of interest and sources within 3 deg
if gta.config['gtlike']['use_scaled_srcmap']:
names = [s.name for s in
gta.roi.get_sources(distance=3.0, skydir=gta.roi[name].skydir)
if not s.diffuse]
gta.reload_sources(names)
# Write the current model
gta.write_xml(xmlfile)
# Optimize the model
gta.optimize(skip=diff_sources,
shape_ts_threshold=kwargs.get('shape_ts_threshold'))
fit_results = _fit_lc(gta, name, **kwargs)
gta.write_xml('fit_model_final.xml')
srcmodel = copy.deepcopy(gta.get_src_model(name))
numfree = gta.get_free_param_vector().count(True)
max_ts_thresholds = [None, 4, 9]
for max_ts in max_ts_thresholds:
if max_ts is not None:
gta.free_sources(minmax_ts=[None, max_ts], free=False, exclude=[name])
# rerun fit using params from full time (constant) fit using same
# param vector as the successful fit to get loglike
specname, spectrum = const_spectrum
gta.set_source_spectrum(name, spectrum_type=specname,
spectrum_pars=spectrum,
update_source=False)
gta.free_source(name, free=False)
const_fit_results = gta.fit()
if not const_fit_results['fit_success']:
continue
const_srcmodel = gta.get_src_model(name)
# rerun using shape fixed to full time fit
# for the fixed-shape lightcurve
gta.free_source(name, pars='norm')
fixed_fit_results = gta.fit()
if not fixed_fit_results['fit_success']:
continue
fixed_srcmodel = gta.get_src_model(name)
break
# special lc output
o = {'flux_const': const_srcmodel['flux'],
'loglike_const': const_fit_results['loglike'],
'fit_success': fit_results['fit_success'],
'fit_success_fixed': fixed_fit_results['fit_success'],
'fit_quality': fit_results['fit_quality'],
'fit_status': fit_results['fit_status'],
'num_free_params': numfree,
'config': config}
# full flux output
if fit_results['fit_success'] == 1:
for k in defaults.source_flux_output.keys():
if not k in srcmodel:
continue
o[k] = srcmodel[k]
o[k+'_fixed'] = fixed_srcmodel[k]
gta.logger.info('Finished time range %i %i' % (time[0], time[1]))
return o
def _make_lc(self, name, **kwargs):
# make array of time values in MET
if kwargs['time_bins']:
times = np.array(kwargs['time_bins'])
elif kwargs['nbins']:
times = np.linspace(self.tmin, self.tmax,
kwargs['nbins'] + 1)
else:
times = np.arange(self.tmin, self.tmax,
kwargs['binsz'])
o = self._create_lc_dict(name, times)
o['config'] = kwargs
diff_sources = [s.name for s in self.roi.sources if s.diffuse]
skydir = self.roi[name].skydir
if kwargs.get('free_radius', None) is not None:
kwargs['free_sources'] += [s.name for s in
self.roi.get_sources(skydir=skydir,
distance=kwargs[
'free_radius'],
exclude=diff_sources)]
for i, time in enumerate(zip(times[:-1], times[1:])):
self.logger.info('Fitting time range %i %i', time[0], time[1])
config = copy.deepcopy(self.config)
config['selection']['tmin'] = time[0]
config['selection']['tmax'] = time[1]
config['ltcube']['use_local_ltcube'] = kwargs['use_local_ltcube']
config['model']['diffuse_dir'] = [self.workdir]
if config['components'] is None:
config['components'] = []
for j, c in enumerate(self.components):
if len(config['components']) <= j:
config['components'] += [{}]
data_cfg = {'evfile': c.files['ft1'],
'scfile': c.data_files['scfile'],
'ltcube': None}
config['components'][j] = \
utils.merge_dict(config['components'][j],
{'data': data_cfg},
add_new_keys=True)
# create output directories labeled in MET vals
outdir = 'lightcurve_%.0f_%.0f' % (time[0], time[1])
config['fileio']['outdir'] = os.path.join(self.workdir, outdir)
utils.mkdir(config['fileio']['outdir'])
yaml.dump(utils.tolist(config),
open(os.path.join(config['fileio']['outdir'],
'config.yaml'), 'w'))
xmlfile = os.path.join(config['fileio']['outdir'], 'base.xml')
# Make a copy of the source maps. TODO: Implement a
# correction to account for the difference in exposure for
# each time bin.
# for c in self.components:
# shutil.copy(c._files['srcmap'],config['fileio']['outdir'])
try:
gta = self.clone(config, loglevel=logging.DEBUG)
gta.setup()
except:
self.logger.warning('Analysis failed in time range %i %i',
time[0], time[1])
continue
# Write the current model
gta.write_xml(xmlfile)
# Optimize the model (skip diffuse?)
gta.optimize(skip=diff_sources)
fit_results = self._fit_lc(gta, name, **kwargs)
gta.write_xml('fit_model_final.xml')
output = gta.get_src_model(name)
if fit_results['fit_success'] == 1:
for k in defaults.source_flux_output.keys():
if not k in output:
continue
if (isinstance(output[k], np.ndarray) and
o[k][i].shape != output[k].shape):
self.logger.warning(
'Incompatible shape for column %s', k)
continue
o[k][i] = output[k]
self.logger.info('Finished time range %i %i', time[0], time[1])
src = self.roi.get_source_by_name(name)
return o
def _process_lc_bin(itime, name, config, basedir, workdir, diff_sources, const_spectrum, roi, lck_params,
**kwargs):
i, time = itime
roi = copy.deepcopy(roi)
config = copy.deepcopy(config)
config['selection']['tmin'] = time[0]
config['selection']['tmax'] = time[1]
# create output directories labeled in MET vals
outdir = basedir + 'lightcurve_%.0f_%.0f' % (time[0], time[1])
config['fileio']['outdir'] = os.path.join(workdir, outdir)
config['logging']['prefix'] = 'lightcurve_%.0f_%.0f ' % (time[0], time[1])
config['fileio']['logfile'] = os.path.join(config['fileio']['outdir'],
'fermipy.log')
utils.mkdir(config['fileio']['outdir'])
yaml.dump(utils.tolist(config),
open(os.path.join(config['fileio']['outdir'],
'config.yaml'), 'w'))
xmlfile = os.path.join(config['fileio']['outdir'], 'base.xml')
try:
from fermipy.gtanalysis import GTAnalysis
gta = GTAnalysis(config, roi, loglevel=logging.DEBUG)
gta.logger.info('Fitting time range %i %i' % (time[0], time[1]))
gta.setup()
except:
print('Analysis failed in time range %i %i' %
(time[0], time[1]))
print(sys.exc_info()[0])
raise
return {}
gta._lck_params = lck_params
# Recompute source map for source of interest and sources within 3 deg
if gta.config['gtlike']['use_scaled_srcmap']:
names = [s.name for s in
gta.roi.get_sources(distance=3.0, skydir=gta.roi[name].skydir)
if not s.diffuse]
gta.reload_sources(names)
# Write the current model
gta.write_xml(xmlfile)
# Optimize the model
gta.optimize(skip=diff_sources,
shape_ts_threshold=kwargs.get('shape_ts_threshold'))
fit_results = _fit_lc(gta, name, **kwargs)
gta.write_xml('fit_model_final.xml')
srcmodel = copy.deepcopy(gta.get_src_model(name))
numfree = gta.get_free_param_vector().count(True)
const_srcmodel = gta.get_src_model(name).copy()
fixed_fit_results = fit_results.copy()
fixed_srcmodel = gta.get_src_model(name).copy()
fixed_fit_results['fit_success'],fixed_srcmodel['fit_success'] = [False,False]
fixed_fit_results['fit_quality'],fixed_srcmodel['fit_quality'] = [0,0]
max_ts_thresholds = [None, 4, 9, 16, 25]
for max_ts in max_ts_thresholds:
if max_ts is not None:
gta.free_sources(minmax_ts=[None, max_ts], free=False, exclude=[name])
# rerun fit using params from full time (constant) fit using same
# param vector as the successful fit to get loglike
specname, spectrum = const_spectrum
gta.set_source_spectrum(name, spectrum_type=specname,
spectrum_pars=spectrum,
update_source=False)
gta.free_source(name, free=False)
const_fit_results = gta.fit()
if not const_fit_results['fit_success']:
continue
const_srcmodel = gta.get_src_model(name)
# rerun using shape fixed to full time fit
# for the fixed-shape lightcurve
gta.free_source(name, pars='norm')
fixed_fit_results = gta.fit()
if not fixed_fit_results['fit_success']:
continue
fixed_srcmodel = gta.get_src_model(name)
break
# special lc output
o = {'flux_const': const_srcmodel['flux'],
'loglike_const': const_fit_results['loglike'],
'fit_success': fit_results['fit_success'],
'fit_success_fixed': fixed_fit_results['fit_success'],
'fit_quality': fit_results['fit_quality'],
'fit_status': fit_results['fit_status'],
'num_free_params': numfree,
'config': config}
# full flux output
if fit_results['fit_success'] == 1:
for k in defaults.source_flux_output.keys():
if not k in srcmodel:
continue
o[k] = srcmodel[k]
o[k+'_fixed'] = fixed_srcmodel[k]
gta.logger.info('Finished time range %i %i' % (time[0], time[1]))
return o
from fermipy.utils import tolist
import yaml
import numpy as np
gtutils.init_function_pars()
par_names = gtutils.FUNCTION_PAR_NAMES
o = {}
for k, v in par_names.items():
o.setdefault(k,{})
o[k]['par_names'] = v
o[k]['norm_par'] = gtutils.FUNCTION_NORM_PARS[k]
o[k]['defaults'] = gtutils.FUNCTION_DEFAULT_PARS[k]
for pname, p in o[k]['defaults'].items():
o[k]['defaults'][pname]['error'] = np.nan
o['CompositeSource'] = {'defaults' : {}, 'norm_par' : None,
'par_names': [] }
modelfile = os.path.join('$FERMIPY_ROOT',
'data', 'models.yaml')
yaml.dump(tolist(o),
open(os.path.expandvars(modelfile), 'w'))#,default_flow_style=False)
import os
from fermipy import gtutils
from fermipy.utils import tolist
import yaml
import numpy as np
gtutils.init_function_pars()
par_names = gtutils.FUNCTION_PAR_NAMES
o = {}
for k, v in par_names.items():
o.setdefault(k, {})
o[k]['par_names'] = v
o[k]['norm_par'] = gtutils.FUNCTION_NORM_PARS[k]
o[k]['defaults'] = gtutils.FUNCTION_DEFAULT_PARS[k]
for pname, p in o[k]['defaults'].items():
o[k]['defaults'][pname]['error'] = np.nan
o['CompositeSource'] = {'defaults': {}, 'norm_par': None, 'par_names': []}
modelfile = os.path.join('$FERMIPY_DATA_DIR', 'models.yaml')
yaml.dump(tolist(o), open(os.path.expandvars(modelfile),
'w')) #,default_flow_style=False)
# else:
# name = target
# config['selection']['target'] = name
print name
dirname = os.path.join(basedir,name)
utils.mkdir(dirname)
script = os.path.basename(args.script)
scriptpath = os.path.abspath(os.path.join(dirname,script))
cfgfile = os.path.abspath(os.path.join(dirname,'config.yaml'))
os.system('ln -sf %s %s'%(os.path.abspath(os.path.join(scriptdir,script)),
scriptpath))
runscript = os.path.abspath(os.path.join(dirname,os.path.splitext(script)[0] + '.sh'))
with open(os.path.join(runscript),'wt') as f:
f.write(bash_script.format(source=name,config=cfgfile,
script=scriptpath))
if not config:
continue
c = copy.deepcopy(config)
c = utils.merge_dict(c,v,add_new_keys=True)
yaml.dump(utils.tolist(c),open(cfgfile,'w'),default_flow_style=False)