class SplitAndMktimeChain(Chain):
"""Chain to run split and mktime and then make livetime and exposure cubes
This chain consists of:
split-and-mktime : `SplitAndMkTime_SG`
Chain to make the binned counts maps for each input file
coadd-split : `CoaddSplit_SG`
Link to co-add the binnec counts maps files
ltsum : `Gtltsum_SG`
Link to co-add the livetime cube files
expcube2 : `Gtexpcube2_SG`
Link to make the corresponding binned exposure maps
"""
appname = 'fermipy-split-and-mktime-chain'
linkname_default = 'split-and-mktime-chain'
usage = '%s [options]' % (appname)
description = 'Run split-and-mktime, coadd-split and exposure'
default_options = dict(
data=diffuse_defaults.diffuse['data'],
comp=diffuse_defaults.diffuse['comp'],
ft1file=diffuse_defaults.diffuse['ft1file'],
ft2file=diffuse_defaults.diffuse['ft2file'],
hpx_order_ccube=diffuse_defaults.diffuse['hpx_order_ccube'],
hpx_order_expcube=diffuse_defaults.diffuse['hpx_order_expcube'],
do_ltsum=diffuse_defaults.diffuse['do_ltsum'],
scratch=diffuse_defaults.diffuse['scratch'],
dry_run=diffuse_defaults.diffuse['dry_run'])
__doc__ += Link.construct_docstring(default_options)
def __init__(self, **kwargs):
"""C'tor
"""
super(SplitAndMktimeChain, self).__init__(**kwargs)
self.comp_dict = None
def _map_arguments(self, args):
"""Map from the top-level arguments to the arguments provided to
the indiviudal links """
data = args.get('data')
comp = args.get('comp')
ft1file = args.get('ft1file')
ft2file = args.get('ft2file')
scratch = args.get('scratch', None)
dry_run = args.get('dry_run', None)
self._set_link('split-and-mktime',
SplitAndMktime_SG,
comp=comp,
data=data,
hpx_order_max=args.get('hpx_order_ccube', 9),
ft1file=ft1file,
ft2file=ft2file,
do_ltsum=args.get('do_ltsum', False),
scratch=scratch,
dry_run=dry_run)
self._set_link('coadd-split',
CoaddSplit_SG,
comp=comp,
data=data,
ft1file=ft1file)
self._set_link('ltsum',
Gtltsum_SG,
comp=comp,
data=data,
ft1file=args['ft1file'],
dry_run=dry_run)
self._set_link('expcube2',
Gtexpcube2_SG,
comp=comp,
data=data,
hpx_order_max=args.get('hpx_order_expcube', 5),
dry_run=dry_run)
class SplitAndMktime(Chain):
"""Small class to split, apply mktime and bin data according to some user-provided specification
This chain consists multiple `Link` objects:
select-energy-EBIN-ZCUT : `Gtlink_select`
Initial splitting by energy bin and zenith angle cut
mktime-EBIN-ZCUT-FILTER : `Gtlink_mktime`
Application of gtmktime filter for zenith angle cut
ltcube-EBIN-ZCUT-FILTER : `Gtlink_ltcube`
Computation of livetime cube for zenith angle cut
select-type-EBIN-ZCUT-FILTER-TYPE : `Gtlink_select`
Refinement of selection from event types
bin-EBIN-ZCUT-FILTER-TYPE : `Gtlink_bin`
Final binning of the data for each event type
"""
appname = 'fermipy-split-and-mktime'
linkname_default = 'split-and-mktime'
usage = '%s [options]' % (appname)
description = 'Run gtselect and gtbin together'
default_options = dict(
comp=diffuse_defaults.diffuse['comp'],
data=diffuse_defaults.diffuse['data'],
hpx_order_max=diffuse_defaults.diffuse['hpx_order_ccube'],
ft1file=diffuse_defaults.diffuse['ft1file'],
ft2file=diffuse_defaults.diffuse['ft2file'],
evclass=(128, 'Event class bit mask', int),
outdir=('counts_cubes', 'Output directory', str),
outkey=(None, 'Key for this particular output file', str),
pfiles=(None, 'Directory for .par files', str),
do_ltsum=(False, 'Sum livetime cube files', bool),
scratch=(None, 'Scratch area', str),
dry_run=(False, 'Print commands but do not run them', bool))
__doc__ += Link.construct_docstring(default_options)
def __init__(self, **kwargs):
"""C'tor
"""
super(SplitAndMktime, self).__init__(**kwargs)
self.comp_dict = None
def _map_arguments(self, args):
"""Map from the top-level arguments to the arguments provided to
the indiviudal links """
comp_file = args.get('comp', None)
datafile = args.get('data', None)
if is_null(comp_file):
return
if is_null(datafile):
return
NAME_FACTORY.update_base_dict(args['data'])
outdir = args.get('outdir')
outkey = args.get('outkey')
ft1file = args['ft1file']
ft2file = args['ft2file']
if is_null(outdir) or is_null(outkey):
return
pfiles = os.path.join(outdir, outkey)
self.comp_dict = yaml.safe_load(open(comp_file))
coordsys = self.comp_dict.pop('coordsys')
full_out_dir = make_nfs_path(os.path.join(outdir, outkey))
for key_e, comp_e in sorted(self.comp_dict.items()):
if 'logebins' in comp_e:
ebins_file = make_nfs_path(
os.path.join(full_out_dir, 'energy_bins.fits'))
write_ebins_file(ebins_file, comp_e['logebins'])
else:
ebins_file = None
enumbins = comp_e['enumbins']
emin = math.pow(10., comp_e['log_emin'])
emax = math.pow(10., comp_e['log_emax'])
zmax = comp_e['zmax']
zcut = "zmax%i" % comp_e['zmax']
tmin = comp_e.get('tmin', None)
tmax = comp_e.get('tmax', None)
if is_null(tmin):
tmin = 'INDEF'
if is_null(tmax):
tmax = 'INDEF'
evclassstr = NAME_FACTORY.base_dict['evclass']
kwargs_select = dict(zcut=zcut,
ebin=key_e,
psftype='ALL',
coordsys=coordsys)
linkname = 'select-energy-%s-%s' % (key_e, zcut)
selectfile_energy = make_full_path(
outdir, outkey, NAME_FACTORY.select(**kwargs_select))
self._set_link(linkname,
Gtlink_select,
infile=ft1file,
outfile=selectfile_energy,
zmax=zmax,
tmin=tmin,
tmax=tmax,
emin=emin,
emax=emax,
evclass=NAME_FACTORY.evclassmask(evclassstr),
pfiles=pfiles,
logfile=os.path.join(full_out_dir,
"%s.log" % linkname))
if 'mktimefilters' in comp_e:
mktimefilters = comp_e['mktimefilters']
else:
mktimefilters = ['none']
for mktimekey in mktimefilters:
kwargs_mktime = kwargs_select.copy()
kwargs_mktime['mktime'] = mktimekey
filterstring = MKTIME_DICT[mktimekey]
mktime_file = make_full_path(
outdir, outkey, NAME_FACTORY.mktime(**kwargs_mktime))
ltcube_file = make_full_path(
outdir, outkey, NAME_FACTORY.ltcube(**kwargs_mktime))
linkname_mktime = 'mktime-%s-%s-%s' % (key_e, zcut, mktimekey)
linkname_ltcube = 'ltcube-%s-%s-%s' % (key_e, zcut, mktimekey)
self._set_link(linkname_mktime,
Gtlink_mktime,
evfile=selectfile_energy,
outfile=mktime_file,
scfile=ft2file,
filter=filterstring,
pfiles=pfiles,
logfile=os.path.join(full_out_dir, "%s.log" %
linkname_mktime))
self._set_link(linkname_ltcube,
Gtlink_ltcube,
evfile=mktime_file,
outfile=ltcube_file,
scfile=ft2file,
zmax=zmax,
pfiles=pfiles,
logfile=os.path.join(full_out_dir, "%s.log" %
linkname_ltcube))
if 'evtclasses' in comp_e:
evtclasslist_vals = comp_e['evtclasses']
else:
evtclasslist_vals = [NAME_FACTORY.base_dict['evclass']]
for evtclassval in evtclasslist_vals:
for psf_type, psf_dict in sorted(
comp_e['psf_types'].items()):
linkname_select = 'select-type-%s-%s-%s-%s-%s' % (
key_e, zcut, mktimekey, evtclassval, psf_type)
linkname_bin = 'bin-%s-%s-%s-%s-%s' % (
key_e, zcut, mktimekey, evtclassval, psf_type)
kwargs_bin = kwargs_mktime.copy()
kwargs_bin['psftype'] = psf_type
kwargs_bin['coordsys'] = coordsys
kwargs_bin['evclass'] = evtclassval
selectfile_psf = make_full_path(
outdir, outkey, NAME_FACTORY.select(**kwargs_bin))
binfile_psf = make_full_path(
outdir, outkey, NAME_FACTORY.ccube(**kwargs_bin))
hpx_order_psf = min(args['hpx_order_max'],
psf_dict['hpx_order'])
linkname_select = 'select-type-%s-%s-%s-%s-%s' % (
key_e, zcut, mktimekey, evtclassval, psf_type)
linkname_bin = 'bin-%s-%s-%s-%s-%s' % (
key_e, zcut, mktimekey, evtclassval, psf_type)
self._set_link(
linkname_select,
Gtlink_select,
infile=selectfile_energy,
outfile=selectfile_psf,
zmax=zmax,
emin=emin,
emax=emax,
tmin=tmin,
tmax=tmax,
evtype=EVT_TYPE_DICT[psf_type],
evclass=NAME_FACTORY.evclassmask(evtclassval),
pfiles=pfiles,
logfile=os.path.join(full_out_dir,
"%s.log" % linkname_select))
print("xx", ebins_file)
if ebins_file is None:
self._set_link(linkname_bin,
Gtlink_bin,
coordsys=coordsys,
hpx_order=hpx_order_psf,
evfile=selectfile_psf,
outfile=binfile_psf,
emin=emin,
emax=emax,
enumbins=enumbins,
pfiles=pfiles,
logfile=os.path.join(
full_out_dir,
"%s.log" % linkname_bin))
else:
self._set_link(linkname_bin,
Gtlink_bin,
coordsys=coordsys,
hpx_order=hpx_order_psf,
evfile=selectfile_psf,
outfile=binfile_psf,
ebinalg='FILE',
ebinfile=ebins_file,
pfiles=pfiles,
logfile=os.path.join(
full_out_dir,
"%s.log" % linkname_bin))
class SplitAndMktime_SG(ScatterGather):
"""Small class to generate configurations for SplitAndMktime
"""
appname = 'fermipy-split-and-mktime-sg'
usage = "%s [options]" % (appname)
description = "Prepare data for diffuse all-sky analysis"
clientclass = SplitAndMktime
job_time = 1500
default_options = dict(
comp=diffuse_defaults.diffuse['comp'],
data=diffuse_defaults.diffuse['data'],
hpx_order_max=diffuse_defaults.diffuse['hpx_order_ccube'],
ft1file=diffuse_defaults.diffuse['ft1file'],
ft2file=diffuse_defaults.diffuse['ft2file'],
do_ltsum=diffuse_defaults.diffuse['do_ltsum'],
scratch=diffuse_defaults.diffuse['scratch'],
dry_run=diffuse_defaults.diffuse['dry_run'])
__doc__ += Link.construct_docstring(default_options)
def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
comp_file = args.get('comp', None)
if comp_file is not None:
comp_dict = yaml.safe_load(open(comp_file))
coordsys = comp_dict.pop('coordsys')
for v in comp_dict.values():
v['coordsys'] = coordsys
else:
return job_configs
datafile = args['data']
if datafile is None or datafile == 'None':
return job_configs
NAME_FACTORY.update_base_dict(args['data'])
inputfiles = create_inputlist(args['ft1file'])
outdir_base = os.path.join(NAME_FACTORY.base_dict['basedir'],
'counts_cubes')
ft2file = os.path.join(NAME_FACTORY.base_dict['basedir'],
args['ft2file'])
data_ver = NAME_FACTORY.base_dict['data_ver']
for idx, infile in enumerate(inputfiles):
key = "%06i" % idx
key_scfile = "%03i" % (idx + 1)
output_dir = os.path.join(outdir_base, key)
try:
os.mkdir(output_dir)
except OSError:
pass
scfile = args['ft2file'].replace('.lst', '_%s.fits' % key_scfile)
logfile = make_nfs_path(
os.path.join(output_dir,
'scatter_mk_%s_%s.log' % (data_ver, key)))
job_configs[key] = comp_dict.copy()
job_configs[key].update(
dict(ft1file=infile,
ft2file=ft2file,
data=args['data'],
scfile=scfile,
comp=args['comp'],
hpx_order_max=args['hpx_order_max'],
outdir=outdir_base,
outkey=key,
logfile=logfile,
pfiles=output_dir))
return job_configs