def print_summary(self, stream=sys.stdout, indent="", recurse_level=2): """Print a summary of the activity done by this `Link`. Parameters ---------- stream : `file` Stream to print to indent : str Indentation at start of line recurse_level : int Number of recursion levels to print """ Link.print_summary(self, stream, indent, recurse_level) if recurse_level > 0: recurse_level -= 1 indent += " " if self._initialize_link is not None: stream.write("\n") self._initialize_link.print_summary(stream, indent, recurse_level) stream.write("\n") self._scatter_link.print_summary(stream, indent, recurse_level) if self._gather_link is not None: stream.write("\n") self._gather_link.print_summary(stream, indent, recurse_level)
def print_summary(self, stream=sys.stdout, indent="", recurse_level=2): """Print a summary of the activity done by this `Link`. Parameters ---------- stream : `file` Stream to print to indent : str Indentation at start of line recurse_level : int Number of recursion levels to print """ Link.print_summary(self, stream, indent, recurse_level) if recurse_level > 0: recurse_level -= 1 indent += " " if self._initialize_link is not None: stream.write("\n") self._initialize_link.print_summary( stream, indent, recurse_level) stream.write("\n") self._scatter_link.print_summary(stream, indent, recurse_level) if self._gather_link is not None: stream.write("\n") self._gather_link.print_summary(stream, indent, recurse_level)
def update_args(self, override_args): """Update the argument used to invoke the application See help for `chain.Link` for details This calls the base class function then fills the parameters of the GtApp object """ Link.update_args(self, override_args) update_gtapp(self.__app, **self.args)
def update_args(self, override_args): """Update the argument used to invoke the application See help for `chain.Link` for details This calls the base class function then fills the parameters of the GtApp object """ Link.update_args(self, override_args) update_gtapp(self.__app, **self.args)
def __init__(self, **kwargs): """C'tor Keyword arguements --------------- config_maker : `ConfigMaker' Object used to translate arguments Must have functions 'add_arguments' and 'build_job_configs' Defaults to ConfigMaker() usage : str Usage string for argument parser description : str Description string for argument parser job_archive : `fermipy.job_archive.JobArchive` [optional] Archive used to track jobs and associated data products Defaults to None initialize : `fermipy.chain.Link` [optional] Link run for the initialization stage Defaults to None scatter : `fermipy.chain.Link` Link run for the scatter stage Defaults to None gather : `fermipy.chain.Link` [optional] Link run for the gather stage Defaults to None no_batch : bool If true, do not send jobs to the batch to run Defaults to False """ linkname = kwargs.pop('linkname', 'ScatterGather') self._config_maker = kwargs.pop('config_maker', None) self._usage = kwargs.pop('usage', "") self._description = kwargs.pop('description', "") self._job_archive = kwargs.pop('job_archive', None) self._initialize_link = kwargs.pop('initialize', None) self._scatter_link = kwargs.pop('scatter', None) self._gather_link = kwargs.pop('gather', None) self._no_batch = kwargs.pop('no_batch', False) options = kwargs.get('options', self.default_options.copy()) self._config_maker.add_options(options) Link.__init__(self, linkname, options=options, parser=self._make_parser(), **kwargs) self._base_config = None self._input_config = {} self._job_configs = {} self._output_config = {}
def __init__(self, **kwargs): """C'tor """ parser = argparse.ArgumentParser(usage="dmpipe-prepare-targets [options]", description="Prepare directories for target analyses") Link.__init__(self, kwargs.pop('linkname', 'prepare-targets'), parser=parser, appname='dmpipe-prepare-targets', options=TargetPreparer.default_options.copy(), **kwargs)
def __init__(self, **kwargs): """C'tor """ parser = argparse.ArgumentParser(usage="dmpipe-analyze-roi [options]", description="Run analysis of a single ROI") Link.__init__(self, kwargs.pop('linkname', 'analyze-roi'), parser=parser, appname='dmpipe-analyze-roi', options=TargetAnalysis.default_options.copy(), **kwargs)
def __init__(self, **kwargs): """C'tor """ parser = argparse.ArgumentParser(usage='dmpipe-analyze-sed', description="Extract the SED for a single target") Link.__init__(self, kwargs.pop('linkname', 'analyze-sed'), parser=parser, appname='dmpipe-analyze-sed', options=SEDAnalysis.default_options.copy(), **kwargs)
def __init__(self, **kwargs): """C'tor Keyword arguements --------------- config_maker : `ConfigMaker' Object used to translate arguments Must have functions 'add_arguments' and 'build_job_configs' Defaults to ConfigMaker() usage : str Usage string for argument parser description : str Description string for argument parser job_archive : `fermipy.job_archive.JobArchive` [optional] Archive used to track jobs and associated data products Defaults to None initialize : `fermipy.chain.Link` [optional] Link run for the initialization stage Defaults to None scatter : `fermipy.chain.Link` Link run for the scatter stage Defaults to None gather : `fermipy.chain.Link` [optional] Link run for the gather stage Defaults to None no_batch : bool If true, do not send jobs to the batch to run Defaults to False """ linkname = kwargs.pop('linkname', 'ScatterGather') self._config_maker = kwargs.pop('config_maker', None) self._usage = kwargs.pop('usage', "") self._description = kwargs.pop('description', "") self._job_archive = kwargs.pop('job_archive', None) self._initialize_link = kwargs.pop('initialize', None) self._scatter_link = kwargs.pop('scatter', None) self._gather_link = kwargs.pop('gather', None) self._no_batch = kwargs.pop('no_batch', False) options = kwargs.get('options', self.default_options.copy()) self._config_maker.add_options(options) Link.__init__(self, linkname, options=options, parser=self._make_parser(), **kwargs) self._base_config = None self._input_config = {} self._job_configs = {} self._output_config = {}
def __init__(self, **kwargs): """C'tor """ self.parser = argparse.ArgumentParser( usage="fermipy-assemble-model [options]", description="Initialize model fitting directory") Link.__init__(self, kwargs.pop('linkname', 'init-model'), appname='fermipy-init-model', options=GtInitModel.default_options.copy(), **kwargs)
def __init__(self, **kwargs): """C'tor """ parser = argparse.ArgumentParser(usage="dmpipe-plot-dm [options]", description="Make castro plots") Link.__init__(self, kwargs.pop('linkname', 'plot-dm'), parser=parser, appname='dmpipe-plot-dm', options=CastroPlotterDM.default_options.copy(), **kwargs)
def __init__(self, linkname, **kwargs): """C'tor See help for `chain.Link` for details This calls the base class c'tor then builds a GtApp object """ Link.__init__(self, linkname, **kwargs) try: self.__app = build_gtapp(self.appname, **self.args) except: raise ValueError("Failed to build link %s" % self.linkname)
def __init__(self, linkname, **kwargs): """C'tor See help for `chain.Link` for details This calls the base class c'tor then builds a GtApp object """ Link.__init__(self, linkname, **kwargs) try: self.__app = build_gtapp(self.appname, **self.args) except: raise ValueError("Failed to build link %s %s %s" % self.linkname, self.appname, self.args)
def __init__(self, **kwargs): """ C'tor to build this object from energy binning and spectral values tables. """ parser = argparse.ArgumentParser( usage="dmpipe-stack-likelihood [options]", description="Stack the likelihood from targets") Link.__init__(self, kwargs.pop('linkname', 'stack-likelihood'), parser=parser, appname='dmpipe-stack-likelihood', options=DMCastroStacker.default_options.copy(), **kwargs)
def __init__(self, **kwargs): """ C'tor to build this object from energy binning and spectral values tables. """ parser = argparse.ArgumentParser( usage="dmpipe-spec-table [options]", description="Build a table with the spectra") Link.__init__(self, kwargs.pop('linkname', 'spec-table'), parser=parser, appname=kwargs.pop('appname', 'dmpipe-spec-table'), options=DMSpecTableBuilder.default_options.copy(), **kwargs)
def __init__(self, **kwargs): """C'tor """ parser = argparse.ArgumentParser( usage="dmpipe-convert-castro [options]", description="Convert SED to DMCastroData") Link.__init__(self, kwargs.pop('linkname', 'convert-castro'), parser=parser, appname=kwargs.pop('appname', 'dmpipe-convert-castro'), options=DMCastroConvertor.default_options.copy(), file_args=dict(), **kwargs)
def __init__(self, **kwargs): """C'tor """ self.parser = argparse.ArgumentParser( usage="fermipy-assemble-model [options]", description= "Copy source maps from the library to a analysis directory") Link.__init__(self, kwargs.pop('linkname', 'assemble-model'), appname='fermipy-assemble-model', options=GtAssembleModel.default_options.copy(), file_args=dict(input=FileFlags.input_mask), **kwargs)
def __init__(self, **kwargs): """C'tor """ parser = argparse.ArgumentParser(usage="fermipy-srcmaps-diffuse [options]", description="Run gtsrcmaps for one or more energy planes for a single source") Link.__init__(self, kwargs.pop('linkname', 'srcmaps-diffuse'), parser=parser, appname='fermipy-srcmaps-diffuse', options=GtSrcmapPartial.default_options.copy(), file_args=dict(expcube=FileFlags.input_mask, cmap=FileFlags.input_mask, bexpmap=FileFlags.input_mask, srcmdl=FileFlags.input_mask, outfile=FileFlags.output_mask))
def __init__(self, **kwargs): """C'tor """ parser = argparse.ArgumentParser(usage="fermipy-srcmaps-catalog [options]", description="Run gtsrcmaps for all the sources in a catalog") Link.__init__(self, kwargs.pop('linkname', 'srcmaps-diffuse'), parser=parser, appname='fermipy-srcmaps-catalog', options=GtSrcmapsCatalog.default_options.copy(), file_args=dict(expcube=FileFlags.input_mask, cmap=FileFlags.input_mask, bexpmap=FileFlags.input_mask, srcmdl=FileFlags.input_mask, outfile=FileFlags.output_mask))
def run_argparser(self, argv): """Initialize a link with a set of arguments using argparser """ args = Link.run_argparser(self, argv) for link in self._links.values(): link.run_link(stream=sys.stdout, dry_run=True) return args
def run_argparser(self, argv): """Initialize a link with a set of arguments using argparser """ args = Link.run_argparser(self, argv) for link in self._links.values(): link.run_link(stream=sys.stdout, dry_run=True) return args
def test_comlink(): kwargs = dict(appname='dummy', options=dict(arg_float=(4.0, 'a float', float), arg_in=('test.in', 'an input file', str), arg_out=('test.out', 'an output file', str)), mapping=dict(arg_in='alias'), file_args=dict(arg_in=FileFlags.input_mask, arg_out=FileFlags.output_mask)) link = Link('link', **kwargs)
def __init__(self, **kwargs): """C'tor """ parser = argparse.ArgumentParser( usage='fermipy-residual-cr', description="Compute the residual cosmic-ray contamination.") Link.__init__(self, kwargs.pop('linkname', 'residual_cr'), appname='fermipy-residual-cr', options=ResidualCRAnalysis.default_options.copy(), parser=parser, file_args=dict(ccube_dirty=FileFlags.input_mask, bexpcube_dirty=FileFlags.input_mask, ccube_clean=FileFlags.input_mask, bexpcube_clean=FileFlags.input_mask, outfile=FileFlags.output_mask), **kwargs)
def _make_coadd_links(self, do_ltsum): """Make the links to run fermipy-coadd for each energy bin X psf type """ links = [] for key_e, comp_e in sorted(self.comp_dict.items()): if comp_e.has_key('mktimefilters'): mktimelist = comp_e['mktimefilters'] else: mktimelist = ['none'] if comp_e.has_key('evtclasses'): evtclasslist = comp_e['evtclasses'] else: evtclasslist = ['default'] for mktimekey in mktimelist: if do_ltsum: ltsum_listfile = 'ltsumlist_%s_%s' % (key_e, mktimekey) ltsum_outfile = 'ltsum_%s_%s' % (key_e, mktimekey) link_ltsum = Gtlink( 'ltsum_%s_%s' % (key_e, mktimekey), appname='gtltsum', mapping={ 'infile1': ltsum_listfile, 'outfile': ltsum_outfile }, options=dict( infile1=(None, "Livetime cube 1 or list of files", str), infile2=("none", "Livetime cube 2", str), outfile=(None, "Output file", str)), file_args=dict(infile1=FileFlags.input_mask, outfile=FileFlags.output_mask)) links.append(link_ltsum) for evtclass in evtclasslist: for psf_type in sorted(comp_e['psf_types'].keys()): key = "%s_%s_%s_%s" % (key_e, mktimekey, evtclass, psf_type) binkey = 'binfile_%s' % key argkey = 'args_%s' % key self.files.file_args[argkey] = FileFlags.gz_mask link = Link( 'coadd_%s' % key, appname='fermipy-coadd', options=dict(args=([], "List of input files", list), output=(None, "Output file", str)), mapping={ 'args': argkey, 'output': binkey }, file_args=dict(args=FileFlags.input_mask, output=FileFlags.output_mask)) links.append(link) return links
def create_link_fermipy_coadd(**kwargs): """Make a `fermipy.jobs.Link` object to run fermipy-coadd """ link = Link(linkname=kwargs.pop('linkname', 'fermipy-coadd'), appname='fermipy-coadd', options=dict(args=([], "List of input files", list), output=(None, "Output file", str)), file_args=dict(args=FileFlags.input_mask, output=FileFlags.output_mask), **kwargs) return link
def create_link_fermipy_vstack(**kwargs): """Make a `fermipy.jobs.Link` object to run fermipy-vstack """ link = Link(linkname=kwargs.pop('linkname', 'fermipy-vstack'), appname='fermipy-vstack', options=dict(output=(None, "Output file name", str), hdu=(None, "Name of HDU to stack", str), args=([], "List of input files", list), gzip=(False, "Compress output", bool)), file_args=dict(args=FileFlags.input_mask, output=FileFlags.output_mask), **kwargs) return link
def create_link_fermipy_healview(**kwargs): """Make a `fermipy.jobs.Link` object to run fermipy-healview """ link = Link(linkname=kwargs.pop('linkname', 'fermipy-healview'), appname='fermipy-healview', options=dict(input=(None, "Input file", str), output=(None, "Output file name", str), extension=(None, "FITS HDU with HEALPix map", str), zscale=("log", "Scaling for color scale", str)), file_args=dict(args=FileFlags.input_mask, output=FileFlags.output_mask), **kwargs) return link
def create_link_fermipy_gather_srcmaps(**kwargs): """Make a `fermipy.jobs.Link` object to run fermipy-gather-srcmaps """ link = Link(linkname=kwargs.pop('linkname', 'fermipy-gather-srcmaps'), appname='fermipy-gather-srcmaps', options=dict(output=(None, "Output file name", str), args=([], "List of input files", list), gzip=(False, "Compress output", bool), rm=(False, "Remove input files", bool), clobber=(False, "Overwrite output", bool)), file_args=dict(args=FileFlags.input_mask, output=FileFlags.output_mask), **kwargs) return link
def make_link(**kwargs): """Make a `fermipy.jobs.Link object to run `GtMergeSourceMaps` """ link = Link(kwargs.pop('linkname', 'merge-srcmaps'), appname='fermipy-merge-srcmaps', options=GtMergeSourceMaps.default_options.copy(), file_args=dict(expcube=FileFlags.input_mask, cmap=FileFlags.input_mask, bexpmap=FileFlags.input_mask, srcmdl=FileFlags.input_mask, outfile=FileFlags.output_mask, outxml=FileFlags.output_mask), **kwargs) return link
def test_chain(): kwargs = dict(appname='dummy', options=dict(arg_float=(4.0, 'a float', float), arg_in=('test.in', 'an input file', str), arg_out=('test.out', 'an output file', str)), mapping=dict(arg_in='alias'), file_args=dict(arg_in=FileFlags.input_mask, arg_out=FileFlags.output_mask)) link = Link('link', **kwargs) kwargs = dict(options=dict(irfs=('CALDB', 'IRF version', str), expcube=(None, 'Livetime cube file', str), bexpmap=(None, 'Binned exposure map', str), cmap=(None, 'Binned counts map', str), srcmdl=(None, 'Input source model xml file', str), outfile=(None, 'Output file', str)), file_args=dict(expcube=FileFlags.input_mask, cmap=FileFlags.input_mask, bexpmap=FileFlags.input_mask, srcmdl=FileFlags.input_mask, outfile=FileFlags.output_mask)) # This should be a Gtlink, but we only really wanna test the chain functionality here link2 = Link('gtsrcmaps', **kwargs) def argmapper(args): basename = args['basename'] ret_dict = dict(expcube="%s_ltcube.fits"%basename, cmap="%s_ccube.fits"%basename, bexpmap="%s_bexpmap.fits"%basename, srcmdl="%s_srcmdl.xml"%basename) return ret_dict chain = Chain('chain', links=[link, link2], options=dict(basename=('dummy', 'Base file name', str)), argmapper=argmapper)
def test_job_archive(): link = Link('test', appname='test_app', options=dict(optstr='CALDB', infile1=None, infile2=None, infile3=None, outfile1=None, outfile2=None), input_file_args=['infile1', 'infile2', 'infile3'], output_file_args=['outfile1', 'outfile2']) job_archive = JobArchive(file_archive_table='archive_files.fits', job_archive_table='archive_jobs.fits', base_path=os.path.abspath('.')) job_archive._file_archive.register_file('input1_1.fits', 0) job_archive._file_archive.register_file('input1_2.fits', 0) job_archive._file_archive.register_file('input1_3.fits', 0) job_archive._file_archive.register_file('input2_1.fits', 0) job_archive._file_archive.register_file('input2_2.fits', 0) job_archive._file_archive.register_file('input2_3.fits', 0) config_1 = dict(infile1='input1_1.fits', infile2='input1_2.fits', infile3='input1_3.fits', outfile1='output1_1.fits', outfile2='output1_2.fits') config_2 = dict(infile1='input2_1.fits', infile2='input2_2.fits', infile3='input2_3.fits', outfile1='output2_1.fits', outfile2='output2_2.fits') link.update_args(config_1) job = job_archive.register_job_from_link(link, 'dummy1', logfile='dummy1.log') link.update_args(config_2) job2 = job_archive.register_job_from_link(link, 'dummy2', logfile='dummy2.log')
def test_job_archive(): link = Link('test', appname='test_app', options=dict(optstr='CALDB', infile1=None, infile2=None, infile3=None, outfile1=None, outfile2=None), input_file_args=['infile1', 'infile2', 'infile3'], output_file_args=['outfile1', 'outfile2']) job_archive = JobArchive(file_archive_table='archive_files.fits', job_archive_table='archive_jobs.fits', base_path=os.path.abspath('.')) job_archive._file_archive.register_file('input1_1.fits', 0) job_archive._file_archive.register_file('input1_2.fits', 0) job_archive._file_archive.register_file('input1_3.fits', 0) job_archive._file_archive.register_file('input2_1.fits', 0) job_archive._file_archive.register_file('input2_2.fits', 0) job_archive._file_archive.register_file('input2_3.fits', 0) config_1 = dict(infile1='input1_1.fits', infile2='input1_2.fits', infile3='input1_3.fits', outfile1='output1_1.fits', outfile2='output1_2.fits') config_2 = dict(infile1='input2_1.fits', infile2='input2_2.fits', infile3='input2_3.fits', outfile1='output2_1.fits', outfile2='output2_2.fits') link.update_args(config_1) job = job_archive.register_job_from_link(link, 'dummy1', logfile='dummy1.log') link.update_args(config_2) job2 = job_archive.register_job_from_link(link, 'dummy2', logfile='dummy2.log')
class SplitAndMktime_SG(ScatterGather): """Small class to generate configurations for SplitAndMktime """ appname = 'fermipy-split-and-mktime-sg' usage = "%s [options]" % (appname) description = "Prepare data for diffuse all-sky analysis" clientclass = SplitAndMktime job_time = 1500 default_options = dict( comp=diffuse_defaults.diffuse['comp'], data=diffuse_defaults.diffuse['data'], hpx_order_max=diffuse_defaults.diffuse['hpx_order_ccube'], ft1file=diffuse_defaults.diffuse['ft1file'], ft2file=diffuse_defaults.diffuse['ft2file'], do_ltsum=diffuse_defaults.diffuse['do_ltsum'], scratch=diffuse_defaults.diffuse['scratch'], dry_run=diffuse_defaults.diffuse['dry_run']) __doc__ += Link.construct_docstring(default_options) def build_job_configs(self, args): """Hook to build job configurations """ job_configs = {} comp_file = args.get('comp', None) if comp_file is not None: comp_dict = yaml.safe_load(open(comp_file)) coordsys = comp_dict.pop('coordsys') for v in comp_dict.values(): v['coordsys'] = coordsys else: return job_configs datafile = args['data'] if datafile is None or datafile == 'None': return job_configs NAME_FACTORY.update_base_dict(args['data']) inputfiles = create_inputlist(args['ft1file']) outdir_base = os.path.join(NAME_FACTORY.base_dict['basedir'], 'counts_cubes') ft2file = os.path.join(NAME_FACTORY.base_dict['basedir'], args['ft2file']) data_ver = NAME_FACTORY.base_dict['data_ver'] for idx, infile in enumerate(inputfiles): key = "%06i" % idx key_scfile = "%03i" % (idx + 1) output_dir = os.path.join(outdir_base, key) try: os.mkdir(output_dir) except OSError: pass scfile = args['ft2file'].replace('.lst', '_%s.fits' % key_scfile) logfile = make_nfs_path( os.path.join(output_dir, 'scatter_mk_%s_%s.log' % (data_ver, key))) job_configs[key] = comp_dict.copy() job_configs[key].update( dict(ft1file=infile, ft2file=ft2file, data=args['data'], scfile=scfile, comp=args['comp'], hpx_order_max=args['hpx_order_max'], outdir=outdir_base, outkey=key, logfile=logfile, pfiles=output_dir)) return job_configs
class SplitAndMktime(Chain): """Small class to split, apply mktime and bin data according to some user-provided specification This chain consists multiple `Link` objects: select-energy-EBIN-ZCUT : `Gtlink_select` Initial splitting by energy bin and zenith angle cut mktime-EBIN-ZCUT-FILTER : `Gtlink_mktime` Application of gtmktime filter for zenith angle cut ltcube-EBIN-ZCUT-FILTER : `Gtlink_ltcube` Computation of livetime cube for zenith angle cut select-type-EBIN-ZCUT-FILTER-TYPE : `Gtlink_select` Refinement of selection from event types bin-EBIN-ZCUT-FILTER-TYPE : `Gtlink_bin` Final binning of the data for each event type """ appname = 'fermipy-split-and-mktime' linkname_default = 'split-and-mktime' usage = '%s [options]' % (appname) description = 'Run gtselect and gtbin together' default_options = dict( comp=diffuse_defaults.diffuse['comp'], data=diffuse_defaults.diffuse['data'], hpx_order_max=diffuse_defaults.diffuse['hpx_order_ccube'], ft1file=diffuse_defaults.diffuse['ft1file'], ft2file=diffuse_defaults.diffuse['ft2file'], evclass=(128, 'Event class bit mask', int), outdir=('counts_cubes', 'Output directory', str), outkey=(None, 'Key for this particular output file', str), pfiles=(None, 'Directory for .par files', str), do_ltsum=(False, 'Sum livetime cube files', bool), scratch=(None, 'Scratch area', str), dry_run=(False, 'Print commands but do not run them', bool)) __doc__ += Link.construct_docstring(default_options) def __init__(self, **kwargs): """C'tor """ super(SplitAndMktime, self).__init__(**kwargs) self.comp_dict = None def _map_arguments(self, args): """Map from the top-level arguments to the arguments provided to the indiviudal links """ comp_file = args.get('comp', None) datafile = args.get('data', None) if is_null(comp_file): return if is_null(datafile): return NAME_FACTORY.update_base_dict(args['data']) outdir = args.get('outdir') outkey = args.get('outkey') ft1file = args['ft1file'] ft2file = args['ft2file'] if is_null(outdir) or is_null(outkey): return pfiles = os.path.join(outdir, outkey) self.comp_dict = yaml.safe_load(open(comp_file)) coordsys = self.comp_dict.pop('coordsys') full_out_dir = make_nfs_path(os.path.join(outdir, outkey)) for key_e, comp_e in sorted(self.comp_dict.items()): if 'logebins' in comp_e: ebins_file = make_nfs_path( os.path.join(full_out_dir, 'energy_bins.fits')) write_ebins_file(ebins_file, comp_e['logebins']) else: ebins_file = None enumbins = comp_e['enumbins'] emin = math.pow(10., comp_e['log_emin']) emax = math.pow(10., comp_e['log_emax']) zmax = comp_e['zmax'] zcut = "zmax%i" % comp_e['zmax'] tmin = comp_e.get('tmin', None) tmax = comp_e.get('tmax', None) if is_null(tmin): tmin = 'INDEF' if is_null(tmax): tmax = 'INDEF' evclassstr = NAME_FACTORY.base_dict['evclass'] kwargs_select = dict(zcut=zcut, ebin=key_e, psftype='ALL', coordsys=coordsys) linkname = 'select-energy-%s-%s' % (key_e, zcut) selectfile_energy = make_full_path( outdir, outkey, NAME_FACTORY.select(**kwargs_select)) self._set_link(linkname, Gtlink_select, infile=ft1file, outfile=selectfile_energy, zmax=zmax, tmin=tmin, tmax=tmax, emin=emin, emax=emax, evclass=NAME_FACTORY.evclassmask(evclassstr), pfiles=pfiles, logfile=os.path.join(full_out_dir, "%s.log" % linkname)) if 'mktimefilters' in comp_e: mktimefilters = comp_e['mktimefilters'] else: mktimefilters = ['none'] for mktimekey in mktimefilters: kwargs_mktime = kwargs_select.copy() kwargs_mktime['mktime'] = mktimekey filterstring = MKTIME_DICT[mktimekey] mktime_file = make_full_path( outdir, outkey, NAME_FACTORY.mktime(**kwargs_mktime)) ltcube_file = make_full_path( outdir, outkey, NAME_FACTORY.ltcube(**kwargs_mktime)) linkname_mktime = 'mktime-%s-%s-%s' % (key_e, zcut, mktimekey) linkname_ltcube = 'ltcube-%s-%s-%s' % (key_e, zcut, mktimekey) self._set_link(linkname_mktime, Gtlink_mktime, evfile=selectfile_energy, outfile=mktime_file, scfile=ft2file, filter=filterstring, pfiles=pfiles, logfile=os.path.join(full_out_dir, "%s.log" % linkname_mktime)) self._set_link(linkname_ltcube, Gtlink_ltcube, evfile=mktime_file, outfile=ltcube_file, scfile=ft2file, zmax=zmax, pfiles=pfiles, logfile=os.path.join(full_out_dir, "%s.log" % linkname_ltcube)) if 'evtclasses' in comp_e: evtclasslist_vals = comp_e['evtclasses'] else: evtclasslist_vals = [NAME_FACTORY.base_dict['evclass']] for evtclassval in evtclasslist_vals: for psf_type, psf_dict in sorted( comp_e['psf_types'].items()): linkname_select = 'select-type-%s-%s-%s-%s-%s' % ( key_e, zcut, mktimekey, evtclassval, psf_type) linkname_bin = 'bin-%s-%s-%s-%s-%s' % ( key_e, zcut, mktimekey, evtclassval, psf_type) kwargs_bin = kwargs_mktime.copy() kwargs_bin['psftype'] = psf_type kwargs_bin['coordsys'] = coordsys kwargs_bin['evclass'] = evtclassval selectfile_psf = make_full_path( outdir, outkey, NAME_FACTORY.select(**kwargs_bin)) binfile_psf = make_full_path( outdir, outkey, NAME_FACTORY.ccube(**kwargs_bin)) hpx_order_psf = min(args['hpx_order_max'], psf_dict['hpx_order']) linkname_select = 'select-type-%s-%s-%s-%s-%s' % ( key_e, zcut, mktimekey, evtclassval, psf_type) linkname_bin = 'bin-%s-%s-%s-%s-%s' % ( key_e, zcut, mktimekey, evtclassval, psf_type) self._set_link( linkname_select, Gtlink_select, infile=selectfile_energy, outfile=selectfile_psf, zmax=zmax, emin=emin, emax=emax, tmin=tmin, tmax=tmax, evtype=EVT_TYPE_DICT[psf_type], evclass=NAME_FACTORY.evclassmask(evtclassval), pfiles=pfiles, logfile=os.path.join(full_out_dir, "%s.log" % linkname_select)) print("xx", ebins_file) if ebins_file is None: self._set_link(linkname_bin, Gtlink_bin, coordsys=coordsys, hpx_order=hpx_order_psf, evfile=selectfile_psf, outfile=binfile_psf, emin=emin, emax=emax, enumbins=enumbins, pfiles=pfiles, logfile=os.path.join( full_out_dir, "%s.log" % linkname_bin)) else: self._set_link(linkname_bin, Gtlink_bin, coordsys=coordsys, hpx_order=hpx_order_psf, evfile=selectfile_psf, outfile=binfile_psf, ebinalg='FILE', ebinfile=ebins_file, pfiles=pfiles, logfile=os.path.join( full_out_dir, "%s.log" % linkname_bin))
class SplitAndMktimeChain(Chain): """Chain to run split and mktime and then make livetime and exposure cubes This chain consists of: split-and-mktime : `SplitAndMkTime_SG` Chain to make the binned counts maps for each input file coadd-split : `CoaddSplit_SG` Link to co-add the binnec counts maps files ltsum : `Gtltsum_SG` Link to co-add the livetime cube files expcube2 : `Gtexpcube2_SG` Link to make the corresponding binned exposure maps """ appname = 'fermipy-split-and-mktime-chain' linkname_default = 'split-and-mktime-chain' usage = '%s [options]' % (appname) description = 'Run split-and-mktime, coadd-split and exposure' default_options = dict( data=diffuse_defaults.diffuse['data'], comp=diffuse_defaults.diffuse['comp'], ft1file=diffuse_defaults.diffuse['ft1file'], ft2file=diffuse_defaults.diffuse['ft2file'], hpx_order_ccube=diffuse_defaults.diffuse['hpx_order_ccube'], hpx_order_expcube=diffuse_defaults.diffuse['hpx_order_expcube'], do_ltsum=diffuse_defaults.diffuse['do_ltsum'], scratch=diffuse_defaults.diffuse['scratch'], dry_run=diffuse_defaults.diffuse['dry_run']) __doc__ += Link.construct_docstring(default_options) def __init__(self, **kwargs): """C'tor """ super(SplitAndMktimeChain, self).__init__(**kwargs) self.comp_dict = None def _map_arguments(self, args): """Map from the top-level arguments to the arguments provided to the indiviudal links """ data = args.get('data') comp = args.get('comp') ft1file = args.get('ft1file') ft2file = args.get('ft2file') scratch = args.get('scratch', None) dry_run = args.get('dry_run', None) self._set_link('split-and-mktime', SplitAndMktime_SG, comp=comp, data=data, hpx_order_max=args.get('hpx_order_ccube', 9), ft1file=ft1file, ft2file=ft2file, do_ltsum=args.get('do_ltsum', False), scratch=scratch, dry_run=dry_run) self._set_link('coadd-split', CoaddSplit_SG, comp=comp, data=data, ft1file=ft1file) self._set_link('ltsum', Gtltsum_SG, comp=comp, data=data, ft1file=args['ft1file'], dry_run=dry_run) self._set_link('expcube2', Gtexpcube2_SG, comp=comp, data=data, hpx_order_max=args.get('hpx_order_expcube', 5), dry_run=dry_run)