def _write_data_target_config(cls, base_config, target, target_dir): """ Write a fermipy configuration file for one target. Parameters ---------- base_config : dict Baseline configuration target : `dict` Specific target target_dir : str Directory to write to Returns ------- output : dict The configuration for this specific target """ target_config_path = os.path.join(target_dir, 'config.yaml') target_config = base_config.copy() target_config['selection']['ra'] = target['ra'] target_config['selection']['dec'] = target['dec'] write_yaml(target_config, target_config_path) return target_config
def _write_astro_value_yaml(cls, target, astro_val_path): """Write a yaml file describing the J-factor and D-factor of one target. Parameters ---------- target : `dmsky.targets.Target` Specific target astro_val_path : str Path for the output file Returns ------- output : dict The description of the target J-factor """ astro_profile_data = target.profile.copy() astro_profile_data['j_integ'] = target.j_integ astro_profile_data['j_sigma'] = target.j_sigma # Put the D-factors in a try-expect block try: astro_profile_data['d_integ'] = target.d_integ astro_profile_data['d_sigma'] = target.d_sigma except Exception: sys.stdout.write( "WARNING, could not compute D-Factors for target %s\n" % target.name) write_yaml(astro_profile_data, astro_val_path) return astro_profile_data
def main(): usage = "combine_dnm.py [options]" description = "add the pos and neg dnm components" parser = argparse.ArgumentParser(usage=usage, description=description) parser.add_argument('-i', '--input', type=str, default=None, help='Input file prefix') parser.add_argument('-o', '--output', type=str, default=None, help='Output ') parser.add_argument('-f', '--fact', default=False, action='store_true', help='Use factors') args = parser.parse_args(sys.argv[1:]) out_dict = utils.load_yaml(args.input) combine_dnm_maps(out_dict, args.fact) utils.write_yaml(out_dict, args.output)
def make_srcmap_manifest(self, modelkey, components, data): """Build a yaml file that specfies how to make the srcmap files for a particular model Parameters ---------- modelkey : str Key used to identify this particular model components : list The binning components used in this analysis data : str Path to file containing dataset definition """ try: model_info = self._models[modelkey] except KeyError: model_info = self.make_model_info(modelkey) self._name_factory.update_base_dict(data) outfile = os.path.join('analysis', 'model_%s' % modelkey, 'srcmap_manifest_%s.yaml' % modelkey) manifest = model_info.make_srcmap_manifest(components, self._name_factory) outdir = os.path.dirname(outfile) try: os.makedirs(outdir) except OSError: pass utils.write_yaml(manifest, outfile)
def main(): usage = "npred_txt2yaml.py [options]" description = "Convert Gardian text files to yaml dictionary" parser = argparse.ArgumentParser(usage=usage, description=description) parser.add_argument('-i', '--input', type=str, default=None, help='Input file') parser.add_argument('-o', '--output', type=str, default=None, help='Output ') parser.add_argument('-d', '--conv_dict', type=str, default=None, help='Output ') args = parser.parse_args(sys.argv[1:]) conv_dict = load_yaml(args.conv_dict) npred_dict = make_npred_dict(args.input, conv_dict) write_yaml(npred_dict, args.output)
def make_srcmap_manifest(self, modelkey, components, data): """Build a yaml file that specfies how to make the srcmap files for a particular model Parameters ---------- modelkey : str Key used to identify this particular model components : list The binning components used in this analysis data : str Path to file containing dataset definition """ try: model_info = self._models[modelkey] except KeyError: model_info = self.make_model_info(modelkey) self._name_factory.update_base_dict(data) outfile = os.path.join('analysis', 'model_%s' % modelkey, 'srcmap_manifest_%s.yaml' % modelkey) manifest = model_info.make_srcmap_manifest( components, self._name_factory) outdir = os.path.dirname(outfile) try: os.makedirs(outdir) except OSError: pass utils.write_yaml(manifest, outfile)
def run_analysis(self, argv): """Run this analysis""" args = self._parser.parse_args(argv) if not HAVE_ST: raise RuntimeError( "Trying to run fermipy analysis, but don't have ST") workdir = os.path.dirname(args.config) _config_file = self._clone_config_and_srcmaps(args.config, args.seed) gta = GTAnalysis(_config_file, logging={'verbosity': 3}, fileio={'workdir_regex': '\.xml$|\.npy$'}) gta.load_roi(args.roi_baseline) simfile = os.path.join(workdir, 'sim_%s_%s.yaml' % (args.sim, args.sim_profile)) mcube_file = "%s_%s_%06i" % (args.sim, args.sim_profile, args.seed) sim_config = utils.load_yaml(simfile) injected_source = sim_config.get('injected_source', None) if injected_source is not None: src_dict = injected_source['source_model'] src_dict['ra'] = gta.config['selection']['ra'] src_dict['dec'] = gta.config['selection']['dec'] injected_name = injected_source['name'] gta.add_source(injected_name, src_dict) gta.write_model_map(mcube_file) mc_spec_dict = dict( true_counts=gta.model_counts_spectrum(injected_name), energies=gta.energies, model=src_dict) mcspec_file = os.path.join( workdir, "mcspec_%s_%06i.yaml" % (mcube_file, args.seed)) utils.write_yaml(mc_spec_dict, mcspec_file) else: injected_name = None gta.write_roi('sim_baseline_%06i' % args.seed) test_sources = {} for profile in args.profiles: profile_path = os.path.join(workdir, 'profile_%s.yaml' % profile) test_source = load_yaml(profile_path) test_sources[profile] = test_source first = args.seed last = first + args.nsims for seed in range(first, last): self._run_simulation(gta, args.roi_baseline, injected_name, test_sources, first, seed, non_null_src=args.non_null_src, do_find_src=args.do_find_src)
def _run_simulation(gta, roi_baseline, injected_name, test_sources, current_seed, seed, non_null_src, do_find_src=False): """Simulate a realization of this analysis""" gta.load_roi('sim_baseline_%06i.npy' % current_seed) gta.set_random_seed(seed) gta.simulate_roi() skip_srcs = [] if injected_name: gta.zero_source(injected_name) skip_srcs.append(injected_name) gta.optimize(skip=skip_srcs) if do_find_src: gta.find_sources(sqrt_ts_threshold=5.0, search_skydir=gta.roi.skydir, search_minmax_radius=[1.0, np.nan]) gta.optimize(skip=skip_srcs) gta.free_sources(skydir=gta.roi.skydir, distance=1.0, pars='norm') if injected_name: gta.free_source(injected_name, False) gta.fit(covar=True) gta.write_roi('sim_refit_%06i' % current_seed) for pkey, test_source in test_sources.items(): test_source_name = test_source['name'] sedfile = "sed_%s_%06i.fits" % (test_source_name, seed) correl_dict, test_src_name = add_source_get_correlated( gta, test_source_name, test_source['source_model'], correl_thresh=0.25, non_null_src=non_null_src) # Write the list of correlated sources correl_yaml = os.path.join( gta.workdir, "correl_%s_%06i.yaml" % (test_source_name, seed)) write_yaml(correl_dict, correl_yaml) gta.free_sources(False) for src_name in correl_dict.keys(): gta.free_source(src_name, pars='norm') if injected_name: gta.free_source(injected_name, False) gta.sed(test_source_name, outfile=sedfile) # Set things back to how they were gta.delete_source(test_source_name) gta.load_xml('sim_refit_%06i' % current_seed)
def main(): usage = "npred_mapSums.py [options]" description = "Convert Gardian text files to yaml dictionary" parser = argparse.ArgumentParser(usage=usage, description=description) parser.add_argument('-i', '--input', type=str, default=None, help='Input file') parser.add_argument('-e', '--exposure', type=str, default=None, help='Exposure file') parser.add_argument('-o', '--output', type=str, default=None, help='Output ') args = parser.parse_args(sys.argv[1:]) erange_0_edges = np.logspace(np.log10(30), 2, 3) erange_1_edges = np.logspace(2, np.log10(300), 3) erange_2_edges = np.logspace(np.log10(300), 3, 4) erange_3_edges = np.logspace(3, 6, 10) erange_0 = np.sqrt(erange_0_edges[0:-1] * erange_0_edges[1:]) erange_1 = np.sqrt(erange_1_edges[0:-1] * erange_1_edges[1:]) erange_2 = np.sqrt(erange_2_edges[0:-1] * erange_2_edges[1:]) erange_3 = np.sqrt(erange_3_edges[0:-1] * erange_3_edges[1:]) energies = [erange_0, erange_1, erange_2, erange_3] ewidths = [ erange_0_edges[1:] - erange_0_edges[0:-1], erange_1_edges[1:] - erange_1_edges[0:-1], erange_2_edges[1:] - erange_2_edges[0:-1], erange_3_edges[1:] - erange_3_edges[0:-1] ] l = [] factor = 12 * 32 * 32 for evals, ew, comp in zip(energies, ewidths, COMPONENTS): exppath = os.path.join(args.exposure, "%s_expcube_moreplanes_ring_P8R3.fits" % (comp)) egbpath = args.input exp_vals = get_expmap_sums(exppath, evals) egb_vals = get_int_egb(egbpath, evals) o_vals = exp_vals * egb_vals * ew / factor l.append(o_vals) o = dict(iso_map=l) write_yaml(o, args.output)
def _write_profile_yaml(cls, target, profile_path, targ_ver, spatial): """ Write a yaml file describing the spatial profile of the target. Parameters ---------- target : dict Specific target profile_path : str Path for the output file targ_ver : str Version of the target, used for bookkeeping spatial : str Spatial model, one of ['point', 'radial', 'map'] Returns ------- output : dict The description of the target spatial profile """ source_model = dict(SpectrumType='PowerLaw', RA=target['ra'], DEC=target['dec']) if spatial in [None, 'point']: source_model.update(dict(SpatialModel='PointSource')) elif spatial in ['map']: if target.j_map_file is None: j_map_file = profile_path.replace('.yaml', '.fits') target.write_jmap_wcs(j_map_file) source_model.update( dict(SpatialModel='DiffuseSource', SpatialType='SpatialMap', Spatial_Filename=target['map'])) elif spatial in ['radial']: target.j_rad_file = profile_path.replace('.yaml', '.dat') target.write_j_rad_file() source_model.update( dict(SpatialModel='DiffuseSource', SpatialType='RadialProfile', radialprofile=target['radial'])) else: raise ValueError('Did not recognize spatial type %s' % spatial) ver_name = "%s_%s" % (targ_ver, spatial) profile_dict = dict(name=ver_name, source_model=source_model) write_yaml(profile_dict, profile_path) return profile_dict
def main(): usage = "fermitool-gardian_srcmap_sums.py [options]" description = "sum source maps files over all pixels" parser = argparse.ArgumentParser(usage=usage, description=description) parser.add_argument('-i', '--input', type=str, default=None, help='Input file prefix') parser.add_argument('-o', '--output', type=str, default=None, help='Output ') parser.add_argument('-d', '--conv_dict', type=str, default=None, help='Conversion dict ') args = parser.parse_args(sys.argv[1:]) out_dict = {} conv_dict = load_yaml(args.conv_dict) comps = conv_dict['COMPONENTS'] src_dict = conv_dict['SRC_NAME_DICT'] sel_dep_comps = conv_dict['SEL_DEP_COMPS'] cache_dirs = conv_dict['CACHE_DIR_DICT'] sys.stdout.write("Reading %i components\n" % (len(SRC_NAME_DICT))) for k, v in sorted(src_dict.items()): sys.stdout.write('.') sys.stdout.flush() cache_dir = cache_dirs[k] for c in comps: if k in sel_dep_comps: fp = os.path.join(args.input, cache_dir, "%s_%s%s_cache.fits.gz" % (c, k, c[0:2])) else: fp = os.path.join(args.input, cache_dir, "%s_%s_cache.fits.gz" % (c, k)) map_in = read_map_from_fits(fp) sums = map_in.counts.sum(1) if not out_dict.has_key(v): out_dict[v] = [] out_dict[v].append(sums) sys.stdout.write('!\n') utils.write_yaml(out_dict, args.output)
def run_analysis(self, argv): """Run this analysis""" args = self._parser.parse_args(argv) if not HAVE_ST: raise RuntimeError( "Trying to run fermipy analysis, but don't have ST") workdir = os.path.dirname(args.config) _config_file = self._clone_config_and_srcmaps(args.config, args.seed) gta = GTAnalysis(_config_file, logging={'verbosity': 3}, fileio={'workdir_regex': '\.xml$|\.npy$'}) gta.load_roi(args.roi_baseline) simfile = os.path.join(workdir, 'sim_%s_%s.yaml' % (args.sim, args.sim_profile)) mcube_file = "%s_%s_%06i" % (args.sim, args.sim_profile, args.seed) sim_config = utils.load_yaml(simfile) injected_source = sim_config.get('injected_source', None) if injected_source is not None: src_dict = injected_source['source_model'] src_dict['ra'] = gta.config['selection']['ra'] src_dict['dec'] = gta.config['selection']['dec'] injected_name = injected_source['name'] gta.add_source(injected_name, src_dict) gta.write_model_map(mcube_file) mc_spec_dict = dict(true_counts=gta.model_counts_spectrum(injected_name), energies=gta.energies, model=src_dict) mcspec_file = os.path.join(workdir, "mcspec_%s_%06i.yaml" % (mcube_file, args.seed)) utils.write_yaml(mc_spec_dict, mcspec_file) else: injected_name = None gta.write_roi('sim_baseline_%06i' % args.seed) test_sources = [] for profile in args.profiles: profile_path = os.path.join(workdir, 'profile_%s.yaml' % profile) test_source = load_yaml(profile_path) test_sources.append(test_source) first = args.seed last = first + args.nsims for seed in range(first, last): self._run_simulation(gta, args.roi_baseline, injected_name, test_sources, first, seed, non_null_src=args.non_null_src)
def _write_sim_target_config(cls, target_config, target_dir, sim_target_dir): """ Write a fermipy configurate file for one target for simulated analysis. This largely copies the configuration for flight data. It does make a few changes to point some of the input files (like the exposure map) to the flight data verions to avoid having to re-compute them. Parameters ---------- target_config : dict Configuration for flight data analysis for this target target_dir : str Analysis directory for flight data for this target sim_target_dir : str Directory to write to Returns ------- output : dict The configuration for this specific target """ sim_target_config_path = os.path.join(sim_target_dir, 'config.yaml') sim_target_config = copy.deepcopy(target_config) comps = sim_target_config.get('components', [sim_target_config]) for i, comp in enumerate(comps): comp_name = "%02i" % i if 'gtlike' not in comp: comp['gtlike'] = {} comp['gtlike']['bexpmap'] = os.path.abspath( os.path.join(target_dir, 'bexpmap_%s.fits' % comp_name)) comp['gtlike']['srcmap'] = os.path.abspath( os.path.join(sim_target_dir, 'srcmap_%s.fits' % comp_name)) comp['gtlike']['use_external_srcmap'] = True write_yaml(sim_target_config, sim_target_config_path) return sim_target_config
def run_analysis(self, argv): """Run this analysis""" args = self._parser.parse_args(argv) if is_null(args.config): raise ValueError("Config yaml file must be specified") if is_null(args.rand_config): raise ValueError( "Random direction config yaml file must be specified") config = load_yaml(args.config) rand_config = load_yaml(args.rand_config) wcsgeom = self._make_wcsgeom_from_config(config) dir_dict = self._build_skydir_dict(wcsgeom, rand_config) if is_not_null(args.outfile): write_yaml(dir_dict, args.outfile)
def _clone_config_and_srcmaps(config_path, seed): """Clone the configuration""" workdir = os.path.dirname(config_path) new_config_path = config_path.replace('.yaml', '_%06i.yaml' % seed) config = load_yaml(config_path) comps = config.get('components', [config]) for i, comp in enumerate(comps): comp_name = "%02i" % i if 'gtlike' not in comp: comp['gtlike'] = {} orig_srcmap = os.path.abspath(os.path.join(workdir, 'srcmap_%s.fits' % (comp_name))) new_srcmap = os.path.abspath(os.path.join(workdir, 'srcmap_%06i_%s.fits' % (seed, comp_name))) comp['gtlike']['srcmap'] = os.path.abspath(os.path.join(workdir, 'srcmap_%06i_%s.fits' % (seed, comp_name))) comp['gtlike']['use_external_srcmap'] = True copyfile(orig_srcmap, new_srcmap) write_yaml(config, new_config_path) return new_config_path
def write_target_dirs(basedir, roster_dict, base_config): """ Create and populate directoris for target analysis """ target_dict = {} target_info_dict = {} roster_info_dict = {} try: os.makedirs(basedir) except OSError: pass for roster_name, rost in roster_dict.items(): tlist = [] for target_name, target in rost.items(): target_key = "%s:%s" % (target_name, target.version) print("Writing %s" % (target_key)) tlist.append(target_key) if target_info_dict.has_key(target_name): target_info_dict[target_name].append(target.version) else: target_info_dict[target_name] = [target.version] target_dir = os.path.join(basedir, target_name) target_config_path = os.path.join(target_dir, 'config_baseline.yaml') jmap_path = os.path.join(target_dir, 'profile_%s.fits' % target.version) profile_path = os.path.join(target_dir, 'profile_%s.yaml' % target.version) if target_dict.has_key(target_name): # Already made the config for this target target_config = target_dict[target_name] else: # Make the config for this target try: os.makedirs(target_dir) except OSError: pass target_config = base_config.copy() target_config['selection']['ra'] = target.ra target_config['selection']['dec'] = target.dec target_dict[target_name] = target_config write_yaml(target_config, target_config_path) profile_data = target.profile.copy() target.write_jmap_wcs(jmap_path, clobber=True) profile_data['j_integ'] = target.j_integ profile_data['j_sigma'] = target.j_sigma profile_data['j_map_file'] = jmap_path write_yaml(profile_data, profile_path) roster_info_dict[roster_name] = tlist write_yaml(roster_info_dict, os.path.join(basedir, 'roster_list.yaml')) write_yaml(target_info_dict, os.path.join(basedir, 'target_list.yaml'))
def main(): usage = "srcmap_sums.py [options]" description = "sum source maps files over all pixels" parser = argparse.ArgumentParser(usage=usage, description=description) parser.add_argument('-i', '--input', type=str, default=None, help='Input file prefix') parser.add_argument('-o', '--output', type=str, default=None, help='Output ') parser.add_argument('-d', '--conv_dict', type=str, default=None, help='Conversion dict ') args = parser.parse_args(sys.argv[1:]) out_dict = {} conv_dict = utils.load_yaml(args.conv_dict) comps = conv_dict['COMPONENTS'] src_dict = conv_dict['SRC_NAME_DICT'] for c in comps: fp = "%s_%s_GAL_V2.fits" % (args.input, c) fin = fits.open(fp) ebins = fits_utils.find_and_read_ebins(fin) for hdu in fin[4:]: if hdu.name in ['__weights__']: continue if hdu.name[0:5] == 'FL8Y ': continue sums = get_sums(hdu, ebins) src_name = src_dict[hdu.name] if not out_dict.has_key(src_name): out_dict[src_name] = [] out_dict[src_name].append(sums) utils.write_yaml(out_dict, args.output)
def main(): usage = "npred_mapSums.py [options]" description = "Convert Gardian text files to yaml dictionary" parser = argparse.ArgumentParser(usage=usage, description=description) parser.add_argument('-i', '--input', type=str, default=None, help='Input file') parser.add_argument('-o', '--output', type=str, default=None, help='Output ') parser.add_argument('-d', '--conv_dict', type=str, default=None, help='Conversion dict ') args = parser.parse_args(sys.argv[1:]) conv_dict = load_yaml(args.conv_dict) comps = conv_dict['COMPONENTS'] src_dict = conv_dict['SRC_NAME_DICT'] sel_dep_comps = conv_dict['SEL_DEP_COMPS'] sum_dict = make_sum_dict(args.input, comps, src_dict, sel_dep_comps) write_yaml(sum_dict, args.output)
def _write_profile_yaml(cls, target, profile_path, targ_ver, spatial): """ Write a yaml file describing the spatial profile of the target. Parameters ---------- target : dict Specific target profile_path : str Path for the output file targ_ver : str Version of the target, used for bookkeeping spatial : str Spatial model, one of ['point', 'radial', 'map'] Returns ------- output : dict The description of the target spatial profile """ source_model = dict(SpectrumType='PowerLaw', RA=target['ra'], DEC=target['dec']) if spatial in [None, 'point']: source_model.update(dict(SpatialModel='PointSource')) else: raise ValueError('Did not recognize spatial type %s' % spatial) ver_name = "%s_%s" % (targ_ver, spatial) profile_dict = dict(name=ver_name, source_model=source_model) write_yaml(profile_dict, profile_path) return profile_dict
def _write_astro_value_yaml(cls, target, astro_val_path): """Write a yaml file describing the target Parameters ---------- target : dict Specific target astro_val_path : str Path for the output file Returns ------- output : dict The description of the targe """ astro_profile_data = target.copy() write_yaml(astro_profile_data, astro_val_path) return astro_profile_data
def _run_simulation(gta, roi_baseline, injected_name, test_sources, current_seed, seed, non_null_src): """Simulate a realization of this analysis""" gta.load_roi('sim_baseline_%06i.npy' % current_seed) gta.set_random_seed(seed) gta.simulate_roi() if injected_name: gta.zero_source(injected_name) gta.optimize() gta.find_sources(sqrt_ts_threshold=5.0, search_skydir=gta.roi.skydir, search_minmax_radius=[1.0, np.nan]) gta.optimize() gta.free_sources(skydir=gta.roi.skydir, distance=1.0, pars='norm') gta.fit(covar=True) gta.write_roi('sim_refit_%06i' % current_seed) for test_source in test_sources: test_source_name = test_source['name'] sedfile = "sed_%s_%06i.fits" % (test_source_name, seed) correl_dict, test_src_name = add_source_get_correlated(gta, test_source_name, test_source['source_model'], correl_thresh=0.25, non_null_src=non_null_src) # Write the list of correlated sources correl_yaml = os.path.join(gta.workdir, "correl_%s_%06i.yaml" % (test_source_name, seed)) write_yaml(correl_dict, correl_yaml) gta.free_sources(False) for src_name in correl_dict.keys(): gta.free_source(src_name, pars='norm') gta.sed(test_source_name, prefix=pkey, outfile=sedfile) # Set things back to how they were gta.delete_source(test_source_name) gta.load_xml('sim_refit_%06i' % current_seed)
def _clone_config_and_srcmaps(config_path, seed): """Clone the configuration""" workdir = os.path.dirname(config_path) new_config_path = config_path.replace('.yaml', '_%06i.yaml' % seed) config = load_yaml(config_path) comps = config.get('components', [config]) for i, comp in enumerate(comps): comp_name = "%02i" % i if 'gtlike' not in comp: comp['gtlike'] = {} orig_srcmap = os.path.abspath( os.path.join(workdir, 'srcmap_%s.fits' % (comp_name))) new_srcmap = os.path.abspath( os.path.join(workdir, 'srcmap_%06i_%s.fits' % (seed, comp_name))) comp['gtlike']['srcmap'] = os.path.abspath( os.path.join(workdir, 'srcmap_%06i_%s.fits' % (seed, comp_name))) comp['gtlike']['use_external_srcmap'] = True copyfile(orig_srcmap, new_srcmap) write_yaml(config, new_config_path) return new_config_path
def main(): usage = "npred_txt2yaml.py [options]" description = "Convert Gardian text files to yaml dictionary" parser = argparse.ArgumentParser(usage=usage, description=description) parser.add_argument('-i', '--input', type=str, default=None, help='Input file') parser.add_argument('-o', '--output', type=str, default=None, help='Output ') args = parser.parse_args(sys.argv[1:]) src_dict = load_npy(args.input) npred_dict = make_npred_dict(src_dict['sources']) npred_dict['Energies'] = get_energies(src_dict['roi']) write_yaml(npred_dict, args.output)
def _write_sim_yaml(cls, target, sim, sim_target_dir, target_key): """Write a yaml file describing the 'True' target parameters for simulated data. Parameters ---------- target : dict Specific target sim : str Name of the simulation scenario. sim_target_dir : str Directory to write to target_key : str Version of the target to write Returns ------- output : dict The description of the 'True' target parameters """ sim_profile_yaml = os.path.join('config', 'sim_%s.yaml' % sim) sim_profile = load_yaml(sim_profile_yaml) injected_source = sim_profile.get('injected_source', None) #if injected_source is not None: # sim_profile['injected_source']['source_model']['norm']['value'] = target['norm'] sim_out_path = os.path.join( sim_target_dir, 'sim_%s_%s.yaml' % (sim, target_key)) write_yaml(sim_profile, sim_out_path) return sim_profile
def run_analysis(self, argv): """Run this analysis""" args = self._parser.parse_args(argv) if not HAVE_ST: raise RuntimeError( "Trying to run fermipy analysis, but don't have ST") if is_null(args.skydirs): skydir_dict = None else: skydir_dict = load_yaml(args.skydirs) gta = GTAnalysis(args.config, logging={'verbosity': 3}, fileio={'workdir_regex': '\.xml$|\.npy$'}) #gta.setup(overwrite=False) gta.load_roi(args.roi_baseline) gta.print_roi() basedir = os.path.dirname(args.config) # This should be a no-op, b/c it was done in the baseline analysis for profile in args.profiles: if skydir_dict is None: skydir_keys = [None] else: skydir_keys = sorted(skydir_dict.keys()) for skydir_key in skydir_keys: if skydir_key is None: pkey, pdict = AnalyzeSED._build_profile_dict( basedir, profile) else: skydir_val = skydir_dict[skydir_key] pkey, pdict = AnalyzeSED._build_profile_dict( basedir, profile) pdict['ra'] = skydir_val['ra'] pdict['dec'] = skydir_val['dec'] pkey += "_%06i" % skydir_key outfile = "sed_%s.fits" % pkey # Add the source and get the list of correlated soruces correl_dict = add_source_get_correlated(gta, pkey, pdict, correl_thresh=0.25) # Write the list of correlated sources correl_yaml = os.path.join(basedir, "correl_%s.yaml" % pkey) write_yaml(correl_dict, correl_yaml) gta.free_sources(False) for src_name in correl_dict.keys(): gta.free_source(src_name, pars='norm') # build the SED gta.sed(pkey, outfile=outfile, make_plots=args.make_plots) # remove the source gta.delete_source(pkey) # put the ROI back to how it was gta.load_xml(args.roi_baseline) return gta
def run_analysis(self, argv): """Run this analysis""" args = self._parser.parse_args(argv) if not HAVE_ST: raise RuntimeError( "Trying to run fermipy analysis, but don't have ST") if is_null(args.skydirs): skydir_dict = None else: skydir_dict = load_yaml(args.skydirs) gta = GTAnalysis(args.config, logging={'verbosity': 3}, fileio={'workdir_regex': '\.xml$|\.npy$'}) #gta.setup(overwrite=False) gta.load_roi(args.roi_baseline) gta.print_roi() basedir = os.path.dirname(args.config) # This should be a no-op, b/c it was done in the baseline analysis for profile in args.profiles: if skydir_dict is None: skydir_keys = [None] else: skydir_keys = sorted(skydir_dict.keys()) for skydir_key in skydir_keys: if skydir_key is None: pkey, psrc_name, pdict = build_profile_dict(basedir, profile) else: skydir_val = skydir_dict[skydir_key] pkey, psrc_name, pdict = build_profile_dict(basedir, profile) pdict['ra'] = skydir_val['ra'] pdict['dec'] = skydir_val['dec'] pkey += "_%06i" % skydir_key outfile = "sed_%s.fits" % pkey # Add the source and get the list of correlated soruces correl_dict, test_src_name = add_source_get_correlated(gta, psrc_name, pdict, correl_thresh=0.25, non_null_src=args.non_null_src) # Write the list of correlated sources correl_yaml = os.path.join(basedir, "correl_%s.yaml" % pkey) write_yaml(correl_dict, correl_yaml) gta.free_sources(False) for src_name in correl_dict.keys(): gta.free_source(src_name, pars='norm') # build the SED if args.non_null_src: gta.update_source(test_src_name, reoptimize=True) gta.write_roi("base_%s"% pkey, make_plots=False) gta.sed(test_src_name, prefix=pkey, outfile=outfile, make_plots=args.make_plots) # remove the source gta.delete_source(test_src_name) # put the ROI back to how it was gta.load_xml(args.roi_baseline) return gta
def make_fermipy_config_yaml(self, modelkey, components, data, **kwargs): """Build a fermipy top-level yaml configuration file Parameters ---------- modelkey : str Key used to identify this particular model components : list The binning components used in this analysis data : str Path to file containing dataset definition """ model_dir = os.path.join('analysis', 'model_%s' % modelkey) hpx_order = kwargs.get('hpx_order', 9) self._name_factory.update_base_dict(data) try: model_info = self._models[modelkey] except KeyError: model_info = self.make_model_info(modelkey) model_info.make_model_rois(components, self._name_factory) #source_names = model_info.component_names master_xml_mdl = os.path.basename( self._name_factory.master_srcmdl_xml(modelkey=modelkey)) master_data = dict(scfile=self._name_factory.ft2file(fullpath=True), cacheft1=False) master_binning = dict(projtype='HPX', coordsys=kwargs.get('coordsys', 'GAL'), roiwidth=180., binsperdec=8, hpx_ordering_scheme="RING", hpx_order=hpx_order, hpx_ebin=True) master_fileio = dict(outdir=model_dir, logfile=os.path.join(model_dir, 'fermipy.log')) master_gtlike = dict(irfs=self._name_factory.irfs(**kwargs), edisp_disable=['isodiff', 'diffuse', 'limb']) master_selection = dict(glat=0., glon=0., radius=180.) master_model = dict(catalogs=[master_xml_mdl]) master = dict(data=master_data, binning=master_binning, fileio=master_fileio, selection=master_selection, gtlike=master_gtlike, model=master_model, components=[]) fermipy_dict = master #comp_rois = {} for comp in components: zcut = "zmax%i" % comp.zmax compkey = "%s_%s" % (zcut, comp.make_key('{ebin_name}_{evtype_name}')) name_keys = dict(zcut=zcut, modelkey=modelkey, component=compkey, fullpath=True) comp_data = dict(ltcube=self._name_factory.ltcube(**name_keys)) comp_selection = dict(logemin=comp.log_emin, logemax=comp.log_emax, zmax=comp.zmax, evtype=comp.evtype) comp_binning = dict(enumbins=comp.enumbins, hpx_order=min(comp.hpx_order, hpx_order)) comp_gtlike = dict( srcmap=self._name_factory.merged_srcmaps(**name_keys), bexpmap=self._name_factory.bexpcube(**name_keys)) #comp_roi_source_info = {} comp_xml_mdl = os.path.basename( self._name_factory.comp_srcmdl_xml(modelkey=modelkey, component=compkey)) comp_model = dict(catalogs=[master_xml_mdl, comp_xml_mdl]) sub_dict = dict(data=comp_data, binning=comp_binning, selection=comp_selection, gtlike=comp_gtlike, model=comp_model) fermipy_dict['components'].append(sub_dict) outfile = os.path.join(model_dir, 'config.yaml') print("Writing fermipy config file %s" % outfile) utils.write_yaml(fermipy_dict, outfile) return fermipy_dict
def write_config(self, outfile): """Write the configuration dictionary to an output file.""" utils.write_yaml(self.config, outfile, default_flow_style=False)
def make_fermipy_config_yaml(self, modelkey, components, data, **kwargs): """Build a fermipy top-level yaml configuration file Parameters ---------- modelkey : str Key used to identify this particular model components : list The binning components used in this analysis data : str Path to file containing dataset definition """ model_dir = os.path.join('analysis', 'model_%s' % modelkey) hpx_order = kwargs.get('hpx_order', 9) self._name_factory.update_base_dict(data) try: model_info = self._models[modelkey] except KeyError: model_info = self.make_model_info(modelkey) model_info.make_model_rois(components, self._name_factory) #source_names = model_info.component_names master_xml_mdl = os.path.basename( self._name_factory.master_srcmdl_xml(modelkey=modelkey)) master_data = dict(scfile=self._name_factory.ft2file(fullpath=True), cacheft1=False) master_binning = dict(projtype='HPX', coordsys=kwargs.get('coordsys', 'GAL'), roiwidth=180., binsperdec=8, hpx_ordering_scheme="RING", hpx_order=hpx_order, hpx_ebin=True) master_fileio = dict(outdir=model_dir, logfile=os.path.join(model_dir, 'fermipy.log')) master_gtlike = dict(irfs=self._name_factory.irfs(**kwargs), edisp_disable=['isodiff', 'diffuse', 'limb']) master_selection = dict(glat=0., glon=0., radius=180.) master_model = dict(catalogs=[master_xml_mdl]) master = dict(data=master_data, binning=master_binning, fileio=master_fileio, selection=master_selection, gtlike=master_gtlike, model=master_model, components=[]) fermipy_dict = master #comp_rois = {} for comp in components: zcut = "zmax%i" % comp.zmax compkey = "%s_%s" % (zcut, comp.make_key( '{ebin_name}_{evtype_name}')) name_keys = dict(zcut=zcut, modelkey=modelkey, component=compkey, fullpath=True) comp_data = dict(ltcube=self._name_factory.ltcube(**name_keys)) comp_selection = dict(logemin=comp.log_emin, logemax=comp.log_emax, zmax=comp.zmax, evtype=comp.evtype) comp_binning = dict(enumbins=comp.enumbins, hpx_order=min(comp.hpx_order, hpx_order)) comp_gtlike = dict(srcmap=self._name_factory.merged_srcmaps(**name_keys), bexpmap=self._name_factory.bexpcube(**name_keys)) #comp_roi_source_info = {} comp_xml_mdl = os.path.basename(self._name_factory.comp_srcmdl_xml(modelkey=modelkey, component=compkey)) comp_model = dict(catalogs=[master_xml_mdl, comp_xml_mdl]) sub_dict = dict(data=comp_data, binning=comp_binning, selection=comp_selection, gtlike=comp_gtlike, model=comp_model) fermipy_dict['components'].append(sub_dict) outfile = os.path.join(model_dir, 'config.yaml') print ("Writing fermipy config file %s"%outfile) utils.write_yaml(fermipy_dict, outfile) return fermipy_dict
def make_fermipy_config_yaml(self, modelkey, components, data, **kwargs): """Build a fermipy top-level yaml configuration file Parameters ---------- modelkey : str Key used to identify this particular model components : list The binning components used in this analysis data : str Path to file containing dataset definition """ model_dir = os.path.join('analysis', 'model_%s' % modelkey) hpx_order = kwargs.get('hpx_order', 9) self._name_factory.update_base_dict(data) try: model_info = self._models[modelkey] except KeyError: model_info = self.make_model_info(modelkey) model_rois = model_info.make_model_rois(components, self._name_factory) #source_names = model_info.component_names master_xml_mdl = os.path.basename( self._name_factory.master_srcmdl_xml(modelkey=modelkey)) master_data = dict(scfile=self._name_factory.ft2file(fullpath=True), cacheft1=False) master_binning = dict(projtype='HPX', roiwidth=360., binsperdec=4, hpx_ordering_scheme="RING", hpx_order=hpx_order, hpx_ebin=True) # master_fileio = dict(outdir=model_dir, # logfile=os.path.join(model_dir, 'fermipy.log')) master_fileio = dict(logfile='fermipy.log') master_gtlike = dict(irfs=self._name_factory.irfs(**kwargs), edisp_disable=model_info.edisp_disable_list(), use_external_srcmap=True) master_selection = dict(glat=0., glon=0., radius=180.) master_model = dict(catalogs=[master_xml_mdl]) master_plotting = dict(label_ts_threshold=1e9) master = dict(data=master_data, binning=master_binning, fileio=master_fileio, selection=master_selection, gtlike=master_gtlike, model=master_model, plotting=master_plotting, components=[]) fermipy_dict = master #comp_rois = {} for comp in components: zcut = "zmax%i" % comp.zmax compkey = "%s_%s" % (zcut, comp.make_key('{ebin_name}_{evtype_name}')) comp_roi = model_rois[compkey] name_keys = dict(zcut=zcut, modelkey=modelkey, component=compkey, mktime='none', coordsys=comp.coordsys, fullpath=True) comp_data = dict(ltcube=self._name_factory.ltcube(**name_keys)) comp_selection = dict(logemin=comp.log_emin, logemax=comp.log_emax, zmax=comp.zmax, evtype=comp.evtype) comp_binning = dict(enumbins=comp.enumbins, hpx_order=min(comp.hpx_order, hpx_order), coordsys=comp.coordsys) comp_gtlike = dict( srcmap=self._name_factory.merged_srcmaps(**name_keys), bexpmap=self._name_factory.bexpcube(**name_keys), use_external_srcmap=True) #comp_roi_source_info = {} diffuse_srcs = [] for src in comp_roi.diffuse_sources: if isinstance(src, MapCubeSource): diffuse_srcs.append(dict(name=src.name, file=src.mapcube)) elif isinstance(src, IsoSource): diffuse_srcs.append( dict(name=src.name, file=src.fileFunction)) else: pass comp_model = dict(diffuse=diffuse_srcs) sub_dict = dict(data=comp_data, binning=comp_binning, selection=comp_selection, gtlike=comp_gtlike, model=comp_model) fermipy_dict['components'].append(sub_dict) # Add placeholder diffuse sources fermipy_dict['model']['diffuse'] = diffuse_srcs outfile = os.path.join(model_dir, 'config.yaml') print("Writing fermipy config file %s" % outfile) utils.write_yaml(fermipy_dict, outfile) return fermipy_dict
def _write_target_dirs(cls, ttype, roster_dict, base_config, sims, spatial_models, aliases): """ Create and populate directoris for target analysis. Parameters ---------- ttype : str Target type, used for bookeeping and directory naming roster_dict : dict Dictionary of dict objects with the analysis targets base_config : dict Baseline configuration sims : list List of names of simulation scenarios spatial_models : dict Dictionary types of spatial models to use in analysis aliases : dict Optional dictionary to remap target verion keys """ target_dict = {} target_info_dict = {} roster_info_dict = {} for roster_name, rost in roster_dict.items(): for target_name, target in rost.items(): print(roster_name, target_name, target) if aliases is not None: try: ver_key = aliases[target.get("version", "default")] except KeyError: ver_key = target.get("version", "default") else: ver_key = target.get("version", "default") target_key = "%s:%s" % (target_name, ver_key) print("Writing %s" % (target_key)) name_keys = dict(target_type=ttype, target_name=target_name, target_version=ver_key, fullpath=True) astro_val_path = NAME_FACTORY.astro_valuefile(**name_keys) target_dir = NAME_FACTORY.targetdir(**name_keys) try: os.makedirs(target_dir) except OSError: pass cls._write_astro_value_yaml(target, astro_val_path) for sim in sims: name_keys['sim_name'] = sim sim_target_dir = NAME_FACTORY.sim_targetdir(**name_keys) sim_astro_val_path = NAME_FACTORY.sim_astro_valuefile(**name_keys) try: os.makedirs(sim_target_dir) except OSError: pass cls._write_astro_value_yaml(target, sim_astro_val_path) cls._write_sim_yaml(target, sim, sim_target_dir, ver_key) name_keys.pop('sim_name') write_config = False if target_name in target_dict: # Already made the config for this target target_config = target_dict[target_name].copy() else: # Make the config for this target target_config = cls._write_data_target_config(base_config, target, target_dir) target_dict[target_name] = target_config write_config = True write_sim_config = write_config for spatial in spatial_models: ver_string = "%s_%s" % (ver_key, spatial) roster_key = "%s_%s" % (roster_name, spatial) full_key = "%s:%s" % (target_name, ver_string) name_keys['profile'] = ver_string profile_path = NAME_FACTORY.profilefile(**name_keys) if target_name in target_info_dict: target_info_dict[target_name].append(ver_string) else: target_info_dict[target_name] = [ver_string] cls._write_profile_yaml(target, profile_path, ver_key, spatial) if roster_key in roster_info_dict: roster_info_dict[roster_key].append(full_key) else: roster_info_dict[roster_key] = [full_key] for sim in sims: name_keys['sim_name'] = sim sim_target_dir = NAME_FACTORY.sim_targetdir(**name_keys) sim_profile_path = NAME_FACTORY.sim_profilefile(**name_keys) if write_sim_config: cls._write_sim_target_config(target_config, target_dir, sim_target_dir) cls._write_profile_yaml(target, sim_profile_path, ver_key, spatial) write_sim_config = False roster_file = os.path.join(ttype, 'roster_list.yaml') target_file = os.path.join(ttype, 'target_list.yaml') write_yaml(roster_info_dict, roster_file) write_yaml(target_info_dict, target_file) for sim in sims: sim_dir = os.path.join("%s_sim" % ttype, "sim_%s" % sim) sim_roster_file = os.path.join(sim_dir, 'roster_list.yaml') sim_target_file = os.path.join(sim_dir, 'target_list.yaml') try: os.makedirs(sim_dir) except OSError: pass copyfile(roster_file, sim_roster_file) copyfile(target_file, sim_target_file)