def main(): usage = "usage: %(prog)s [config file]" description = "Run fermipy analysis chain." parser = argparse.ArgumentParser(usage=usage,description=description) parser.add_argument('--config', default = 'sample_config.yaml') parser.add_argument('--source', default = None) args = parser.parse_args() gta = GTAnalysis(args.config) if args.source is None: src_name = gta.roi.sources[0].name gta.setup() gta.optimize() if (gta.roi[src_name]['ts'] > 1000. and gta.roi[src_name]['SpectrumType'] == 'PowerLaw'): gta.set_source_spectrum(src_name, spectrum_type='LogParabola', spectrum_pars={'beta' : {'value' : 0.0, 'scale' : 1.0, 'min' : 0.0, 'max' : 2.0}}) gta.free_source(src_name) gta.fit() gta.free_source(src_name, False) loc = gta.localize(src_name, free_radius=1.0, update=True, make_plots=True) model = {'Index' : 2.0, 'SpatialModel' : 'PointSource'} srcs = gta.find_sources(model=model, sqrt_ts_threshold=5.0, min_separation=0.5) sed = gta.sed(src_name, free_radius=1.0, make_plots=True) gta.tsmap(make_plots=True) gta.tsmap(prefix='excludeSource', exclude=[src_name], make_plots=True) gta.write_roi('fit0') lc = gta.lightcurve(src_name, binsz=86400.*28.0, free_radius=3.0, use_scaled_srcmap=True, multithread=False)
class ExtensionFit: def __init__(self, configFile): self.gta = GTAnalysis(configFile, logging={'verbosity': 3}) self.target = None self.targetRadius = None self.distance = None self.catalog = fits.getdata('/users-data/mfalxa/code/gll_psch_v13.fit', 1) def setSourceName(self, sourceObject, newName): self.gta.delete_source(sourceObject['name']) self.gta.add_source(newName, sourceObject) ''' INITIALIZE ''' def initialize(self, sizeROI, rInner, addToROI, TSMin, debug): self.gta.setup() if self.gta.config['selection']['emin'] >= 10000: self.gta.set_parameter('galdiff', 'Scale', 30000) if debug == True: self.gta.make_plots('startAll') self.gta.residmap(prefix='startAll', make_plots=True) # Get model source names sourceList = self.gta.get_sources(exclude=['isodiff', 'galdiff']) # Delete sources unassociated with TS < 50 for i in range(len(sourceList)): if sourceList[i]['catalog']['TS_value'] < TSMin and self.catalog[ 'CLASS'][self.catalog['Source_Name'] == sourceList[i] ['name']][0] == '': self.gta.delete_source(sourceList[i]['name']) closests = self.gta.get_sources(distance=rInner, exclude=['isodiff', 'galdiff']) # Delete all unidentified sources for i in range(len(closests)): if self.catalog['CLASS'][self.catalog['Source_Name'] == closests[i] ['name']][0].isupper() == False: self.gta.delete_source(closests[i]['name']) if self.catalog['CLASS'][self.catalog['Source_Name'] == closests[i] ['name']][0] == 'SFR': self.target = closests[i] self.setSourceName(self.target, 'TESTSOURCE') # If debug, save ROI and make plots if debug == True: self.gta.write_roi('startModel') self.gta.residmap(prefix='start', make_plots=True) self.gta.make_plots('start') # Optmize spectral parameters for sources with npred > 1 self.gta.optimize(npred_threshold=1, skip=['isodiff']) # Get model source names sourceList = self.gta.get_sources(distance=sizeROI + addToROI, square=True, exclude=['isodiff', 'galdiff']) # Iterate source localizing on source list for i in range(len(sourceList)): if sourceList[i].extended == False: self.gta.localize(sourceList[i]['name'], write_fits=False, write_npy=False, update=True) # Free sources within ROI size + extra distance from center self.gta.free_sources(distance=sizeROI + addToROI, square=True) # Re-optimize ROI self.gta.optimize(skip=['isodiff']) # Save and make plots if debug if debug == True: self.gta.write_roi('modelInitialized') self.gta.residmap(prefix='initialized', make_plots=True) self.gta.make_plots('initialized') # Lock sources self.gta.free_sources(free=False) ''' OUTER REGION ''' def outerRegionAnalysis(self, sizeROI, rInner, sqrtTsThreshold, minSeparation, debug): self.gta.free_sources(distance=sizeROI, pars='norm', square=True, free=True) self.gta.free_sources(distance=rInner, free=False) self.gta.free_source('galdiff', free=True) self.gta.free_source('isodiff', free=False) # Seek new sources until none are found sourceModel = { 'SpectrumType': 'PowerLaw', 'Index': 2.0, 'Scale': 30000, 'Prefactor': 1.e-15, 'SpatialModel': 'PointSource' } newSources = self.gta.find_sources(sqrt_ts_threshold=sqrtTsThreshold, min_separation=minSeparation, model=sourceModel, **{ 'search_skydir': self.gta.roi.skydir, 'search_minmax_radius': [rInner, sizeROI] }) if len(newSources) > 0: for i in range(len(newSources)): if newSources['sources'][i]['ts'] > 100.: self.gta.set_source_spectrum( newSources['sources'][i]['name'], spectrum_type='LogParabola') self.gta.free_source(newSources['sources'][i]['name']) self.gta.fit() self.gta.free_source(newSources['sources'][i]['name'], free=False) # Optimize all ROI self.gta.optimize(skip=['isodiff']) # Save sources found if debug == True: self.gta.residmap(prefix='outer', make_plots=True) self.gta.write_roi('outerAnalysisROI') self.gta.make_plots('outer') ''' INNER REGION ''' def innerRegionAnalysis(self, sizeROI, rInner, maxIter, sqrtTsThreshold, minSeparation, dmMin, TSm1Min, TSextMin, debug): self.gta.free_sources(distance=sizeROI, square=True, free=False) self.gta.free_sources(distance=rInner, free=True, exclude=['isodiff']) # Keep closest source if identified with star forming region in catalog or look for new source closest to center within Rinner if self.target != None: print('Closest source identified with star forming region : ', self.target['name']) self.gta.set_source_morphology('TESTSOURCE', **{'spatial_model': 'PointSource'}) else: closeSources = self.gta.find_sources(sqrt_ts_threshold=2., min_separation=minSeparation, max_iter=1, **{ 'search_skydir': self.gta.roi.skydir, 'search_minmax_radius': [0., rInner] }) dCenter = np.array([]) for i in range(len(closeSources['sources'])): dCenter = np.append( dCenter, self.gta.roi.skydir.separation( closeSources['sources'][i].skydir).value) self.target = closeSources['sources'][np.argmin(dCenter)] print('Target name : ', self.target['name']) self.setSourceName(self.target, 'TESTSOURCE') for i in [ x for x in range(len(closeSources['sources'])) if x != (np.argmin(dCenter)) ]: self.gta.delete_source(closeSources['sources'][i]['name']) self.gta.optimize(skip=['isodiff']) # Initialize n sources array nSources = [] # Save ROI without extension fit self.gta.write_roi('nSourcesFit') if debug == True: self.gta.make_plots('innerInit') self.gta.residmap(prefix='innerInit', make_plots=True) # Test for extension extensionTest = self.gta.extension('TESTSOURCE', make_plots=True, write_npy=debug, write_fits=debug, spatial_model='RadialDisk', update=True, free_background=True, fit_position=True) extLike = extensionTest['loglike_ext'] TSext = extensionTest['ts_ext'] print('TSext : ', TSext) extAIC = 2 * (len(self.gta.get_free_param_vector()) - self.gta._roi_data['loglike']) self.gta.write_roi('extFit') if debug == True: self.gta.residmap(prefix='ext0', make_plots=True) self.gta.make_plots('ext0') self.gta.load_roi('nSourcesFit', reload_sources=True) for i in range(1, maxIter + 1): # Test for n point sources nSourcesTest = self.gta.find_sources( sources_per_iter=1, sqrt_ts_threshold=sqrtTsThreshold, min_separation=minSeparation, max_iter=1, **{ 'search_skydir': self.gta.roi.skydir, 'search_minmax_radius': [0., rInner] }) if len(nSourcesTest['sources']) > 0: if nSourcesTest['sources'][0]['ts'] > 100.: self.gta.set_source_spectrum( nSourcesTest['sources'][0]['name'], spectrum_type='LogParabola') self.gta.free_source(nSourcesTest['sources'][0]['name']) self.gta.fit() self.gta.free_source(nSourcesTest['sources'][0]['name'], free=False) if debug == True: self.gta.make_plots('nSources' + str(i)) nSources.append(nSourcesTest['sources']) self.gta.localize(nSourcesTest['sources'][0]['name'], write_npy=debug, write_fits=debug, update=True) nAIC = 2 * (len(self.gta.get_free_param_vector()) - self.gta._roi_data['loglike']) self.gta.free_source(nSourcesTest['sources'][0]['name'], free=True) self.gta.residmap(prefix='nSources' + str(i), make_plots=True) self.gta.write_roi('n1SourcesFit') # Estimate Akaike Information Criterion difference between both models dm = extAIC - nAIC print('AIC difference between both models = ', dm) # Estimate TS_m+1 extensionTestPlus = self.gta.extension( 'TESTSOURCE', make_plots=True, write_npy=debug, write_fits=debug, spatial_model='RadialDisk', update=True, free_background=True, fit_position=True) TSm1 = 2 * (extensionTestPlus['loglike_ext'] - extLike) print('TSm+1 = ', TSm1) if debug == True: self.gta.residmap(prefix='ext' + str(i), make_plots=True) self.gta.make_plots('ext' + str(i)) if dm < dmMin and TSm1 < TSm1Min: self.gta.load_roi('extFit', reload_sources=True) break else: # Set extension test to current state and save current extension fit ROI and load previous nSources fit ROI extensionTest = extensionTestPlus extLike = extensionTestPlus['loglike_ext'] TSext = extensionTestPlus['ts_ext'] print('TSext : ', TSext) extAIC = 2 * (len(self.gta.get_free_param_vector()) - self.gta._roi_data['loglike']) self.gta.write_roi('extFit') self.gta.load_roi('n1SourcesFit', reload_sources=True) self.gta.write_roi('nSourcesFit') else: if TSext > TSextMin: self.gta.load_roi('extFit', reload_sources=True) break else: self.gta.load_roi('nSourcesFit', reload_sources=True) break self.gta.fit() # Get source radius depending on spatial model endSources = self.gta.get_sources() for i in range(len(endSources)): if endSources[i]['name'] == 'TESTSOURCE': self.target = endSources[i] self.distance = self.gta.roi.skydir.separation( endSources[i].skydir).value if endSources[i].extended == True: self.targetRadius = endSources[i]['SpatialWidth'] else: self.targetRadius = endSources[i]['pos_r95'] ''' CHECK OVERLAP ''' def overlapDisk(self, rInner, radiusCatalog): print('Target radius : ', self.targetRadius) # Check radius sizes if radiusCatalog < self.targetRadius: r = float(radiusCatalog) R = float(self.targetRadius) else: r = float(self.targetRadius) R = float(radiusCatalog) # Estimating overlapping area d = self.distance print('Distance from center : ', d) if d < (r + R): if R < (r + d): area = r**2 * np.arccos( (d**2 + r**2 - R**2) / (2 * d * r)) + R**2 * np.arccos( (d**2 + R**2 - r**2) / (2 * d * R)) - 0.5 * np.sqrt( (-d + r + R) * (d + r - R) * (d - r + R) * (d + r + R)) overlap = round((area / (np.pi * r**2)) * 100, 2) else: area = np.pi * r**2 overlap = 100.0 else: area = 0. overlap = 0. print('Overlapping surface : ', area) print('Overlap : ', overlap) if overlap > 68. and self.distance < rInner: associated = True else: associated = False return associated ''' CHECK UPPER LIMIT ''' def upperLimit(self, name, radius): sourceModel = { 'SpectrumType': 'PowerLaw', 'Index': 2.0, 'Scale': 30000, 'Prefactor': 1.e-15, 'SpatialModel': 'RadialDisk', 'SpatialWidth': radius, 'glon': self.gta.config['selection']['glon'], 'glat': self.gta.config['selection']['glat'] } self.gta.add_source(name, sourceModel, free=True) self.gta.fit() self.gta.residmap(prefix='upperLimit', make_plots=True) print('Upper limit : ', self.gta.get_sources()[0]['flux_ul95'])
def _get_fermipy_instance(configuration, likelihood_model): """ Generate a 'model' configuration section for fermipy starting from a likelihood model from astromodels :param configuration: a dictionary containing the configuration for fermipy :param likelihood_model: the input likelihood model from astromodels :type likelihood_model: astromodels.Model :return: a dictionary with the 'model' section of the fermipy configuration """ # Generate a new 'model' section in the configuration which reflects the model # provided as input # Get center and radius of ROI ra_center = float(configuration["selection"]["ra"]) dec_center = float(configuration["selection"]["dec"]) roi_width = float(configuration["binning"]["roiwidth"]) roi_radius = old_div(roi_width, np.sqrt(2.0)) # Get IRFS irfs = evclass_irf[int(configuration["selection"]["evclass"])] log.info(f"Using IRFs {irfs}") if "gtlike" in configuration and "irfs" in configuration["gtlike"]: if irfs.upper() != configuration["gtlike"]["irfs"].upper(): log.critical( "Evclass points to IRFS %s, while you specified %s in the " "configuration" % (irfs, configuration["gtlike"]["irfs"])) else: if not "gtlike" in configuration: configuration["gtlike"] = {} configuration["gtlike"]["irfs"] = irfs # The fermipy model is just a dictionary. It corresponds to the 'model' section # of the configuration file (http://fermipy.readthedocs.io/en/latest/config.html#model) fermipy_model = {} # Find Galactic and Isotropic templates appropriate for this IRFS # (information on the ROI is used to cut the Galactic template, which speeds up the # analysis a lot) # NOTE: these are going to be absolute paths galactic_template = str( sanitize_filename( findGalacticTemplate(irfs, ra_center, dec_center, roi_radius), True # noqa: F821 )) isotropic_template = str( sanitize_filename(findIsotropicTemplate(irfs), True)) # noqa: F821 # Add them to the fermipy model fermipy_model["galdiff"] = galactic_template fermipy_model["isodiff"] = isotropic_template # Now iterate over all sources contained in the likelihood model sources = [] # point sources for point_source in list(likelihood_model.point_sources.values() ): # type: astromodels.PointSource this_source = { "Index": 2.56233, "Scale": 572.78, "Prefactor": 2.4090e-12 } this_source["name"] = point_source.name this_source["ra"] = point_source.position.ra.value this_source["dec"] = point_source.position.dec.value # The spectrum used here is unconsequential, as it will be substituted by a FileFunction # later on. So I will just use PowerLaw for everything this_source["SpectrumType"] = "PowerLaw" sources.append(this_source) # extended sources for extended_source in list(likelihood_model.extended_sources.values() ): # type: astromodels.ExtendedSource raise NotImplementedError("Extended sources are not supported yet") # Add all sources to the model fermipy_model["sources"] = sources # Now we can finally instance the GTAnalysis instance configuration["model"] = fermipy_model gta = GTAnalysis(configuration) # noqa: F821 # This will take a long time if it's the first time we run with this model gta.setup() # Substitute all spectra for point sources with FileSpectrum, so that we will be able to control # them from 3ML energies_keV = None for point_source in list(likelihood_model.point_sources.values() ): # type: astromodels.PointSource # Fix this source, so fermipy will not optimize by itself the parameters gta.free_source(point_source.name, False) # This will substitute the current spectrum with a FileFunction with the same shape and flux gta.set_source_spectrum(point_source.name, "FileFunction", update_source=False) # Get the energies at which to evaluate this source this_log_energies, _flux = gta.get_source_dnde(point_source.name) this_energies_keV = (10**this_log_energies * 1e3 ) # fermipy energies are in GeV, we need keV if energies_keV is None: energies_keV = this_energies_keV else: # This is to make sure that all sources are evaluated at the same energies if not np.all(energies_keV == this_energies_keV): log.critical( "All sources should be evaluated at the same energies.") dnde = point_source(energies_keV) # ph / (cm2 s keV) dnde_per_MeV = dnde * 1000.0 # ph / (cm2 s MeV) gta.set_source_dnde(point_source.name, dnde_per_MeV, False) # Same for extended source for extended_source in list(likelihood_model.extended_sources.values() ): # type: astromodels.ExtendedSource raise NotImplementedError("Extended sources are not supported yet") return gta, energies_keV
def main(): usage = "usage: %(prog)s -c config.yaml" description = "Run the lc analysis" parser = argparse.ArgumentParser(usage=usage, description=description) parser.add_argument('-c', '--conf', required=True) parser.add_argument('-i', required=False, default=0, help='Set local or scratch calculation', type=int) parser.add_argument('--state', help='analysis state', choices=['avgspec', 'setup'], default='avgspec') parser.add_argument('--forcepl', default=0, help='Force the target source to have power-law shape', type=int) parser.add_argument('--createsed', default=0, help='Create SED from best fit model', type=int) parser.add_argument( '--adaptive', default=0, help='Use adaptive binning for minute scale light curves', type=int) parser.add_argument('--srcprob', default = 0, help='Calculate the source probability for the photons,' \ ' only works when no sub orbit time scales are used', type=int) parser.add_argument( '--mincounts', default=2, help='Minimum number of counts within LC bin to run analysis', type=int) parser.add_argument('--simulate', default = None, help='None or full path to yaml file which contains src name' \ 'and spec to be simulated', ) parser.add_argument( '--make_plots', default=0, type=int, help='Create sed plot', ) parser.add_argument( '--randomize', default=1, help= 'If you simulate, use Poisson realization. If false, use Asimov data set', type=int) args = parser.parse_args() utils.init_logging('DEBUG') config = yaml.load(open(args.conf)) tmpdir, job_id = lsf.init_lsf() if not job_id: job_id = args.i logging.info('tmpdir: {0:s}, job_id: {1:n}'.format(tmpdir, job_id)) os.chdir(tmpdir) # go to tmp directory logging.info('Entering directory {0:s}'.format(tmpdir)) logging.info('PWD is {0:s}'.format(os.environ["PWD"])) # copy the ft1,ft2 and ltcube files #for k in ['evfile','scfile','ltcube']: # don't stage them, done automatically by fermipy if needed # config[k] = utils.copy2scratch(config[k], tmpdir) # set the scratch directories logging.debug(config['data']) config['fileio']['scratchdir'] = tmpdir # set the log file logdir = copy.deepcopy(config['fileio']['logfile']) config['fileio']['logfile'] = path.join(tmpdir, 'fermipy.log') # debugging: all files will be saved (default is False) #config['fileio']['savefits'] = True # if simulating an orbit, save fits files if args.simulate is not None: config['fileio']['savefits'] = True # copy all fits files already present in outdir # run the analysis lc_config = copy.deepcopy(config['lightcurve']) fit_config = copy.deepcopy(config['fit_pars']) # remove parameters from config file not accepted by fermipy for k in ['configname', 'tmp', 'log', 'fit_pars']: config.pop(k, None) if 'adaptive' in config['lightcurve'].keys(): config['lightcurve'].pop('adaptive', None) # set the correct time bin config['selection']['tmin'], config['selection']['tmax'], nj = set_lc_bin( config['selection']['tmin'], config['selection']['tmax'], config['lightcurve']['binsz'], job_id - 1 if job_id > 0 else 0, ft1=config['data']['evfile']) logging.debug('setting light curve bin' + \ '{0:n}, between {1[tmin]:.0f} and {1[tmax]:.0f}'.format(job_id, config['selection'])) if args.adaptive: config['fileio']['outdir'] = utils.mkdir( path.join(config['fileio']['outdir'], 'adaptive{0:.0f}/'.format(lc_config['adaptive']))) if args.state == 'setup': config['fileio']['outdir'] = utils.mkdir( path.join(config['fileio']['outdir'], 'setup{0:05n}/'.format(job_id if job_id > 0 else 1))) else: config['fileio']['outdir'] = utils.mkdir( path.join(config['fileio']['outdir'], '{0:05n}/'.format(job_id if job_id > 0 else 1))) logging.info('Starting with fermipy analysis') logging.info('using fermipy version {0:s}'.format(fermipy.__version__)) logging.info('located at {0:s}'.format(fermipy.__file__)) if config['data']['ltcube'] == '': config['data'].pop('ltcube', None) compute_sub_gti_lc = False if type(config['lightcurve']['binsz']) == str: if len(config['lightcurve']['binsz'].strip('gti')): compute_sub_gti_lc = True if config['lightcurve']['binsz'].find('min') > 0: config['lightcurve']['binsz'] = float( config['lightcurve']['binsz'].strip('gti').strip( 'min')) * 60. logging.info("set time bin length to {0:.2f}s".format( config['lightcurve']['binsz'])) else: config['lightcurve']['binsz'] = 3. * 3600. try: gta = GTAnalysis(config, logging={'verbosity': 3}) except Exception as e: logging.error("{0}".format(e)) config['selection']['target'] = None gta = GTAnalysis(config, logging={'verbosity': 3}) sep = gta.roi.sources[0]['offset'] logging.warning( "Source closets to ROI center is {0:.3f} degree away".format(sep)) if sep < 0.1: config['selection']['target'] = gta.roi.sources[0]['name'] gta.config['selection']['target'] = config['selection']['target'] logging.info("Set target to {0:s}".format( config['selection']['target'])) # stage the full time array analysis results to the tmp dir # do not copy png images files = [ fn for fn in glob(fit_config['avgspec']) if fn.find('.xml') > 0 or fn.find('.npy') > 0 ] files += [config['data']['evfile']] utils.copy2scratch(files, gta.workdir) # we're using actual data if args.simulate is None: # check before the analysis start if there are any events in the master file # in the specified time range logging.info('Checking for events in initial ft1 file') t = Table.read(path.join(gta.workdir, path.basename(config['data']['evfile'])), hdu='EVENTS') logging.info("times in base ft1: {0} {1} {2}".format( t["TIME"].max(), t["TIME"].min(), t["TIME"].max() - t["TIME"].min())) m = (t["TIME"] >= config['selection']['tmin']) & ( t["TIME"] <= config['selection']['tmax']) if np.sum(m) < args.mincounts + 1: logging.error( "*** Only {0:n} events between tmin and tmax! Exiting".format( np.sum(m))) assert np.sum(m) > args.mincounts else: logging.info("{0:n} events between tmin and tmax".format( np.sum(m))) # check how many bins are in each potential light curve bin if compute_sub_gti_lc: # select time of first and last # photon instead of GTI time m = (t["TIME"] >= config['selection']['tmin']) & \ (t["TIME"] <= config['selection']['tmax']) tmin = t["TIME"][m].min() - 1. tmax = t["TIME"][m].max() + 1. logging.info("There will be up to {0:n} time bins".format(np.ceil( (tmax - tmin) / \ config['lightcurve']['binsz']))) bins = np.arange(tmin, tmax, config['lightcurve']['binsz']) bins = np.concatenate([bins, [config['selection']['tmax']]]) counts = calc_counts(t, bins) # remove the starting times of the bins with zero counts # and rebin the data logging.info("Counts before rebinning: {0}".format(counts)) mincounts = 10. mc = counts < mincounts if np.sum(mc): # remove trailing zeros if np.any(counts == 0.): mcounts_post, mcounts_pre = rm_trailing_zeros(counts) counts = counts[mcounts_post & mcounts_pre] bins = np.concatenate([ bins[:-1][mcounts_post & mcounts_pre], [bins[1:][mcounts_post & mcounts_pre].max()] ]) bins = rebin(counts, bins) logging.info("Bin lengths after rebinning: {0}".format( np.diff(bins))) logging.info("Bin times after rebinning: {0}".format(bins)) counts = calc_counts(t, bins) logging.info("Counts after rebinning: {0}".format(counts)) else: logging.info("Regular time binning will be used") bins = list(bins) logging.info('Running fermipy setup') try: gta.setup() except (RuntimeError, IndexError) as e: logging.error( 'Caught Runtime/Index Error while initializing analysis object') logging.error('Printing error:') logging.error(e) if e.message.find("File not found") >= 0 and e.message.find( 'srcmap') >= 0: logging.error("*** Srcmap calculation failed ***") if e.message.find("NDSKEYS") >= 0 and e.message.find('srcmap') >= 0: logging.error( "*** Srcmap calculation failed with NDSKEYS keyword not found in header ***" ) logging.info("Checking if there are events in ft1 file") ft1 = path.join(gta.workdir, 'ft1_00.fits') f = glob(ft1) if not len(f): logging.error( "*** no ft1 file found at location {0:s}".format(ft1)) raise t = Table.read(f[0], hdu='EVENTS') if not len(t): logging.error("*** The ft1 file contains no events!! ***".format( len(t))) else: logging.info("The ft1 file contains {0:n} event(s)".format(len(t))) return # end here if you only want to calulate # intermediate fits files if args.state == 'setup': return gta logging.info('Loading the fit for the average spectrum') gta.load_roi('avgspec') # reload the average spectral fit logging.info('Running fermipy optimize and fit') # we're using actual data if args.simulate is None: if args.forcepl: gta = set_src_spec_pl( gta, gta.get_source_name(config['selection']['target'])) # to do add EBL absorption at some stage ... # gta = add_ebl_atten(gta, gta.get_source_name(config['selection']['target']), fit_config['z']) # make sure you are fitting data gta.simulate_roi(restore=True) if compute_sub_gti_lc: if args.adaptive: # do import only here since root must be compiled from fermiAnalysis import adaptivebinning as ab # compute the exposure energy = 1000. texp, front, back = ab.comp_exposure_phi(gta, energy=1000.) # compute the bins result = ab.time_bins( gta, texp, 0.5 * (front + back), #critval = 20., # bins with ~20% unc critval=lc_config['adaptive'], Epivot=None, # compute on the fly # tstart = config['selection']['tmin'], # tstop = config['selection']['tmax'] ) # cut the bins to this GTI mask = result['tstop'] > config['selection']['tmin'] mask = mask & (result['tstart'] < config['selection']['tmax']) # try again with catalog values if not np.sum(mask): logging.error( "Adaptive bins outside time window, trying catalog values for flux" ) result = ab.time_bins( gta, texp, 0.5 * (front + back), critval=lc_config['adaptive'], # bins with ~20% unc Epivot=None, # compute on the fly forcecatalog=True, # tstart = config['selection']['tmin'], # tstop = config['selection']['tmax'] ) # cut the bins to this GTI mask = result['tstop'] > config['selection']['tmin'] mask = mask & (result['tstart'] < config['selection']['tmax']) if not np.sum(mask): logging.error( "Adaptive bins do not cover selected time interval!" ) logging.error("Using original bins") else: bins = np.concatenate((result['tstart'][mask], [result['tstop'][mask][-1]])) bins[0] = np.max( [config['selection']['tmin'], bins[0]]) bins[-1] = np.min( [config['selection']['tmax'], bins[-1]]) bins = list(bins) # removing trailing zeros counts = calc_counts(t, bins) mcounts_post, mcounts_pre = rm_trailing_zeros(counts) logging.info( "count masks: {0} {1}, bins: {2}, counts: {3}". format(mcounts_post, mcounts_pre, bins, counts)) counts = counts[mcounts_post & mcounts_pre] bins = np.concatenate([ np.array(bins)[:-1][mcounts_post & mcounts_pre], [ np.array(bins)[1:][mcounts_post & mcounts_pre].max() ] ]) logging.info( "Using bins {0}, total n={1:n} bins".format( bins, len(bins) - 1)) logging.info("bins widths : {0}".format(np.diff(bins))) logging.info("counts per bin: {0} ".format( calc_counts(t, bins))) bins = list(bins) # TODO: test that this is working also with GTIs that have little or no counts lc = gta.lightcurve( config['selection']['target'], binsz=config['lightcurve']['binsz'], free_background=config['lightcurve']['free_background'], free_params=config['lightcurve']['free_params'], free_radius=config['lightcurve']['free_radius'], make_plots=False, multithread=True, nthread=4, #multithread = False, #nthread = 1, save_bin_data=True, shape_ts_threshold=16., use_scaled_srcmap=True, use_local_ltcube=True, write_fits=True, write_npy=True, time_bins=bins, outdir='{0:.0f}s'.format(config['lightcurve']['binsz'])) else: # run the fitting of the entire time and energy range try: o = gta.optimize() # perform an initial fit logging.debug(o) except RuntimeError as e: logging.error("Error in optimize: {0}".format(e)) logging.info("Trying to continue ...") gta = set_free_pars_lc(gta, config, fit_config) f = gta.fit() if 'fix_sources' in fit_config.keys(): skip = fit_config['fix_sources'].keys() else: skip = [] gta, f = refit(gta, config['selection']['target'], f, fit_config['ts_fixed'], skip=skip) gta.print_roi() gta.write_roi('lc') if args.createsed: if args.make_plots: init_matplotlib_backend() gta.load_roi('lc') # reload the average spectral fit logging.info('Running sed for {0[target]:s}'.format( config['selection'])) sed = gta.sed(config['selection']['target'], prefix = 'lc_sed', free_radius = None if config['sed']['free_radius'] == 0. \ else config['sed']['free_radius'], free_background= config['sed']['free_background'], free_pars = fa.allnorm, make_plots = args.make_plots, cov_scale = config['sed']['cov_scale'], use_local_index = config['sed']['use_local_index'], bin_index = config['sed']['bin_index'] ) # debugging: calculate sed and resid maps for each light curve bin #logging.info('Running sed for {0[target]:s}'.format(config['selection'])) #sed = gta.sed(config['selection']['target'], prefix = 'lc') #model = {'Scale': 1000., 'Index' : fit_config['new_src_pl_index'], 'SpatialModel' : 'PointSource'} #resid_maps = gta.residmap('lc',model=model, make_plots=True, write_fits = True, write_npy = True) if args.srcprob: logging.info("Running srcprob with srcmdl {0:s}".format('lc')) gta.compute_srcprob(xmlfile='lc', overwrite=True) # we are simulating a source else: # TODO: I probably have to run the setup here. Do on weekly files, i.e., no time cut? Only do that later? with open(args.simulate) as f: simsource = np.load(f, allow_pickle=True).flat[0] # set the source to the simulation value gta.set_source_spectrum( simsource['target'], spectrum_type=simsource['spectrum_type'], spectrum_pars=simsource['spectrum_pars'][job_id - 1]) logging.info("changed spectral parameters to {0}".format( gta.roi.get_source_by_name(simsource['target']).spectral_pars)) # simulate the ROI gta.simulate_roi(randomize=bool(args.randomize)) gta = set_free_pars_lc(gta, config, fit_config) # fit the simulation f = gta.fit() gta, f = refit(gta, config['selection']['target'], f, fit_config['ts_fixed']) gta.print_roi() gta.write_roi('lc_simulate_{0:s}'.format(simsource['suffix'])) return gta
def _process_lc_bin(itime, name, config, basedir, workdir, diff_sources, const_spectrum, roi, lck_params, **kwargs): i, time = itime roi = copy.deepcopy(roi) config = copy.deepcopy(config) config['selection']['tmin'] = time[0] config['selection']['tmax'] = time[1] # create output directories labeled in MET vals outdir = basedir + 'lightcurve_%.0f_%.0f' % (time[0], time[1]) config['fileio']['outdir'] = os.path.join(workdir, outdir) config['logging']['prefix'] = 'lightcurve_%.0f_%.0f ' % (time[0], time[1]) config['fileio']['logfile'] = os.path.join(config['fileio']['outdir'], 'fermipy.log') utils.mkdir(config['fileio']['outdir']) yaml.dump(utils.tolist(config), open(os.path.join(config['fileio']['outdir'], 'config.yaml'), 'w')) xmlfile = os.path.join(config['fileio']['outdir'], 'base.xml') try: from fermipy.gtanalysis import GTAnalysis gta = GTAnalysis(config, roi, loglevel=logging.DEBUG) gta.logger.info('Fitting time range %i %i' % (time[0], time[1])) gta.setup() except: print('Analysis failed in time range %i %i' % (time[0], time[1])) print(sys.exc_info()[0]) raise return {} gta._lck_params = lck_params # Recompute source map for source of interest and sources within 3 deg if gta.config['gtlike']['use_scaled_srcmap']: names = [s.name for s in gta.roi.get_sources(distance=3.0, skydir=gta.roi[name].skydir) if not s.diffuse] gta.reload_sources(names) # Write the current model gta.write_xml(xmlfile) # Optimize the model gta.optimize(skip=diff_sources, shape_ts_threshold=kwargs.get('shape_ts_threshold')) fit_results = _fit_lc(gta, name, **kwargs) gta.write_xml('fit_model_final.xml') srcmodel = copy.deepcopy(gta.get_src_model(name)) numfree = gta.get_free_param_vector().count(True) max_ts_thresholds = [None, 4, 9] for max_ts in max_ts_thresholds: if max_ts is not None: gta.free_sources(minmax_ts=[None, max_ts], free=False, exclude=[name]) # rerun fit using params from full time (constant) fit using same # param vector as the successful fit to get loglike specname, spectrum = const_spectrum gta.set_source_spectrum(name, spectrum_type=specname, spectrum_pars=spectrum, update_source=False) gta.free_source(name, free=False) const_fit_results = gta.fit() if not const_fit_results['fit_success']: continue const_srcmodel = gta.get_src_model(name) # rerun using shape fixed to full time fit # for the fixed-shape lightcurve gta.free_source(name, pars='norm') fixed_fit_results = gta.fit() if not fixed_fit_results['fit_success']: continue fixed_srcmodel = gta.get_src_model(name) break # special lc output o = {'flux_const': const_srcmodel['flux'], 'loglike_const': const_fit_results['loglike'], 'fit_success': fit_results['fit_success'], 'fit_success_fixed': fixed_fit_results['fit_success'], 'fit_quality': fit_results['fit_quality'], 'fit_status': fit_results['fit_status'], 'num_free_params': numfree, 'config': config} # full flux output if fit_results['fit_success'] == 1: for k in defaults.source_flux_output.keys(): if not k in srcmodel: continue o[k] = srcmodel[k] o[k+'_fixed'] = fixed_srcmodel[k] gta.logger.info('Finished time range %i %i' % (time[0], time[1])) return o
def _process_lc_bin(itime, name, config, basedir, workdir, diff_sources, const_spectrum, roi, lck_params, **kwargs): i, time = itime roi = copy.deepcopy(roi) config = copy.deepcopy(config) config['selection']['tmin'] = time[0] config['selection']['tmax'] = time[1] # create output directories labeled in MET vals outdir = basedir + 'lightcurve_%.0f_%.0f' % (time[0], time[1]) config['fileio']['outdir'] = os.path.join(workdir, outdir) config['logging']['prefix'] = 'lightcurve_%.0f_%.0f ' % (time[0], time[1]) config['fileio']['logfile'] = os.path.join(config['fileio']['outdir'], 'fermipy.log') utils.mkdir(config['fileio']['outdir']) yaml.dump(utils.tolist(config), open(os.path.join(config['fileio']['outdir'], 'config.yaml'), 'w')) xmlfile = os.path.join(config['fileio']['outdir'], 'base.xml') try: from fermipy.gtanalysis import GTAnalysis gta = GTAnalysis(config, roi, loglevel=logging.DEBUG) gta.logger.info('Fitting time range %i %i' % (time[0], time[1])) gta.setup() except: print('Analysis failed in time range %i %i' % (time[0], time[1])) print(sys.exc_info()[0]) raise return {} gta._lck_params = lck_params # Recompute source map for source of interest and sources within 3 deg if gta.config['gtlike']['use_scaled_srcmap']: names = [s.name for s in gta.roi.get_sources(distance=3.0, skydir=gta.roi[name].skydir) if not s.diffuse] gta.reload_sources(names) # Write the current model gta.write_xml(xmlfile) # Optimize the model gta.optimize(skip=diff_sources, shape_ts_threshold=kwargs.get('shape_ts_threshold')) fit_results = _fit_lc(gta, name, **kwargs) gta.write_xml('fit_model_final.xml') srcmodel = copy.deepcopy(gta.get_src_model(name)) numfree = gta.get_free_param_vector().count(True) const_srcmodel = gta.get_src_model(name).copy() fixed_fit_results = fit_results.copy() fixed_srcmodel = gta.get_src_model(name).copy() fixed_fit_results['fit_success'],fixed_srcmodel['fit_success'] = [False,False] fixed_fit_results['fit_quality'],fixed_srcmodel['fit_quality'] = [0,0] max_ts_thresholds = [None, 4, 9, 16, 25] for max_ts in max_ts_thresholds: if max_ts is not None: gta.free_sources(minmax_ts=[None, max_ts], free=False, exclude=[name]) # rerun fit using params from full time (constant) fit using same # param vector as the successful fit to get loglike specname, spectrum = const_spectrum gta.set_source_spectrum(name, spectrum_type=specname, spectrum_pars=spectrum, update_source=False) gta.free_source(name, free=False) const_fit_results = gta.fit() if not const_fit_results['fit_success']: continue const_srcmodel = gta.get_src_model(name) # rerun using shape fixed to full time fit # for the fixed-shape lightcurve gta.free_source(name, pars='norm') fixed_fit_results = gta.fit() if not fixed_fit_results['fit_success']: continue fixed_srcmodel = gta.get_src_model(name) break # special lc output o = {'flux_const': const_srcmodel['flux'], 'loglike_const': const_fit_results['loglike'], 'fit_success': fit_results['fit_success'], 'fit_success_fixed': fixed_fit_results['fit_success'], 'fit_quality': fit_results['fit_quality'], 'fit_status': fit_results['fit_status'], 'num_free_params': numfree, 'config': config} # full flux output if fit_results['fit_success'] == 1: for k in defaults.source_flux_output.keys(): if not k in srcmodel: continue o[k] = srcmodel[k] o[k+'_fixed'] = fixed_srcmodel[k] gta.logger.info('Finished time range %i %i' % (time[0], time[1])) return o