def get_hess_lightcurve(filename='pks-hess-data.csv'):
"""
Get HESS lightcurve
Parameters
----------
filename : str, optional
Light curve data file name
Returns
-------
x, y, y_err : tuple of floats
Energy, flux and flux error
"""
# Open data file
file1 = gammalib.GFilename('$CTAGITROOT/analysis/hess_dr1/%s' % filename)
file2 = gammalib.GFilename('%s' % filename)
if file1.exists():
csv = gammalib.GCsv(file1,'\t')
else:
csv = gammalib.GCsv(file2,'\t')
# Initialise arrays
x = []
y = []
y_err = []
# Loop over data
for row in range(csv.nrows()):
x.append(float(csv[row,0])+1.0/(24.0*60.0))
y.append(float(csv[row,1])*1.0e-9)
y_err.append((float(csv[row,2])-float(csv[row,1]))*1.0e-9)
# Return
return x,y,y_err
def __init__(self, *argv):
"""
Constructor
"""
# Set name and version
self._name = 'cstsmapsplit'
self._version = '1.1.0'
# Set data members
self._outmap = gammalib.GFilename()
self._bins_per_job = 0
self._compute_null = False
self._outfile = gammalib.GFilename()
self._map = gammalib.GSkyMap()
self._cmd = []
self._srcname = ''
# Initialise observation container from constructor arguments.
self._obs, argv = self._set_input_obs(argv)
# Initialise application by calling the appropriate class
# constructor.
self._init_cscript(argv)
# Return
return
def _check_copy(self, pathname):
"""
Check copy of IACT data store
Parameters
----------
pathname : str
Path to copied IACT data store
"""
# Set file names
hdu_index_name = gammalib.GFilename(pathname+'/hdu-index.fits')
obs_index_name = gammalib.GFilename(pathname+'/obs-index.fits')
master_name = gammalib.GFilename(pathname+'/master.json')
# Check for existence of files
self.test_assert(hdu_index_name.exists(),
'Check if file "hdu-index.fits" exists')
self.test_assert(obs_index_name.exists(),
'Check if file "obs-index.fits" exists')
self.test_assert(master_name.exists(),
'Check if file "master.json" exists')
# Check if index files are FITS file
self.test_assert(hdu_index_name.is_fits(),
'Check if file "hdu-index.fits" is FITS file')
self.test_assert(obs_index_name.is_fits(),
'Check if file "obs-index.fits" is FITS file')
# Return
return
def _check_copy(self, pathname):
"""
Check copy.
"""
# Set file names
hdu_index_name = gammalib.GFilename(pathname + '/hdu-index.fits')
obs_index_name = gammalib.GFilename(pathname + '/obs-index.fits')
master_name = gammalib.GFilename(pathname + '/master.json')
# Check for existence of files
self.test_assert(hdu_index_name.exists(),
'Check if file "hdu-index.fits" exists.')
self.test_assert(obs_index_name.exists(),
'Check if file "obs-index.fits" exists.')
self.test_assert(master_name.exists(),
'Check if file "master.json" exists.')
# Check if index files are FITS file
self.test_assert(hdu_index_name.is_fits(),
'Check if file "hdu-index.fits" is FITS file.')
self.test_assert(obs_index_name.is_fits(),
'Check if file "obs-index.fits" is FITS file.')
# Return
return
def run(self):
"""
Run the script
"""
# Switch screen logging on in debug mode
if self._logDebug():
self._log.cout(True)
# Get parameters
self._get_parameters()
# Write header into logger
if self._logTerse():
self._log('\n')
self._log.header1('Data storage entries')
self._log.parformat('Master index file')
self._log(self._master_file)
self._log('\n')
# Open and load JSON file
json_data = open(self._master_file).read()
data = json.loads(json_data)
configs = data['datasets']
# Initialise array for available names
self._prodnames = []
# Write header into logger
self._log('\n')
self._log.header2('Available data configs')
# Loop over configs and log available configs
for config in configs:
# Create hdu and obs index files
hdu = os.path.join(self._datapath, config['hduindx'])
obs = os.path.join(self._datapath, config['obsindx'])
filename_hdu = gammalib.GFilename(str(hdu) + '[HDU_INDEX]')
filename_obs = gammalib.GFilename(str(obs) + '[OBS_INDEX]')
# If index files are available then log configuration
# information ...
if (filename_hdu.is_fits() and filename_obs.is_fits()):
# Append to available names
self._prodnames.append(str(config['name']))
# Log information
self._log_configuration(config)
# ... otherwise log that the configuration is not available
else:
if self._logTerse():
self._log.header3(str(config['name']))
self._log(' Not available\n')
# Return
return
def __init__(self, *argv):
"""
Constructor.
"""
# Set name
self._name = "cstsdist"
self._version = "1.1.0"
# Initialise some members
self._obs = gammalib.GObservations()
self._pattern = "single"
self._srcname = ""
self._enumbins = 0
self._npix = 0
self._binsz = 0.0
self._outfile = gammalib.GFilename("ts.dat")
self._ntrials = 10
self._edisp = False
self._debug = False
self._log_clients = False
# Initialise application by calling the appropriate class
# constructor.
self._init_cscript(argv)
# Return
return
def _check_n_obs(self, pathname, n_expected):
"""
Check number of available observations
Parameters
----------
pathname : str
Path to copied IACT data store
n_expected : int
Expected number of observations in IACT data store
"""
# Set file name
obs_index_name = gammalib.GFilename(pathname+'/obs-index.fits[OBS_INDEX]')
# Open index file
fits = gammalib.GFits(obs_index_name)
# Get number of observations
n_obs = fits[obs_index_name.extname()].nrows()
# Close FITS file
fits.close()
# Check for existence of observations
self.test_value(n_obs, n_expected, 'Check for number of observations')
# Return
return
def __init__(self, *argv):
"""
Constructor.
"""
# Set name
self._name = "csmodelinfo"
self._version = "1.1.0"
# Initialise class members
self._models = gammalib.GModels()
self._ds9file = gammalib.GFilename("NONE")
self._pnt_type = ""
self._pnt_mark_size = 12
self._show_labels = True
self._width = 2
self._fontfamily = "helvetica"
self._fontsize = 12
self._fontweight = "normal"
self._fontslant = "roman"
self._show_ext_type = True
self._free_color = "green"
self._fixed_color = "magenta"
# Initialise application by calling the appropriate class
# constructor.
self._init_cscript(argv)
# Return
return
def _test_cmd(self):
"""
Test csworkflow on the command line
"""
# Set script name
csworkflow = self._script('csworkflow')
# Remove result file
gammalib.GFilename('wf_crab_results.xml').remove()
# Setup csworkflow command
cmd = csworkflow+' inflow="'+self._workflow+'"'+ \
' logfile="csworkflow_cmd1.log" chatter=1'
# Check if execution of wrong command fails
self.test_assert(self._execute('command_that_does_not_exist') != 0,
'Self test of test script')
# Check if execution was successful
self.test_assert(self._execute(cmd) == 0,
'Check successful execution from command line')
# Check fit result
self._check_fit_result('wf_crab_results.xml')
# Return
return
def __init__(self, *argv):
"""
Constructor.
"""
# Set name and version
self._name = 'csobs2caldb'
self._version = '1.1.0'
# Initialise members
self._observation = gammalib.GCTAObservation()
self._mission = 'cta'
self._caldb = 'cta'
self._outfile = gammalib.GFilename('irf_file.fits')
self._base_dir = ''
self._cal_dir = ''
self._rsp_dir = ''
self._caldb_inx = gammalib.GFits()
self._irf_fits = gammalib.GFits()
# Initialise observation container from constructor arguments.
self._obs, argv = self._set_input_obs(argv)
# Initialise script by calling the appropriate class constructor.
self._init_cscript(argv)
# Return
return
def bin2gammalib(filename, flux_thresh, outdir):
data = open(filename).readlines()
# convert phasogram to numpy array
phasogram = np.genfromtxt(data[14:])
# only include model if peak in lightcurve is > threshold
if np.max(phasogram[:, 2]) > flux_thresh:
# final name
num = int(data[0].split('#')[-1])
name = 'bin{:03d}'.format(num)
# spatial model
# read Galactic longitude
lon = float(data[1].split(' ')[-1])
# Galactic latitude is random with Gaussian distribution with 95% containment at bmax
lat = np.random.normal(0, bmax / 2)
src_dir = gammalib.GSkyDir()
src_dir.lb_deg(lon, lat)
spatial = gammalib.GModelSpatialPointSource(src_dir)
# spectral model
# average flux in ph/cm2/s
f0 = np.average(phasogram[:, 1])
# convert to differential flux at eref in ph/cm2/s/MeV
n0 = (gamma - 1) * f0 / eref
spectral = gammalib.GModelSpectralPlaw(
n0, -gamma, gammalib.GEnergy(np.double(eref), 'MeV'))
# orbital phase model
# create phasogram file
fname = 'phasecurve_' + name + '.fits'
phasogram_file(phasogram, outdir + fname, outdir)
# period in days
per = float(data[4].split(' ')[-1])
# convert to frequency in s
per *= 86400
f0 = 1 / per
temporal = gammalib.GModelTemporalPhaseCurve(
gammalib.GFilename(fname),
t0,
np.random.random(), # phase at t0 randomly drawn
f0,
0.,
0., # orbit is stable, no derivatives
1.,
True) # normalized so that spectral model represents average flux
# assemble final model
model = gammalib.GModelSky(spatial, spectral, temporal)
model.name(name)
else:
# empty model
lat = 0.
model = gammalib.GModelSky()
return model, lat
def __init__(self, *argv):
"""
Constructor.
"""
# Set name and version
self._name = 'csiactobs'
self._version = '1.1.0'
# Initialise some members
self._ebounds = gammalib.GEbounds()
self._datapath = os.getenv('VHEFITS', '')
self._inmodels = gammalib.GModels()
self._prodname = ''
self._xml = gammalib.GXml()
self._models = gammalib.GModels()
self._runlist = []
self._runlistfile = gammalib.GFilename()
self._bkgpars = 0
self._outmodel = gammalib.GFilename()
self._outobs = gammalib.GFilename()
self._master_indx = ''
self._use_bkg_scale = False
self._ev_hiera = ['']
self._aeff_hiera = ['']
self._psf_hiera = ['']
self._bkg_hiera = ['']
self._edisp_hiera = ['']
self._bkg_mod_hiera = ['']
self._bkg_gauss_norm = 1.0
self._bkg_gauss_index = 0.0
self._bkg_gauss_sigma = 1.0
self._bkg_aeff_index = 0.0
self._bkg_aeff_norm = 1.0
self._bkg_range_factor = 1.0
self._hdu_index = ''
self._obs_index = ''
self._subdir = ''
self._debug = False
# Initialise application by calling the appropriate class
# constructor.
self._init_cscript(argv)
# Return
return
def __init__(self, *argv):
"""
Constructor
"""
# Initialise application by calling the base class constructor
self._init_cscript(self.__class__.__name__, ctools.__version__, argv)
# Set name
self._datapath = os.getenv('VHEFITS', '')
self._remote_master = gammalib.GFilename()
self._remote_base = ''
self._prodname = ''
self._outpath = ''
self._runlist = gammalib.GFilename()
self._runs = []
# Return
return
def __init__(self, *argv):
"""
Constructor
"""
# Initialise application by calling the appropriate class constructor
self._init_cslikelihood(self.__class__.__name__, ctools.__version__,
argv)
# Set data members
self._outmap = gammalib.GFilename()
self._bins_per_job = 0
self._compute_null = False
self._outfile = gammalib.GFilename()
self._map = gammalib.GSkyMap()
self._cmd = []
self._srcname = ''
# Return
return
def make_xml(nametag):
spatial = gammalib.GModelSpatialDiffuseCube(
gammalib.GFilename(nametag + '_map.fits'))
spectral = gammalib.GModelSpectralConst(1.)
model = gammalib.GModelSky(spatial, spectral)
model.name(nametag)
# fill to model container and write to disk
models = gammalib.GModels()
models.append(model)
models.save(nametag + '.xml')
def _check_cmdfile(self, filename):
"""
Check ASCII files containing commands
"""
# Create file name instance
fname = gammalib.GFilename(filename)
# Check if execution was successful
self.test_assert(fname.exists(), 'Check of output filename exists')
# Return
return
def __init__(self, *argv):
"""
Constructor.
"""
# Set name
self._name = 'csiactcopy'
self._version = '1.1.0'
self._datapath = os.getenv('VHEFITS', '')
self._remote_master = gammalib.GFilename()
self._remote_base = ''
self._prodname = ''
self._outpath = ''
self._runlist = gammalib.GFilename()
self._runs = []
# Initialise application by calling the appropriate class
# constructor.
self._init_cscript(argv)
# Return
return
def read_regions(fpath, init_map):
"""
Read a regions WCS FITS or ds9 file and return a mask WCS map.
Parameters
----------
fpath : str
Path to regions file
init_map : `~gammalib.GSkyMap`
Sky map representing fov of interest. Used for initialisation of
regions map.
Returns
-------
map : `~gammalib.GSkyMap`
Sky map.
"""
# Create starter map for user fov filled with zeros
regions_map = init_map.copy()
regions_map *= 0.0
# Take care about ds9 region files
if fpath.lower().endswith('.reg'):
# Read ds9 regions
ds9_regions = gammalib.GSkyRegions(fpath)
# Loop over regions
for reg in ds9_regions:
# Make map from region
ds9_map = gammalib.GSkyRegionMap(reg)
# Add ds9 region map to global map
regions_map += ds9_map.map()
# Take care about fits WCS files
elif gammalib.GFilename(fpath).is_fits():
# Read wcs regions map
wcs_regions = gammalib.GSkyMap(fpath)
# Add wcs regions map to global map
regions_map += wcs_regions
else:
raise RuntimeError('Invalid regions file detected. Please provide ' +
'a valid ds9 or FITS WCS regions file.')
# Return
return regions_map
def _test_python(self):
"""
Test csworkflow from Python
"""
# Remove result file
gammalib.GFilename('wf_crab_results.xml').remove()
# Set-up csworkflow
workflow = cscripts.csworkflow()
workflow['inflow'] = self._workflow
workflow['logfile'] = 'csworkflow_py1.log'
workflow['chatter'] = 2
# Run script
workflow.logFileOpen() # Make sure we get a log file
workflow.run()
# Check fit result
self._check_fit_result('wf_crab_results.xml')
# Remove result file
gammalib.GFilename('wf_crab_results.xml').remove()
# Set-up csworkflow
workflow = cscripts.csworkflow()
workflow['inflow'] = self._workflow
workflow['logfile'] = 'csworkflow_py2.log'
workflow['chatter'] = 3
# Execute script
workflow.execute()
# Check fit result
self._check_fit_result('wf_crab_results.xml')
# Return
return
def _check_response(self, caldb, irf):
"""
Check response
"""
# Open calibration database
db = gammalib.GCaldb('caldb')
db.open('cta', caldb)
# Get filenames of response components
expr = 'NAME(' + irf + ')'
aeff = gammalib.GFilename(db.filename('', '', 'EFF_AREA', '', '',
expr))
psf = gammalib.GFilename(db.filename('', '', 'RPSF', '', '', expr))
edisp = gammalib.GFilename(db.filename('', '', 'EDISP', '', '', expr))
bkg = gammalib.GFilename(db.filename('', '', 'BKG', '', '', expr))
# Check whether files exist
self.test_assert(aeff.exists(), 'Effective area file exists')
self.test_assert(psf.exists(), 'Point spread function file exists')
self.test_assert(edisp.exists(), 'Energy dispersion file exists')
self.test_assert(bkg.exists(), 'Background file exists')
# Return
return
def __init__(self, *argv):
"""
Constructor
"""
# Initialise application by calling the base class constructor
self._init_cscript(self.__class__.__name__, ctools.__version__, argv)
# Initialise some members
self._obs = gammalib.GObservations()
self._ebounds = gammalib.GEbounds()
self._datapath = os.getenv('VHEFITS', '')
self._prodname = ''
self._xml = gammalib.GXml()
self._models = gammalib.GModels()
self._runlist = []
self._runlistfile = gammalib.GFilename()
self._bkgpars = 0
self._master_indx = ''
self._use_bkg_scale = False
self._ev_hiera = ['']
self._aeff_hiera = ['']
self._psf_hiera = ['']
self._bkg_hiera = ['']
self._edisp_hiera = ['']
self._bkg_mod_hiera = ['']
self._bkg_gauss_norm = 1.0
self._bkg_gauss_index = 0.0
self._bkg_gauss_sigma = 1.0
self._bkg_aeff_index = 0.0
self._bkg_aeff_norm = 1.0
self._bkg_range_factor = 1.0
self._hdu_index = ''
self._obs_index = ''
self._subdir = ''
self._debug = False
# Initialise empty observation definition XML file
self._xml.append(
gammalib.GXmlElement('observation_list '
'title="observation list"'))
# Append an observation list to XML instance
self._xml.append(
gammalib.GXmlElement('observation_list title="observation list"'))
self._xml_obslist = self._xml.element('observation_list', 0)
# Return
return
def run_csobsinfo(filename, ra, dec, debug=True):
"""
Run csobsinfo script
Parameters
----------
filename : str
File name of observation definition XML file
ra : str
Target Right Ascension (deg)
dec : str
Target declination of pointing (deg)
debug : bool, optional
Switch on debugging in csobsinfo run
Returns
-------
info : `~cscripts.csobsinfo`
csobsinfo instance
"""
# Get observation definition XML filename
obsdef = gammalib.GFilename(filename)
# Setup csobsinfo script
info = cscripts.csobsinfo()
info['inobs'] = obsdef.url()
info['debug'] = debug
# Set offset if the "ra" and "dec" arguments are not empty strings
if ra != '' and dec != '':
info['ra'] = float(ra)
info['dec'] = float(dec)
info['offset'] = True
else:
info['offset'] = False
# Run csobsinfo
info.run()
# Return
return info
def run_csobsinfo(filename, ra=None, dec=None, debug=True):
"""
Run csobsinfo script
Parameters
----------
filename : str
File name of observation definition XML file
ra : float, optional
Target Right Ascension (deg)
dec : float, optional
Target declination of pointing (deg)
debug : bool, optional
Switch on debugging in csobsinfo run
Returns
-------
info : `~cscripts.csobsinfo`
csobsinfo instance
"""
# Get observation definition XML filename
obsdef = gammalib.GFilename(filename)
# Setup csobsinfo script
info = cscripts.csobsinfo()
info['inobs'] = obsdef.url()
info['debug'] = debug
# Set offset
if ra != None and dec != None:
info['ra'] = ra
info['dec'] = dec
info['offset'] = True
else:
info['offset'] = False
# Run csobsinfo
info.run()
# Return
return info
def __init__(self, *argv):
"""
Constructor
"""
# Initialise application by calling the appropriate class constructor
self._init_csobservation(self.__class__.__name__, ctools.__version__,
argv)
# Initialise members
self._observation = gammalib.GCTAObservation()
self._mission = 'cta'
self._caldb = 'cta'
self._outfile = gammalib.GFilename('irf_file.fits')
self._base_dir = ''
self._cal_dir = ''
self._rsp_dir = ''
self._caldb_inx = gammalib.GFits()
self._irf_fits = gammalib.GFits()
# Return
return
def _is_present(self, filename, obs_id, filetype):
"""
Checks if a filename is present
Parameters
----------
filename : str
Filename
obs_id : int
Observation ID
filetype : str
Type of file
Returns
-------
present : bool
True if filename is present, False otherwise
"""
# Initialise return value
present = True
# Create filename instance
fname = gammalib.GFilename(filename)
# Check if file is empty
if fname.is_empty():
msg = 'Skipping observation "%s": No %s found' % (obs_id, filetype)
self._log_string(gammalib.NORMAL, msg)
present = False
# Check if file exists
elif not fname.exists():
msg = ('Skipping observation "%s": %s "%s" does not exist' %
(obs_id, filetype, filename))
self._log_string(gammalib.NORMAL, msg)
present = False
# Return present flag
return present
def __init__(self, *argv):
"""
Constructor.
"""
# Set name
self._name = 'cspull'
self._version = '1.1.0'
# Initialise some members
self._edisp = False
self._exposure = None
self._psfcube = None
self._bckcube = None
self._edispcube = None
self._stackmodels = None
self._coordsys = 'CEL'
self._proj = 'TAN'
self._log_clients = False # Static parameter
self._offset = 0.0
self._ntrials = 0
self._outfile = gammalib.GFilename()
self._npix = 0
self._binsz = 0.0
self._pattern = 'single'
self._enumbins = 0
self._seed = 1
self._chatter = 2
# Initialise observation container from constructor arguments.
self._obs, argv = self._set_input_obs(argv)
# Initialise application by calling the appropriate class
# constructor.
self._init_cscript(argv)
# Return
return
def __init__(self, *argv):
"""
Constructor.
"""
# Set name
self._name = "cssens"
self._version = "1.1.0"
# Initialise class members
self._obs = gammalib.GObservations()
self._ebounds = gammalib.GEbounds()
self._obs_ebounds = []
self._srcname = ""
self._outfile = gammalib.GFilename()
self._ra = None
self._dec = None
self._edisp = False
self._emin = 0.020
self._emax = 200.0
self._bins = 21
self._enumbins = 0
self._npix = 200
self._binsz = 0.05
self._type = "Differential"
self._ts_thres = 25.0
self._max_iter = 50
self._num_avg = 3
self._log_clients = False
self._debug = False
# Initialise application by calling the appropriate class
# constructor.
self._init_cscript(argv)
# Return
return
def _get_parameters(self):
"""
Get parameters from parfile and setup the observation.
"""
# Get parameters
if self._datapath == '':
self._datapath = self['datapath'].string()
# Expand environment
self._datapath = gammalib.expand_env(self._datapath)
# Get production name
self._prodname = self['prodname'].string()
# Master index file name
master_indx = self['master_indx'].string()
# Initialise flag if spatial selection is required
self._select_radec = True
# Initialise invalid radius
self._radius = 0.0
# Check for validity of spatial parameters
if (self['ra'].is_valid() and self['dec'].is_valid()
and self['rad'].is_valid()):
# Read spatial parameters
self._ra = self['ra'].real()
self._dec = self['dec'].real()
self._radius = self['rad'].real()
# ... otherwise signal that there are no spatial parameters for
# selection
else:
self._select_radec = False
# Check Radius for validity
if self._radius <= 0.0:
self._select_radec = False
# Query other parameters
self['min_qual'].integer()
self['expression'].string()
# Read ahead output parameters
if self._read_ahead():
self['outfile'].filename()
# Set filename of JSON master file and raise an exception if the file
# does not exist
master_file = os.path.join(self._datapath, master_indx)
if not os.path.isfile(master_file):
msg = ('FITS data store not available. No master index file found '
'at "%s". Make sure the file is copied from the server and '
'your datapath is set correctly.' % master_file)
raise RuntimeError(msg)
# Open and load JSON master file. If the "dataset" key is not available
# then raise an exception
json_data = open(master_file).read()
data = json.loads(json_data)
if not 'datasets' in data:
msg = ('Key "datasets" not available in master index file.')
raise RuntimeError(msg)
# Get configurations from JSON master file
configs = data['datasets']
# Initialise obs index file
self._obs_index = ''
# Get name of observation index file
for config in configs:
if self._prodname == config['name']:
self._obs_index = str(
os.path.join(self._datapath, config['obsindx']))
break
# If the observation index file name is empty then raise an exception
if self._obs_index == '':
msg = ('FITS data store "%s" not available. Run csiactdata to get '
'a list of available storage names.' % self._prodname)
raise RuntimeError(msg)
# If the observation index file is not a FITS file then raise an
# exception
filename = gammalib.GFilename(self._obs_index + '[OBS_INDEX]')
if not filename.is_fits():
msg = (
'Observation index file "%s[OBS_INDEX]" for FITS data store '
'"%s" not available. Check your master index file or run '
'csiactdata to get a list of available storage names.' %
(self._obs_index, self._prodname))
raise RuntimeError(msg)
# Write input parameters into logger
self._log_parameters(gammalib.TERSE)
# Return
return
filepath = model.spatial().filename().path()
# copy file to output directory
shutil.copy(filepath + filename, './')
# replace file with the one in output directory
model.spatial(gammalib.GModelSpatialDiffuseMap(filename))
# if model contains spatial map take care of it
if model.spectral().type() == 'FileFunction':
# find spectrum file name and path
filename = model.spectral().filename().file()
filepath = model.spectral().filename().path()
# copy file to output directory
shutil.copy(filepath + filename, './')
# replace file with the one in output directory
model.spectral(
gammalib.GModelSpectralFunc(
gammalib.GFilename(filename),
model.spectral()['Normalization'].value()))
# append model to container
models.append(model)
msg = 'Replaced {} gamma-cat sources with templates. Added {} sources as templates\n'.format(
replaced, added)
print(msg)
outfile.write(msg)
# re-make distributions from gammalib model container
lons, lats, radii, fluxes, names = dist_from_gammalib(models)
# change lon range from 0...360 to -180...180
lons = np.array(lons)
lons[lons > 180] = lons[lons > 180] - 360.
ax1.hist(fluxes,
def run(self):
"""
Run the script
"""
# Switch screen logging on in debug mode
if self._logDebug():
self._log.cout(True)
# Get parameters
self._get_parameters()
# Make destination directory if not available
if not os.path.isdir(self._outpath):
os.makedirs(self._outpath)
# Set run list
self._runs = self._set_runs(self._runlist)
# Check for availability of remote master file
if not self._remote_master.exists():
raise RuntimeError('*** ERROR: Remote master file "' +
self._remote_master + '" does not exist.')
# Retrieve json data from remote master
json_data = open(self._remote_master.url()).read()
data = json.loads(json_data)
if not 'datasets' in data:
raise RuntimeError('*** ERROR: Key "datasets" not available '
'in remote master index file "' +
self._remote_master + '".')
# Get array of configurations
configs = data['datasets']
# Get remote paths
self._remote_base = self._remote_master.path()
# Initialise flag if prod has been found
has_prod = False
# Initialise file names to be copied
files = set()
# Write header into logger
self._log_header2(gammalib.TERSE, 'Loop over remote configs')
# Loop over configs
for config in configs:
# Write header into logger. The str() is needed since
# "config['name']" is <type 'unicode'>, and under Python 2 a
# string is expected.
self._log_header2(gammalib.VERBOSE, str(config['name']))
# Check if prodname was found
if config['name'] == self._prodname:
# Inidicate that prodname has been found
has_prod = True
# Build path of index files
remote_hdu = str(
os.path.join(self._remote_base, config['hduindx']))
remote_obs = str(
os.path.join(self._remote_base, config['obsindx']))
# Log information
self._log_header3(gammalib.NORMAL,
'Remote config "' + self._prodname + '"')
self._log_value(gammalib.NORMAL, 'HDU index', remote_hdu)
self._log_value(gammalib.NORMAL, 'Observation index',
remote_obs)
# Open remote HDU index file
fits = gammalib.GFits(str(remote_hdu))
table = fits['HDU_INDEX']
# Initialise flag if SIZE column is present
has_size = table.contains('SIZE')
# Initialise file size
if has_size:
cp_size = 0
# Initialise remote observation IDs
remote_ids = set()
for row in range(table.nrows()):
remote_ids.add(table['OBS_ID'][row])
# Log runs that are not available remotely
for run in self._runs:
# Check for run not in remote data store
if not run in remote_ids:
msg = ('Skip observation "%s": ID not available '
'remotely' % str(run))
self._log_string(msg)
# Loop over remote HDU index file
for row in range(table.nrows()):
# Get observation ID
obs_id = table['OBS_ID'][row]
file_dir = table['FILE_DIR'][row]
file_name = table['FILE_NAME'][row]
# Skip if filename is empty
if file_name == '':
continue
# Check if we need to consider an input runlist
if len(self._runs):
# Check if obs id is in runlist
if obs_id in self._runs:
# Get filename
fname = os.path.join(os.path.dirname(remote_hdu),
file_dir, file_name)
# Add file
oldlen = len(files)
files.add(fname)
newlen = len(files)
# Add file size
if has_size and newlen > oldlen:
cp_size += table['SIZE'][row]
# Otherwise add every file
else:
# Get filename
fname = os.path.join(os.path.dirname(remote_hdu),
file_dir, file_name)
# Add file
oldlen = len(files)
files.add(fname)
newlen = len(files)
# Add file size
if has_size and newlen > oldlen:
cp_size += table['SIZE'][row]
# Log file information
self._log_header2(gammalib.NORMAL, 'File information')
self._log_value(gammalib.NORMAL, 'Number of files', len(files))
if has_size:
size = float(cp_size) * 1.0e-6
self._log_value(gammalib.NORMAL, 'Size', '%.2f MB' % size)
self._log_header3(gammalib.VERBOSE, 'File names')
for filename in files:
self._log_string(gammalib.VERBOSE, str(filename) + '\n')
# Close HDU index file
fits.close()
# If prodname is not found just log that we skip the config
else:
self._log_header3(
gammalib.EXPLICIT,
'Skipping config "' + str(config['name']) + '"')
# Raise Exception if prodname was not found
if not has_prod:
msg = '*** ERROR: FITS production "' + self._prodname + '" not '
msg += 'available. Available productions are:\n'
for config in configs:
msg += ' - ' + config['name'] + '\n'
raise RuntimeError(msg)
# Write header
self._log_header1(gammalib.NORMAL, 'Copying files')
# Intialise counter
k = 0
# Initialise values for logging
last_fraction = 0.0
fraction_increment = 20.0
# Use 10% step increase
if self._logNormal():
fraction_increment = 10.0
# Use 5% step increase
if self._logTerse():
fraction_increment = 5.0
# Use 2% step increase
if self._logExplicit():
fraction_increment = 2.0
# Initialise logging properties
n_copied = 0
total_size = 0.0
# Loop over files and copy
for filename in files:
# Log progress
fraction = float(k) / float(len(files)) * 100.0
while fraction > last_fraction:
# Print status of copying procedure
self._log_value(gammalib.NORMAL, 'Status',
'%d %%' % int(last_fraction))
last_fraction += fraction_increment
# Copy file
filesize = self._copy(filename, self._clobber())
# If the filesize is positive then increment the number of copied
# files and add the size to the total copied filesize
if filesize > 0.0:
total_size += filesize
n_copied += 1
# Increment file counter
k += 1
# Logging
self._log_value(gammalib.NORMAL, 'Status', 'Finished')
# Logging about index files
self._log_header1(gammalib.TERSE, 'Updating index files')
# Build local hdu index file name
local_hdu = os.path.join(
self._outpath, os.path.relpath(remote_hdu, self._remote_base))
# Build local obs index file name
local_obs = os.path.join(
self._outpath, os.path.relpath(remote_obs, self._remote_base))
# If we have a runlist then merge index file
if len(self._runs):
# Logging
self._log_header3(gammalib.TERSE, 'HDU index')
# Merge remote index files with local files
self._merge(local_hdu, remote_hdu, 'HDU_INDEX', self._clobber())
# Logging
self._log_header3(gammalib.TERSE, 'OBS index')
# Merge remote index files with local files
self._merge(local_obs, remote_obs, 'OBS_INDEX', self._clobber())
else:
# If all files were copied, just copy index files too
self._copy(remote_hdu, self._clobber())
self._copy(remote_obs, self._clobber())
# Logging
self._log_header3(gammalib.TERSE, 'Master index file')
# Adding prodname to local master
localmaster = os.path.join(self._outpath, 'master.json')
# If no local master is found, copy master over first
if not os.path.isfile(localmaster):
self._copy(self._remote_master.url(), self._clobber())
# Load local master
json_data = open(localmaster).read()
data = json.loads(json_data)
configs = data['datasets']
# Initialise flag indicating if we already have prodname in master
# file
has_config = False
# Initialise new configs array
newconfigs = []
# Loop over configs in master index file
for config in configs:
# Get hdu and obs index files
hdu = os.path.join(self._outpath, config['hduindx'])
obs = os.path.join(self._outpath, config['obsindx'])
# Check if index files are available
if not (gammalib.GFilename(str(hdu)).is_fits()
and gammalib.GFilename(str(obs)).is_fits()):
self._log_value(gammalib.NORMAL,
'Removing "' + str(config['name']),
'Not available')
else:
# Append config if valid
newconfigs.append(config)
self._log_value(gammalib.NORMAL,
'Keeping "' + str(config['name']), 'Available')
# Signals that downloaded config is available
if config['name'] == self._prodname:
has_config = True
# Create new entry if config was not available
if not has_config:
newdict = dict.fromkeys(['name', 'hduindx', 'obsindx'])
newdict['name'] = self._prodname
newdict['hduindx'] = os.path.relpath(local_hdu, self._outpath)
newdict['obsindx'] = os.path.relpath(local_obs, self._outpath)
newconfigs.append(newdict)
if self._logTerse():
self._log('Adding "' + str(newdict['name']) + '"')
self._log('\n')
# Write new json master file. Make sure that we have write permission
# before writing the file. This is needed as the original master file
# may be read-only, and the shutil.copy2 function copies over the
# access permissions.
os.chmod(localmaster, 420)
f = open(localmaster, 'w')
data['datasets'] = newconfigs
json.dump(data, f, indent=2)
f.close()
# Compute total size in MB
size_MB = float(total_size) * 1.0e-6
# Log summary
self._log_header1(gammalib.NORMAL, 'Summary')
self._log_value(gammalib.NORMAL, 'Data files found', len(files))
self._log_value(gammalib.NORMAL, 'Data files copied', n_copied)
self._log_value(gammalib.NORMAL, 'Size', '%.2f MB' % size_MB)
self._log_value(gammalib.NORMAL, 'Local configs', len(newconfigs))
self._log_header3(gammalib.TERSE, 'Content of master index')
for config in newconfigs:
self._log_string(gammalib.TERSE, str(config['name']) + '\n')
# Return
return
