def get_hess_lightcurve(filename='pks-hess-data.csv'): """ Get HESS lightcurve Parameters ---------- filename : str, optional Light curve data file name Returns ------- x, y, y_err : tuple of floats Energy, flux and flux error """ # Open data file file1 = gammalib.GFilename('$CTAGITROOT/analysis/hess_dr1/%s' % filename) file2 = gammalib.GFilename('%s' % filename) if file1.exists(): csv = gammalib.GCsv(file1,'\t') else: csv = gammalib.GCsv(file2,'\t') # Initialise arrays x = [] y = [] y_err = [] # Loop over data for row in range(csv.nrows()): x.append(float(csv[row,0])+1.0/(24.0*60.0)) y.append(float(csv[row,1])*1.0e-9) y_err.append((float(csv[row,2])-float(csv[row,1]))*1.0e-9) # Return return x,y,y_err
def __init__(self, *argv): """ Constructor """ # Set name and version self._name = 'cstsmapsplit' self._version = '1.1.0' # Set data members self._outmap = gammalib.GFilename() self._bins_per_job = 0 self._compute_null = False self._outfile = gammalib.GFilename() self._map = gammalib.GSkyMap() self._cmd = [] self._srcname = '' # Initialise observation container from constructor arguments. self._obs, argv = self._set_input_obs(argv) # Initialise application by calling the appropriate class # constructor. self._init_cscript(argv) # Return return
def _check_copy(self, pathname): """ Check copy of IACT data store Parameters ---------- pathname : str Path to copied IACT data store """ # Set file names hdu_index_name = gammalib.GFilename(pathname+'/hdu-index.fits') obs_index_name = gammalib.GFilename(pathname+'/obs-index.fits') master_name = gammalib.GFilename(pathname+'/master.json') # Check for existence of files self.test_assert(hdu_index_name.exists(), 'Check if file "hdu-index.fits" exists') self.test_assert(obs_index_name.exists(), 'Check if file "obs-index.fits" exists') self.test_assert(master_name.exists(), 'Check if file "master.json" exists') # Check if index files are FITS file self.test_assert(hdu_index_name.is_fits(), 'Check if file "hdu-index.fits" is FITS file') self.test_assert(obs_index_name.is_fits(), 'Check if file "obs-index.fits" is FITS file') # Return return
def _check_copy(self, pathname): """ Check copy. """ # Set file names hdu_index_name = gammalib.GFilename(pathname + '/hdu-index.fits') obs_index_name = gammalib.GFilename(pathname + '/obs-index.fits') master_name = gammalib.GFilename(pathname + '/master.json') # Check for existence of files self.test_assert(hdu_index_name.exists(), 'Check if file "hdu-index.fits" exists.') self.test_assert(obs_index_name.exists(), 'Check if file "obs-index.fits" exists.') self.test_assert(master_name.exists(), 'Check if file "master.json" exists.') # Check if index files are FITS file self.test_assert(hdu_index_name.is_fits(), 'Check if file "hdu-index.fits" is FITS file.') self.test_assert(obs_index_name.is_fits(), 'Check if file "obs-index.fits" is FITS file.') # Return return
def run(self): """ Run the script """ # Switch screen logging on in debug mode if self._logDebug(): self._log.cout(True) # Get parameters self._get_parameters() # Write header into logger if self._logTerse(): self._log('\n') self._log.header1('Data storage entries') self._log.parformat('Master index file') self._log(self._master_file) self._log('\n') # Open and load JSON file json_data = open(self._master_file).read() data = json.loads(json_data) configs = data['datasets'] # Initialise array for available names self._prodnames = [] # Write header into logger self._log('\n') self._log.header2('Available data configs') # Loop over configs and log available configs for config in configs: # Create hdu and obs index files hdu = os.path.join(self._datapath, config['hduindx']) obs = os.path.join(self._datapath, config['obsindx']) filename_hdu = gammalib.GFilename(str(hdu) + '[HDU_INDEX]') filename_obs = gammalib.GFilename(str(obs) + '[OBS_INDEX]') # If index files are available then log configuration # information ... if (filename_hdu.is_fits() and filename_obs.is_fits()): # Append to available names self._prodnames.append(str(config['name'])) # Log information self._log_configuration(config) # ... otherwise log that the configuration is not available else: if self._logTerse(): self._log.header3(str(config['name'])) self._log(' Not available\n') # Return return
def __init__(self, *argv): """ Constructor. """ # Set name self._name = "cstsdist" self._version = "1.1.0" # Initialise some members self._obs = gammalib.GObservations() self._pattern = "single" self._srcname = "" self._enumbins = 0 self._npix = 0 self._binsz = 0.0 self._outfile = gammalib.GFilename("ts.dat") self._ntrials = 10 self._edisp = False self._debug = False self._log_clients = False # Initialise application by calling the appropriate class # constructor. self._init_cscript(argv) # Return return
def _check_n_obs(self, pathname, n_expected): """ Check number of available observations Parameters ---------- pathname : str Path to copied IACT data store n_expected : int Expected number of observations in IACT data store """ # Set file name obs_index_name = gammalib.GFilename(pathname+'/obs-index.fits[OBS_INDEX]') # Open index file fits = gammalib.GFits(obs_index_name) # Get number of observations n_obs = fits[obs_index_name.extname()].nrows() # Close FITS file fits.close() # Check for existence of observations self.test_value(n_obs, n_expected, 'Check for number of observations') # Return return
def __init__(self, *argv): """ Constructor. """ # Set name self._name = "csmodelinfo" self._version = "1.1.0" # Initialise class members self._models = gammalib.GModels() self._ds9file = gammalib.GFilename("NONE") self._pnt_type = "" self._pnt_mark_size = 12 self._show_labels = True self._width = 2 self._fontfamily = "helvetica" self._fontsize = 12 self._fontweight = "normal" self._fontslant = "roman" self._show_ext_type = True self._free_color = "green" self._fixed_color = "magenta" # Initialise application by calling the appropriate class # constructor. self._init_cscript(argv) # Return return
def _test_cmd(self): """ Test csworkflow on the command line """ # Set script name csworkflow = self._script('csworkflow') # Remove result file gammalib.GFilename('wf_crab_results.xml').remove() # Setup csworkflow command cmd = csworkflow+' inflow="'+self._workflow+'"'+ \ ' logfile="csworkflow_cmd1.log" chatter=1' # Check if execution of wrong command fails self.test_assert(self._execute('command_that_does_not_exist') != 0, 'Self test of test script') # Check if execution was successful self.test_assert(self._execute(cmd) == 0, 'Check successful execution from command line') # Check fit result self._check_fit_result('wf_crab_results.xml') # Return return
def __init__(self, *argv): """ Constructor. """ # Set name and version self._name = 'csobs2caldb' self._version = '1.1.0' # Initialise members self._observation = gammalib.GCTAObservation() self._mission = 'cta' self._caldb = 'cta' self._outfile = gammalib.GFilename('irf_file.fits') self._base_dir = '' self._cal_dir = '' self._rsp_dir = '' self._caldb_inx = gammalib.GFits() self._irf_fits = gammalib.GFits() # Initialise observation container from constructor arguments. self._obs, argv = self._set_input_obs(argv) # Initialise script by calling the appropriate class constructor. self._init_cscript(argv) # Return return
def bin2gammalib(filename, flux_thresh, outdir): data = open(filename).readlines() # convert phasogram to numpy array phasogram = np.genfromtxt(data[14:]) # only include model if peak in lightcurve is > threshold if np.max(phasogram[:, 2]) > flux_thresh: # final name num = int(data[0].split('#')[-1]) name = 'bin{:03d}'.format(num) # spatial model # read Galactic longitude lon = float(data[1].split(' ')[-1]) # Galactic latitude is random with Gaussian distribution with 95% containment at bmax lat = np.random.normal(0, bmax / 2) src_dir = gammalib.GSkyDir() src_dir.lb_deg(lon, lat) spatial = gammalib.GModelSpatialPointSource(src_dir) # spectral model # average flux in ph/cm2/s f0 = np.average(phasogram[:, 1]) # convert to differential flux at eref in ph/cm2/s/MeV n0 = (gamma - 1) * f0 / eref spectral = gammalib.GModelSpectralPlaw( n0, -gamma, gammalib.GEnergy(np.double(eref), 'MeV')) # orbital phase model # create phasogram file fname = 'phasecurve_' + name + '.fits' phasogram_file(phasogram, outdir + fname, outdir) # period in days per = float(data[4].split(' ')[-1]) # convert to frequency in s per *= 86400 f0 = 1 / per temporal = gammalib.GModelTemporalPhaseCurve( gammalib.GFilename(fname), t0, np.random.random(), # phase at t0 randomly drawn f0, 0., 0., # orbit is stable, no derivatives 1., True) # normalized so that spectral model represents average flux # assemble final model model = gammalib.GModelSky(spatial, spectral, temporal) model.name(name) else: # empty model lat = 0. model = gammalib.GModelSky() return model, lat
def __init__(self, *argv): """ Constructor. """ # Set name and version self._name = 'csiactobs' self._version = '1.1.0' # Initialise some members self._ebounds = gammalib.GEbounds() self._datapath = os.getenv('VHEFITS', '') self._inmodels = gammalib.GModels() self._prodname = '' self._xml = gammalib.GXml() self._models = gammalib.GModels() self._runlist = [] self._runlistfile = gammalib.GFilename() self._bkgpars = 0 self._outmodel = gammalib.GFilename() self._outobs = gammalib.GFilename() self._master_indx = '' self._use_bkg_scale = False self._ev_hiera = [''] self._aeff_hiera = [''] self._psf_hiera = [''] self._bkg_hiera = [''] self._edisp_hiera = [''] self._bkg_mod_hiera = [''] self._bkg_gauss_norm = 1.0 self._bkg_gauss_index = 0.0 self._bkg_gauss_sigma = 1.0 self._bkg_aeff_index = 0.0 self._bkg_aeff_norm = 1.0 self._bkg_range_factor = 1.0 self._hdu_index = '' self._obs_index = '' self._subdir = '' self._debug = False # Initialise application by calling the appropriate class # constructor. self._init_cscript(argv) # Return return
def __init__(self, *argv): """ Constructor """ # Initialise application by calling the base class constructor self._init_cscript(self.__class__.__name__, ctools.__version__, argv) # Set name self._datapath = os.getenv('VHEFITS', '') self._remote_master = gammalib.GFilename() self._remote_base = '' self._prodname = '' self._outpath = '' self._runlist = gammalib.GFilename() self._runs = [] # Return return
def __init__(self, *argv): """ Constructor """ # Initialise application by calling the appropriate class constructor self._init_cslikelihood(self.__class__.__name__, ctools.__version__, argv) # Set data members self._outmap = gammalib.GFilename() self._bins_per_job = 0 self._compute_null = False self._outfile = gammalib.GFilename() self._map = gammalib.GSkyMap() self._cmd = [] self._srcname = '' # Return return
def make_xml(nametag): spatial = gammalib.GModelSpatialDiffuseCube( gammalib.GFilename(nametag + '_map.fits')) spectral = gammalib.GModelSpectralConst(1.) model = gammalib.GModelSky(spatial, spectral) model.name(nametag) # fill to model container and write to disk models = gammalib.GModels() models.append(model) models.save(nametag + '.xml')
def _check_cmdfile(self, filename): """ Check ASCII files containing commands """ # Create file name instance fname = gammalib.GFilename(filename) # Check if execution was successful self.test_assert(fname.exists(), 'Check of output filename exists') # Return return
def __init__(self, *argv): """ Constructor. """ # Set name self._name = 'csiactcopy' self._version = '1.1.0' self._datapath = os.getenv('VHEFITS', '') self._remote_master = gammalib.GFilename() self._remote_base = '' self._prodname = '' self._outpath = '' self._runlist = gammalib.GFilename() self._runs = [] # Initialise application by calling the appropriate class # constructor. self._init_cscript(argv) # Return return
def read_regions(fpath, init_map): """ Read a regions WCS FITS or ds9 file and return a mask WCS map. Parameters ---------- fpath : str Path to regions file init_map : `~gammalib.GSkyMap` Sky map representing fov of interest. Used for initialisation of regions map. Returns ------- map : `~gammalib.GSkyMap` Sky map. """ # Create starter map for user fov filled with zeros regions_map = init_map.copy() regions_map *= 0.0 # Take care about ds9 region files if fpath.lower().endswith('.reg'): # Read ds9 regions ds9_regions = gammalib.GSkyRegions(fpath) # Loop over regions for reg in ds9_regions: # Make map from region ds9_map = gammalib.GSkyRegionMap(reg) # Add ds9 region map to global map regions_map += ds9_map.map() # Take care about fits WCS files elif gammalib.GFilename(fpath).is_fits(): # Read wcs regions map wcs_regions = gammalib.GSkyMap(fpath) # Add wcs regions map to global map regions_map += wcs_regions else: raise RuntimeError('Invalid regions file detected. Please provide ' + 'a valid ds9 or FITS WCS regions file.') # Return return regions_map
def _test_python(self): """ Test csworkflow from Python """ # Remove result file gammalib.GFilename('wf_crab_results.xml').remove() # Set-up csworkflow workflow = cscripts.csworkflow() workflow['inflow'] = self._workflow workflow['logfile'] = 'csworkflow_py1.log' workflow['chatter'] = 2 # Run script workflow.logFileOpen() # Make sure we get a log file workflow.run() # Check fit result self._check_fit_result('wf_crab_results.xml') # Remove result file gammalib.GFilename('wf_crab_results.xml').remove() # Set-up csworkflow workflow = cscripts.csworkflow() workflow['inflow'] = self._workflow workflow['logfile'] = 'csworkflow_py2.log' workflow['chatter'] = 3 # Execute script workflow.execute() # Check fit result self._check_fit_result('wf_crab_results.xml') # Return return
def _check_response(self, caldb, irf): """ Check response """ # Open calibration database db = gammalib.GCaldb('caldb') db.open('cta', caldb) # Get filenames of response components expr = 'NAME(' + irf + ')' aeff = gammalib.GFilename(db.filename('', '', 'EFF_AREA', '', '', expr)) psf = gammalib.GFilename(db.filename('', '', 'RPSF', '', '', expr)) edisp = gammalib.GFilename(db.filename('', '', 'EDISP', '', '', expr)) bkg = gammalib.GFilename(db.filename('', '', 'BKG', '', '', expr)) # Check whether files exist self.test_assert(aeff.exists(), 'Effective area file exists') self.test_assert(psf.exists(), 'Point spread function file exists') self.test_assert(edisp.exists(), 'Energy dispersion file exists') self.test_assert(bkg.exists(), 'Background file exists') # Return return
def __init__(self, *argv): """ Constructor """ # Initialise application by calling the base class constructor self._init_cscript(self.__class__.__name__, ctools.__version__, argv) # Initialise some members self._obs = gammalib.GObservations() self._ebounds = gammalib.GEbounds() self._datapath = os.getenv('VHEFITS', '') self._prodname = '' self._xml = gammalib.GXml() self._models = gammalib.GModels() self._runlist = [] self._runlistfile = gammalib.GFilename() self._bkgpars = 0 self._master_indx = '' self._use_bkg_scale = False self._ev_hiera = [''] self._aeff_hiera = [''] self._psf_hiera = [''] self._bkg_hiera = [''] self._edisp_hiera = [''] self._bkg_mod_hiera = [''] self._bkg_gauss_norm = 1.0 self._bkg_gauss_index = 0.0 self._bkg_gauss_sigma = 1.0 self._bkg_aeff_index = 0.0 self._bkg_aeff_norm = 1.0 self._bkg_range_factor = 1.0 self._hdu_index = '' self._obs_index = '' self._subdir = '' self._debug = False # Initialise empty observation definition XML file self._xml.append( gammalib.GXmlElement('observation_list ' 'title="observation list"')) # Append an observation list to XML instance self._xml.append( gammalib.GXmlElement('observation_list title="observation list"')) self._xml_obslist = self._xml.element('observation_list', 0) # Return return
def run_csobsinfo(filename, ra, dec, debug=True): """ Run csobsinfo script Parameters ---------- filename : str File name of observation definition XML file ra : str Target Right Ascension (deg) dec : str Target declination of pointing (deg) debug : bool, optional Switch on debugging in csobsinfo run Returns ------- info : `~cscripts.csobsinfo` csobsinfo instance """ # Get observation definition XML filename obsdef = gammalib.GFilename(filename) # Setup csobsinfo script info = cscripts.csobsinfo() info['inobs'] = obsdef.url() info['debug'] = debug # Set offset if the "ra" and "dec" arguments are not empty strings if ra != '' and dec != '': info['ra'] = float(ra) info['dec'] = float(dec) info['offset'] = True else: info['offset'] = False # Run csobsinfo info.run() # Return return info
def run_csobsinfo(filename, ra=None, dec=None, debug=True): """ Run csobsinfo script Parameters ---------- filename : str File name of observation definition XML file ra : float, optional Target Right Ascension (deg) dec : float, optional Target declination of pointing (deg) debug : bool, optional Switch on debugging in csobsinfo run Returns ------- info : `~cscripts.csobsinfo` csobsinfo instance """ # Get observation definition XML filename obsdef = gammalib.GFilename(filename) # Setup csobsinfo script info = cscripts.csobsinfo() info['inobs'] = obsdef.url() info['debug'] = debug # Set offset if ra != None and dec != None: info['ra'] = ra info['dec'] = dec info['offset'] = True else: info['offset'] = False # Run csobsinfo info.run() # Return return info
def __init__(self, *argv): """ Constructor """ # Initialise application by calling the appropriate class constructor self._init_csobservation(self.__class__.__name__, ctools.__version__, argv) # Initialise members self._observation = gammalib.GCTAObservation() self._mission = 'cta' self._caldb = 'cta' self._outfile = gammalib.GFilename('irf_file.fits') self._base_dir = '' self._cal_dir = '' self._rsp_dir = '' self._caldb_inx = gammalib.GFits() self._irf_fits = gammalib.GFits() # Return return
def _is_present(self, filename, obs_id, filetype): """ Checks if a filename is present Parameters ---------- filename : str Filename obs_id : int Observation ID filetype : str Type of file Returns ------- present : bool True if filename is present, False otherwise """ # Initialise return value present = True # Create filename instance fname = gammalib.GFilename(filename) # Check if file is empty if fname.is_empty(): msg = 'Skipping observation "%s": No %s found' % (obs_id, filetype) self._log_string(gammalib.NORMAL, msg) present = False # Check if file exists elif not fname.exists(): msg = ('Skipping observation "%s": %s "%s" does not exist' % (obs_id, filetype, filename)) self._log_string(gammalib.NORMAL, msg) present = False # Return present flag return present
def __init__(self, *argv): """ Constructor. """ # Set name self._name = 'cspull' self._version = '1.1.0' # Initialise some members self._edisp = False self._exposure = None self._psfcube = None self._bckcube = None self._edispcube = None self._stackmodels = None self._coordsys = 'CEL' self._proj = 'TAN' self._log_clients = False # Static parameter self._offset = 0.0 self._ntrials = 0 self._outfile = gammalib.GFilename() self._npix = 0 self._binsz = 0.0 self._pattern = 'single' self._enumbins = 0 self._seed = 1 self._chatter = 2 # Initialise observation container from constructor arguments. self._obs, argv = self._set_input_obs(argv) # Initialise application by calling the appropriate class # constructor. self._init_cscript(argv) # Return return
def __init__(self, *argv): """ Constructor. """ # Set name self._name = "cssens" self._version = "1.1.0" # Initialise class members self._obs = gammalib.GObservations() self._ebounds = gammalib.GEbounds() self._obs_ebounds = [] self._srcname = "" self._outfile = gammalib.GFilename() self._ra = None self._dec = None self._edisp = False self._emin = 0.020 self._emax = 200.0 self._bins = 21 self._enumbins = 0 self._npix = 200 self._binsz = 0.05 self._type = "Differential" self._ts_thres = 25.0 self._max_iter = 50 self._num_avg = 3 self._log_clients = False self._debug = False # Initialise application by calling the appropriate class # constructor. self._init_cscript(argv) # Return return
def _get_parameters(self): """ Get parameters from parfile and setup the observation. """ # Get parameters if self._datapath == '': self._datapath = self['datapath'].string() # Expand environment self._datapath = gammalib.expand_env(self._datapath) # Get production name self._prodname = self['prodname'].string() # Master index file name master_indx = self['master_indx'].string() # Initialise flag if spatial selection is required self._select_radec = True # Initialise invalid radius self._radius = 0.0 # Check for validity of spatial parameters if (self['ra'].is_valid() and self['dec'].is_valid() and self['rad'].is_valid()): # Read spatial parameters self._ra = self['ra'].real() self._dec = self['dec'].real() self._radius = self['rad'].real() # ... otherwise signal that there are no spatial parameters for # selection else: self._select_radec = False # Check Radius for validity if self._radius <= 0.0: self._select_radec = False # Query other parameters self['min_qual'].integer() self['expression'].string() # Read ahead output parameters if self._read_ahead(): self['outfile'].filename() # Set filename of JSON master file and raise an exception if the file # does not exist master_file = os.path.join(self._datapath, master_indx) if not os.path.isfile(master_file): msg = ('FITS data store not available. No master index file found ' 'at "%s". Make sure the file is copied from the server and ' 'your datapath is set correctly.' % master_file) raise RuntimeError(msg) # Open and load JSON master file. If the "dataset" key is not available # then raise an exception json_data = open(master_file).read() data = json.loads(json_data) if not 'datasets' in data: msg = ('Key "datasets" not available in master index file.') raise RuntimeError(msg) # Get configurations from JSON master file configs = data['datasets'] # Initialise obs index file self._obs_index = '' # Get name of observation index file for config in configs: if self._prodname == config['name']: self._obs_index = str( os.path.join(self._datapath, config['obsindx'])) break # If the observation index file name is empty then raise an exception if self._obs_index == '': msg = ('FITS data store "%s" not available. Run csiactdata to get ' 'a list of available storage names.' % self._prodname) raise RuntimeError(msg) # If the observation index file is not a FITS file then raise an # exception filename = gammalib.GFilename(self._obs_index + '[OBS_INDEX]') if not filename.is_fits(): msg = ( 'Observation index file "%s[OBS_INDEX]" for FITS data store ' '"%s" not available. Check your master index file or run ' 'csiactdata to get a list of available storage names.' % (self._obs_index, self._prodname)) raise RuntimeError(msg) # Write input parameters into logger self._log_parameters(gammalib.TERSE) # Return return
filepath = model.spatial().filename().path() # copy file to output directory shutil.copy(filepath + filename, './') # replace file with the one in output directory model.spatial(gammalib.GModelSpatialDiffuseMap(filename)) # if model contains spatial map take care of it if model.spectral().type() == 'FileFunction': # find spectrum file name and path filename = model.spectral().filename().file() filepath = model.spectral().filename().path() # copy file to output directory shutil.copy(filepath + filename, './') # replace file with the one in output directory model.spectral( gammalib.GModelSpectralFunc( gammalib.GFilename(filename), model.spectral()['Normalization'].value())) # append model to container models.append(model) msg = 'Replaced {} gamma-cat sources with templates. Added {} sources as templates\n'.format( replaced, added) print(msg) outfile.write(msg) # re-make distributions from gammalib model container lons, lats, radii, fluxes, names = dist_from_gammalib(models) # change lon range from 0...360 to -180...180 lons = np.array(lons) lons[lons > 180] = lons[lons > 180] - 360. ax1.hist(fluxes,
def run(self): """ Run the script """ # Switch screen logging on in debug mode if self._logDebug(): self._log.cout(True) # Get parameters self._get_parameters() # Make destination directory if not available if not os.path.isdir(self._outpath): os.makedirs(self._outpath) # Set run list self._runs = self._set_runs(self._runlist) # Check for availability of remote master file if not self._remote_master.exists(): raise RuntimeError('*** ERROR: Remote master file "' + self._remote_master + '" does not exist.') # Retrieve json data from remote master json_data = open(self._remote_master.url()).read() data = json.loads(json_data) if not 'datasets' in data: raise RuntimeError('*** ERROR: Key "datasets" not available ' 'in remote master index file "' + self._remote_master + '".') # Get array of configurations configs = data['datasets'] # Get remote paths self._remote_base = self._remote_master.path() # Initialise flag if prod has been found has_prod = False # Initialise file names to be copied files = set() # Write header into logger self._log_header2(gammalib.TERSE, 'Loop over remote configs') # Loop over configs for config in configs: # Write header into logger. The str() is needed since # "config['name']" is <type 'unicode'>, and under Python 2 a # string is expected. self._log_header2(gammalib.VERBOSE, str(config['name'])) # Check if prodname was found if config['name'] == self._prodname: # Inidicate that prodname has been found has_prod = True # Build path of index files remote_hdu = str( os.path.join(self._remote_base, config['hduindx'])) remote_obs = str( os.path.join(self._remote_base, config['obsindx'])) # Log information self._log_header3(gammalib.NORMAL, 'Remote config "' + self._prodname + '"') self._log_value(gammalib.NORMAL, 'HDU index', remote_hdu) self._log_value(gammalib.NORMAL, 'Observation index', remote_obs) # Open remote HDU index file fits = gammalib.GFits(str(remote_hdu)) table = fits['HDU_INDEX'] # Initialise flag if SIZE column is present has_size = table.contains('SIZE') # Initialise file size if has_size: cp_size = 0 # Initialise remote observation IDs remote_ids = set() for row in range(table.nrows()): remote_ids.add(table['OBS_ID'][row]) # Log runs that are not available remotely for run in self._runs: # Check for run not in remote data store if not run in remote_ids: msg = ('Skip observation "%s": ID not available ' 'remotely' % str(run)) self._log_string(msg) # Loop over remote HDU index file for row in range(table.nrows()): # Get observation ID obs_id = table['OBS_ID'][row] file_dir = table['FILE_DIR'][row] file_name = table['FILE_NAME'][row] # Skip if filename is empty if file_name == '': continue # Check if we need to consider an input runlist if len(self._runs): # Check if obs id is in runlist if obs_id in self._runs: # Get filename fname = os.path.join(os.path.dirname(remote_hdu), file_dir, file_name) # Add file oldlen = len(files) files.add(fname) newlen = len(files) # Add file size if has_size and newlen > oldlen: cp_size += table['SIZE'][row] # Otherwise add every file else: # Get filename fname = os.path.join(os.path.dirname(remote_hdu), file_dir, file_name) # Add file oldlen = len(files) files.add(fname) newlen = len(files) # Add file size if has_size and newlen > oldlen: cp_size += table['SIZE'][row] # Log file information self._log_header2(gammalib.NORMAL, 'File information') self._log_value(gammalib.NORMAL, 'Number of files', len(files)) if has_size: size = float(cp_size) * 1.0e-6 self._log_value(gammalib.NORMAL, 'Size', '%.2f MB' % size) self._log_header3(gammalib.VERBOSE, 'File names') for filename in files: self._log_string(gammalib.VERBOSE, str(filename) + '\n') # Close HDU index file fits.close() # If prodname is not found just log that we skip the config else: self._log_header3( gammalib.EXPLICIT, 'Skipping config "' + str(config['name']) + '"') # Raise Exception if prodname was not found if not has_prod: msg = '*** ERROR: FITS production "' + self._prodname + '" not ' msg += 'available. Available productions are:\n' for config in configs: msg += ' - ' + config['name'] + '\n' raise RuntimeError(msg) # Write header self._log_header1(gammalib.NORMAL, 'Copying files') # Intialise counter k = 0 # Initialise values for logging last_fraction = 0.0 fraction_increment = 20.0 # Use 10% step increase if self._logNormal(): fraction_increment = 10.0 # Use 5% step increase if self._logTerse(): fraction_increment = 5.0 # Use 2% step increase if self._logExplicit(): fraction_increment = 2.0 # Initialise logging properties n_copied = 0 total_size = 0.0 # Loop over files and copy for filename in files: # Log progress fraction = float(k) / float(len(files)) * 100.0 while fraction > last_fraction: # Print status of copying procedure self._log_value(gammalib.NORMAL, 'Status', '%d %%' % int(last_fraction)) last_fraction += fraction_increment # Copy file filesize = self._copy(filename, self._clobber()) # If the filesize is positive then increment the number of copied # files and add the size to the total copied filesize if filesize > 0.0: total_size += filesize n_copied += 1 # Increment file counter k += 1 # Logging self._log_value(gammalib.NORMAL, 'Status', 'Finished') # Logging about index files self._log_header1(gammalib.TERSE, 'Updating index files') # Build local hdu index file name local_hdu = os.path.join( self._outpath, os.path.relpath(remote_hdu, self._remote_base)) # Build local obs index file name local_obs = os.path.join( self._outpath, os.path.relpath(remote_obs, self._remote_base)) # If we have a runlist then merge index file if len(self._runs): # Logging self._log_header3(gammalib.TERSE, 'HDU index') # Merge remote index files with local files self._merge(local_hdu, remote_hdu, 'HDU_INDEX', self._clobber()) # Logging self._log_header3(gammalib.TERSE, 'OBS index') # Merge remote index files with local files self._merge(local_obs, remote_obs, 'OBS_INDEX', self._clobber()) else: # If all files were copied, just copy index files too self._copy(remote_hdu, self._clobber()) self._copy(remote_obs, self._clobber()) # Logging self._log_header3(gammalib.TERSE, 'Master index file') # Adding prodname to local master localmaster = os.path.join(self._outpath, 'master.json') # If no local master is found, copy master over first if not os.path.isfile(localmaster): self._copy(self._remote_master.url(), self._clobber()) # Load local master json_data = open(localmaster).read() data = json.loads(json_data) configs = data['datasets'] # Initialise flag indicating if we already have prodname in master # file has_config = False # Initialise new configs array newconfigs = [] # Loop over configs in master index file for config in configs: # Get hdu and obs index files hdu = os.path.join(self._outpath, config['hduindx']) obs = os.path.join(self._outpath, config['obsindx']) # Check if index files are available if not (gammalib.GFilename(str(hdu)).is_fits() and gammalib.GFilename(str(obs)).is_fits()): self._log_value(gammalib.NORMAL, 'Removing "' + str(config['name']), 'Not available') else: # Append config if valid newconfigs.append(config) self._log_value(gammalib.NORMAL, 'Keeping "' + str(config['name']), 'Available') # Signals that downloaded config is available if config['name'] == self._prodname: has_config = True # Create new entry if config was not available if not has_config: newdict = dict.fromkeys(['name', 'hduindx', 'obsindx']) newdict['name'] = self._prodname newdict['hduindx'] = os.path.relpath(local_hdu, self._outpath) newdict['obsindx'] = os.path.relpath(local_obs, self._outpath) newconfigs.append(newdict) if self._logTerse(): self._log('Adding "' + str(newdict['name']) + '"') self._log('\n') # Write new json master file. Make sure that we have write permission # before writing the file. This is needed as the original master file # may be read-only, and the shutil.copy2 function copies over the # access permissions. os.chmod(localmaster, 420) f = open(localmaster, 'w') data['datasets'] = newconfigs json.dump(data, f, indent=2) f.close() # Compute total size in MB size_MB = float(total_size) * 1.0e-6 # Log summary self._log_header1(gammalib.NORMAL, 'Summary') self._log_value(gammalib.NORMAL, 'Data files found', len(files)) self._log_value(gammalib.NORMAL, 'Data files copied', n_copied) self._log_value(gammalib.NORMAL, 'Size', '%.2f MB' % size_MB) self._log_value(gammalib.NORMAL, 'Local configs', len(newconfigs)) self._log_header3(gammalib.TERSE, 'Content of master index') for config in newconfigs: self._log_string(gammalib.TERSE, str(config['name']) + '\n') # Return return