def submit(self, runList=None, runRange=None, submitCommand=None, extraCommands=None): ''' Submit a run script as a job. The submit command can be given by \ submitCommand or it will be taken from the config.yml file. Parameters ---------- runList: list List of run numbers to be simulated. runRange: list List of len 2 with the limits ofthe range for runs to be simulated. Raises ------ InvalidRunsToSimulate If runs in runList or runRange are invalid. ''' subCmd = submitCommand if submitCommand is not None else cfg.get('submissionCommand') self._logger.info('Submission command: {}'.format(subCmd)) runsToSimulate = self._getRunsToSimulate(runList, runRange) self._logger.info('Submitting run scripts for {} runs'.format(len(runsToSimulate))) self._logger.info('Starting submission') for run in runsToSimulate: runScript = self._corsikaRunner.getRunScriptFile( runNumber=run, extraCommands=extraCommands ) self._logger.info('Run {} - Submitting script {}'.format(run, runScript)) shellCommand = subCmd + ' ' + str(runScript) self._logger.debug(shellCommand) os.system(shellCommand)
def getTestOutputDirectory(): ''' Get path of a test directory, using the testDataLocation taken from the config file. Returns ------- Path ''' return Path(cfg.get('dataLocation')).joinpath('test-output')
def test_reading_db_sites(self): # This test is only relevant for the MongoDB if not cfg.get('useMongoDB'): return self.logger.info('----Testing reading La Palma parameters-----') pars = self.db.getSiteParameters('North', 'Current') if cfg.get('useMongoDB'): assert (pars['altitude']['Value'] == 2158) else: assert (pars['altitude'] == 2158) self.logger.info('Listing files written in {}'.format( self.testDataDirectory)) subprocess.call(['ls -lh {}'.format(self.testDataDirectory)], shell=True) self.logger.info('Removing the files written in {}'.format( self.testDataDirectory)) subprocess.call(['rm -f {}/*'.format(self.testDataDirectory)], shell=True) self.logger.info('----Testing reading Paranal parameters-----') pars = self.db.getSiteParameters('South', 'Current') if cfg.get('useMongoDB'): assert (pars['altitude']['Value'] == 2147) else: assert (pars['altitude'] == 2147) self.logger.info('Listing files written in {}'.format( self.testDataDirectory)) subprocess.call(['ls -lh {}'.format(self.testDataDirectory)], shell=True) self.logger.info('Removing the files written in {}'.format( self.testDataDirectory)) subprocess.call(['rm -f {}/*'.format(self.testDataDirectory)], shell=True) return
def _getExtraCommands(self, extra): ''' Get extra commands by combining the one given as argument and what is given in config.yml ''' extra = gen.copyAsList(extra) if extra is not None else list() extraFromConfig = cfg.get('extraCommands') extraFromConfig = gen.copyAsList(extraFromConfig) if extraFromConfig is not None else list() extra.extend(extraFromConfig) return extra
def getTestPlotFile(fileName): ''' Get path of a test plot file, using the testDataLocation taken from the config file. Parameters ---------- filesName: str File name Returns ------- Path ''' return Path(cfg.get('dataLocation')).joinpath('test-plots').joinpath(fileName).absolute()
def getDataFile(parentDir, fileName): ''' Get path of a data file, using the dataLocation taken from the config file. Parameters ---------- parentDir: str Parent directory of the file. filesName: str File name. Returns ------- Path ''' return Path(cfg.get('dataLocation')).joinpath(parentDir).joinpath(fileName).absolute()
def test_reading_db_lst(self): self.logger.info('----Testing reading LST-----') pars = self.db.getModelParameters('north', 'lst-1', 'Current') if cfg.get('useMongoDB'): assert (pars['parabolic_dish']['Value'] == 1) assert (pars['camera_pixels']['Value'] == 1855) else: assert (pars['parabolic_dish'] == 1) assert (pars['camera_pixels'] == 1855) self.db.exportModelFiles(pars, self.testDataDirectory) self.logger.info('Listing files written in {}'.format( self.testDataDirectory)) subprocess.call(['ls -lh {}'.format(self.testDataDirectory)], shell=True)
def test_reading_db_sst(self): self.logger.info('----Testing reading SST-----') pars = self.db.getModelParameters('south', 'sst-D', 'Current') if cfg.get('useMongoDB'): assert (pars['camera_pixels']['Value'] == 2048) else: assert (pars['camera_pixels'] == 2048) self.logger.info('Listing files written in {}'.format( self.testDataDirectory)) subprocess.call(['ls -lh {}'.format(self.testDataDirectory)], shell=True) self.logger.info('Removing the files written in {}'.format( self.testDataDirectory)) subprocess.call(['rm -f {}/*'.format(self.testDataDirectory)], shell=True)
def test_modify_db(self): # This test is only relevant for the MongoDB if not cfg.get('useMongoDB'): return self.logger.info('----Testing copying a whole telescope-----') self.db.copyTelescope(self.DB_CTA_SIMULATION_MODEL, 'North-LST-1', 'Current', 'North-LST-Test', 'sandbox') self.db.copyDocuments(self.DB_CTA_SIMULATION_MODEL, 'metadata', {'Entry': 'Simulation-Model-Tags'}, 'sandbox') pars = self.db.readMongoDB('sandbox', 'North-LST-Test', 'Current', self.testDataDirectory, False) assert (pars['camera_pixels']['Value'] == 1855) self.logger.info('----Testing adding a parameter-----') self.db.addParameter('sandbox', 'North-LST-Test', 'camera_config_version', 'test', 42) pars = self.db.readMongoDB('sandbox', 'North-LST-Test', 'test', self.testDataDirectory, False) assert (pars['camera_config_version']['Value'] == 42) self.logger.info('----Testing updating a parameter-----') self.db.updateParameter('sandbox', 'North-LST-Test', 'test', 'camera_config_version', 999) pars = self.db.readMongoDB('sandbox', 'North-LST-Test', 'test', self.testDataDirectory, False) assert (pars['camera_config_version']['Value'] == 999) self.logger.info('----Testing adding a new parameter-----') self.db.addNewParameter('sandbox', 'North-LST-Test', 'test', 'camera_config_version_test', 999) pars = self.db.readMongoDB('sandbox', 'North-LST-Test', 'test', self.testDataDirectory, False) assert (pars['camera_config_version_test']['Value'] == 999) self.logger.info( 'Testing deleting a query (a whole telescope in this case and metadata)' ) query = {'Telescope': 'North-LST-Test'} self.db.deleteQuery('sandbox', 'telescopes', query) query = {'Entry': 'Simulation-Model-Tags'} self.db.deleteQuery('sandbox', 'metadata', query)
def submit(self, inputFileList, submitCommand=None, extraCommands=None, test=False): ''' Submit a run script as a job. The submit command can be given by \ submitCommand or it will be taken from the config.yml file. Parameters ---------- inputFileList: str or list of str Single file or list of files of shower simulations. submitCommand: str Command to be used before the script name. extraCommands: str or list of str Extra commands to be added to the run script before the run command, test: bool If True, job is not submitted. ''' subCmd = submitCommand if submitCommand is not None else cfg.get( 'submissionCommand') self._logger.info('Submission command: {}'.format(subCmd)) inputFileList = self._makeInputList(inputFileList) self._logger.info('Starting submission') for file in inputFileList: run = self._guessRunFromFile(file) runScript = self._simtelRunner.getRunScript( run=run, extraCommands=extraCommands) self._logger.info('Run {} - Submitting script {}'.format( run, runScript)) shellCommand = subCmd + ' ' + str(runScript) self._logger.debug(shellCommand) if not test: os.system(shellCommand) self._fillResults(file, run)
'-v', '--verbosity', dest='logLevel', action='store', default='info', help='Log level to print (default is INFO)' ) args = parser.parse_args() label = 'trigger_rates' logger = logging.getLogger() logger.setLevel(gen.getLogLevelFromUser(args.logLevel)) # Output directory to save files related directly to this app outputDir = io.getApplicationOutputDirectory(cfg.get('outputLocation'), label) showerConfigData = { 'corsikaDataDirectory': args.output, 'site': args.site, 'layoutName': args.array, 'runRange': [1, args.nruns + 1], 'nshow': args.nevents, 'primary': args.primary, 'erange': [10 * u.GeV, 300 * u.TeV], 'eslope': -2, 'zenith': args.zenith * u.deg, 'azimuth': args.azimuth * u.deg, 'viewcone': 10 * u.deg, 'cscat': [20, 1500 * u.m, 0] }
def test_get(): modelFilesLocations = cfg.get('modelFilesLocations') logging.info(modelFilesLocations)
def test_input_options(): print('modelFilesLocations: {}'.format(cfg.get('modelFilesLocations'))) cfg.setConfigFileName('config.yml') print('cfg.CONFIG_FILE_NAME: {}'.format(cfg.CONFIG_FILE_NAME)) print('modelFilesLocations: {}'.format(cfg.get('modelFilesLocations')))
'the entries of the last 5 versions are printed.'), type=str, default='all') parser.add_argument('-V', '--verbosity', dest='logLevel', action='store', default='info', help='Log level to print (default is INFO)') args = parser.parse_args() logger = logging.getLogger() logger.setLevel(gen.getLogLevelFromUser(args.logLevel)) if not cfg.get('useMongoDB'): raise ValueError( 'This application works only with MongoDB and you asked not to use it' ) logger.info('TEST') db = db_handler.DatabaseHandler() if args.model_version == 'all': raise NotImplemented( 'Printing last 5 versions is not implemented yet.') else: version = args.model_version pars = db.getModelParameters(args.site, args.telescope, version) print()