def run(self, args, variables): self.__prepare(args, variables) self.__version = self.__variables['version'] if not self.__can_run(): return False commands = [ 'cd {workspace}', 'export SCRAM_ARCH={SCRAM_ARCH}', 'source /cvmfs/cms.cern.ch/cmsset_default.sh', '/cvmfs/cms.cern.ch/common/scram project CMSSW {cmssw_version}', 'cd {cmssw_version}/src', 'eval `/cvmfs/cms.cern.ch/common/scram runtime -sh`', ] if self.__variables['init-git']: commands.append('git cms-init') all_in_one = ' && '.join(commands) all_in_one = all_in_one.format(workspace=WORKSPACE, SCRAM_ARCH=SCRAM_ARCH, cmssw_version=self.__version) from hepshell.interpreter import call call(all_in_one, logger=LOG, shell=True) return True
def run(self, args, variables): from ntp import NTPROOT self.__prepare(args, variables) conda_path = os.path.join(NTPROOT, 'external', 'miniconda') dps_script_path = os.path.join(conda_path, 'envs', 'ntp', 'lib', 'python2.7', 'site-packages', 'dps', 'analysis', 'xsection', '04_make_plots_matplotlib.py') python = os.path.join(conda_path, 'envs', 'ntp', 'bin', 'python') params = ' '.join([ '-v {0}'.format(self.__variables['variable']), '--visiblePS', '-c {0}'.format(self.__variables['centre_of_mass_energy']), '-p {0}'.format(self.__variables['json_path']), '-o {0}'.format(DPS_PLOTDIR), ]) if self.__variables['show_generator_ratio']: # temporary solution params += ' --show-generator-ratio' command = '{python} {script} {params}'.format( python=python, script=dps_script_path, params=params, ) conda_activate = os.path.join(conda_path, 'bin', 'activate') conda_env = 'source {0} ntp'.format(conda_activate) command = '{0} && {1}'.format(conda_env, command) from hepshell.interpreter import call call(command, logger=LOG, shell=True) return True
def run(self, args, variables): self.__prepare(args, variables) vo = self.__variables['vo'] hours = str(self.__variables['hours']) command = ['voms-proxy-init', '-voms', vo, '-hours', hours] from hepshell.interpreter import call call(command, logger=LOG, shell=True, redirect=False) return True
def __extract_tarball(self, tarball): commands = [ 'cd {CMSSW_BASE}', 'tar xzf {tarball}', 'cd {CMSSW_SRC}', 'echo "{tarball}" > .tarball_setup' ] all_in_one = ' && '.join(commands) all_in_one = all_in_one.format( CMSSW_BASE=CMSSW_BASE, CMSSW_SRC=CMSSW_SRC, tarball=tarball, ) from hepshell.interpreter import call call(all_in_one, logger=LOG, shell=True)
def run(self, args, variables): input_file = os.path.abspath(args[0]) LOG.info('Using simulation config: {0}'.format(input_file)) self._replace_output_dir(input_file) LOG.info('Snapshot at: {0}'.format(self.__input_file)) commands = ['cd {LZ_LUXSIM_PATH}', './{LUX_SIM_EXE} {input_file}'] all_commands = ' && '.join(commands) all_commands = all_commands.format( LZ_LUXSIM_PATH=LZ_LUXSIM_PATH, LUX_SIM_EXE=LUX_SIM_EXE, input_file=input_file, ) from hepshell.interpreter import call code, _, _ = call(all_commands, LOG, shell=True) if code == 0: # now find the actual output file (has a random number in the end) output_file_prefix = self._get_parameter_from_input_file( '/LUXSim/io/outputName', self.__input_file) self.__output_file = self._find_output_file(output_file_prefix) self.__text = 'Produced file {0}'.format(self.__output_file) return True else: return False
def run(self, args, variables): self.__prepare(args, variables) campaign = self.__variables['campaign'] dataset = self.__variables['dataset'] from hepshell.interpreter import call crab_config = CONFIG_PATH.format( NTPROOT=NTPROOT, campaign=campaign, dataset=dataset, ) if not os.path.exists(crab_config): LOG.error( 'Requested config file ({0}) does not exist!'.format(crab_config)) return False LOG.info('Submitting config file {0} to the grid'.format(crab_config)) commands = [ 'cd {CMSSW_SRC}', 'source /cvmfs/cms.cern.ch/cmsset_default.sh', 'eval `/cvmfs/cms.cern.ch/common/scram runtime -sh`', 'crab submit {crab_config}', ] all_in_one = ' && '.join(commands) all_in_one = all_in_one.format( CMSSW_SRC=CMSSW_SRC, crab_config=crab_config ) code, stdout, stderr = call(all_in_one, logger=LOG) return True
def __run_analysisSoftware(self, dataset, mode, output_file): commands = [ 'cd {CMSSW_SRC}', 'source /cvmfs/cms.cern.ch/cmsset_default.sh', 'eval `/cvmfs/cms.cern.ch/common/scram runtime -sh`', 'sample="{dataset}" analysisMode="{mode}" BAT {PSET} {params}', ] # BAT expects parameters to start with '--' params = self.__extract_additional_parameters( prefix='--', skip=['isData', 'isReHLT', 'isTTbarMC'], ) all_in_one = ' && '.join(commands) all_in_one = all_in_one.format( CMSSW_SRC=CMSSW_SRC, dataset=dataset, mode=mode, PSET=PSET, params=params, ) print(params) LOG.info("Executing BAT") from hepshell.interpreter import call code, _, _ = call([all_in_one], LOG, stdout_log_level=logging.INFO, shell=True) self.__move_output_file(output_file) return code
def __run_hadd(self, output_file, files, compression): ''' If the option -k is used, hadd will not exit on corrupt or non-existant input files but skip the offending files instead. ''' commands = [ 'cd {CMSSW_SRC}', 'source /cvmfs/cms.cern.ch/cmsset_default.sh', 'eval `/cvmfs/cms.cern.ch/common/scram runtime -sh`', 'hadd -k -f{compression} {output_file} {files}', ] all_in_one = ' && '.join(commands) all_in_one = all_in_one.format( CMSSW_SRC=CMSSW_SRC, files=' '.join(files), output_file=output_file, compression=compression, ) LOG.info("Merging files") from hepshell.interpreter import call code, _, _ = call([all_in_one], LOG, stdout_log_level=logging.INFO, shell=True) return code
def run(self, args, variables): self.__prepare(args, variables) from hepshell.interpreter import call proxy = get_x509_proxy() is_valid = os.path.isfile(proxy) time_left = 0 if is_valid: _, stdout, stderr = call('voms-proxy-info --timeleft', logger=LOG, shell=True) raw_output = stdout if not stdout == '' else stderr try: time_left = int(raw_output) except: time_left = 0 msg = 'Proxy exists ({0}) but '.format(proxy) msg + 'could not read time left on proxy.' LOG.warning(msg) LOG.debug("Command output: {0}".format( stdout.encode('string-escape'))) self.__results['timeleft_in_seconds'] = time_left self.__results['timeleft_in_minutes'] = time_left / 60 LOG.info('Time left on proxy: {0} min'.format( self.__results['timeleft_in_minutes'])) else: LOG.error('Could not find proxy {0}'.format(proxy)) return is_valid
def run(self, args, variables): if not self.__can_run(): return False from ..setup import CMSSW_SRC self.__prepare(args, variables) n_jobs = int(self.__variables['ncpu']) commands = [ 'cd {CMSSW_SRC}', 'source /cvmfs/cms.cern.ch/cmsset_default.sh', 'eval `/cvmfs/cms.cern.ch/common/scram runtime -sh`', 'scram b -j{n_jobs}' ] all_in_one = ' && '.join(commands) all_in_one = all_in_one.format(CMSSW_SRC=CMSSW_SRC, n_jobs=n_jobs) from hepshell.interpreter import call call(all_in_one, logger=LOG, shell=True) return True
def runBACCARAT(input_file): BAC_EXE = './BACCARATExecutable' commands = [ 'cd {BACCARAT_DIR}', '{BAC_EXE} {input_file}', ] all_in_one = ' && '.join(commands) all_in_one = all_in_one.format( BACCARAT_DIR=BACCARAT_DIR, BAC_EXE=BAC_EXE, input_file=input_file) from hepshell.interpreter import call code, stdout, stderr = call( [all_in_one], logger, stdout_log_level=logging.INFO, shell=True) return code, stdout, stderr
def run(self, args, variables): self.__prepare(args, variables) input_file = args[0] LOG.debug('Reading {0}'.format(input_file)) output_format = self.__variables['format'] commands = [ 'cd {CMSSW_SRC}', 'source /cvmfs/cms.cern.ch/cmsset_default.sh', 'eval `/cvmfs/cms.cern.ch/common/scram runtime -sh`', 'python {CURRENT_FOLDER}/{script}.py {input_file} {args}', ] script = 'create_table' if output_format.lower() == 'table': script = 'create_table' elif output_format.lower() == 'json': script = 'create_json' else: self.__text = 'Unknown format "{0}"'.format(output_format) return False all_in_one = ' && '.join(commands) all_in_one = all_in_one.format( CMSSW_SRC=CMSSW_SRC, CURRENT_FOLDER=CURRENT_FOLDER, input_file=input_file, script=script, args=self.__variables['prefix'], ) from hepshell.interpreter import call code, stdout, stderr = call([all_in_one], LOG, stdout_log_level=logging.DEBUG, shell=True) if not code == 0: LOG.error('An error occured: ' + stdout + stderr) return False self.__text = '' import json output = self.__clean_stdout(stdout) text = json.loads(output) self.__text = '\n'.join(text) return True
def __run_cmssw(self): commands = [ 'cd {CMSSW_SRC}', 'source /cvmfs/cms.cern.ch/cmsset_default.sh', 'eval `/cvmfs/cms.cern.ch/common/scram runtime -sh`', 'cmsRun {PSET} {params}', ] all_in_one = ' && '.join(commands) all_in_one = all_in_one.format( CMSSW_SRC=CMSSW_SRC, PSET=PSET, params=self.__extract_params()) LOG.info("Executing cmsRun") from hepshell.interpreter import call code, _, _ = call( [all_in_one], LOG, stdout_log_level=logging.INFO, shell=True) return code
def run(self, args, variables): if not self.__can_run(): return False self.__prepare(args, variables) commands = [ 'cd {CMSSW_SRC}', 'source /cvmfs/cms.cern.ch/cmsset_default.sh', 'eval `/cvmfs/cms.cern.ch/common/scram runtime -sh`', ] from .. import DEPENDENCIES, CMSSW_SRC, CMSSW_VERSION from hepshell.interpreter import call rcs = [] for dep in DEPENDENCIES: LOG.info('Setting up dependency "{0}"'.format(dep['name'])) provider = dep['provider'] source = dep['source'] destination = dep['destination'] command = '' if provider == 'git': command = 'git clone {source} {destination}'.format( source=source, destination=destination) elif provider == 'git-cms-merge-topic': command = 'git-cms-merge-topic {source}'.format(source=source) elif provider == 'pip': command = 'pip install -U --user {source}'.format( source=source) else: LOG.error('Unknown provider "{0}"'.format(provider)) return False if 'setup-cmds' in dep: additional_commands = ' && '.join(dep['setup-cmds']) command = command + ' && ' + additional_commands all_in_one = ' && '.join(commands) all_in_one = all_in_one + ' && ' + command + ' \n' all_in_one = all_in_one.format( CMSSW_VERSION=CMSSW_VERSION, CMSSW_SRC=CMSSW_SRC, ) rc = call(all_in_one, logger=LOG, shell=True) rcs.append(rc) return all(rcs)
def run(self, args, variables): self.__prepare(args, variables) input_file = args[0] if not input_file.startswith('/store'): input_file = os.path.abspath(input_file) else: if not self.__has_valid_proxy(): self.__create_proxy() input_file = 'root://{XROOTD_SERVER}/{input_file}'.format( XROOTD_SERVER=XROOTD_SERVER, input_file=input_file) commands = [ 'cd {CMSSW_SRC}', 'source /cvmfs/cms.cern.ch/cmsset_default.sh', 'eval `/cvmfs/cms.cern.ch/common/scram runtime -sh`', 'python {CURRENT_FOLDER}/run.py {input_file}', ] all_in_one = ' && '.join(commands) all_in_one = all_in_one.format(CMSSW_SRC=CMSSW_SRC, CURRENT_FOLDER=CURRENT_FOLDER, input_file=input_file) from hepshell.interpreter import call code, stdout, stderr = call([all_in_one], LOG, stdout_log_level=logging.DEBUG, shell=True) if code == 0: stdout = stdout.replace('Welcome to rootlogon.C', '') self.__text = 'Found {0} events in file {1}'.format( stdout, input_file) else: LOG.error('An error occured: ' + stderr) return False return True