def run(self, args, variables):
self.__prepare(args, variables)
self.__version = self.__variables['version']
if not self.__can_run():
return False
commands = [
'cd {workspace}',
'export SCRAM_ARCH={SCRAM_ARCH}',
'source /cvmfs/cms.cern.ch/cmsset_default.sh',
'/cvmfs/cms.cern.ch/common/scram project CMSSW {cmssw_version}',
'cd {cmssw_version}/src',
'eval `/cvmfs/cms.cern.ch/common/scram runtime -sh`',
]
if self.__variables['init-git']:
commands.append('git cms-init')
all_in_one = ' && '.join(commands)
all_in_one = all_in_one.format(workspace=WORKSPACE,
SCRAM_ARCH=SCRAM_ARCH,
cmssw_version=self.__version)
from hepshell.interpreter import call
call(all_in_one, logger=LOG, shell=True)
return True
def run(self, args, variables):
from ntp import NTPROOT
self.__prepare(args, variables)
conda_path = os.path.join(NTPROOT, 'external', 'miniconda')
dps_script_path = os.path.join(conda_path, 'envs', 'ntp', 'lib',
'python2.7', 'site-packages', 'dps',
'analysis', 'xsection',
'04_make_plots_matplotlib.py')
python = os.path.join(conda_path, 'envs', 'ntp', 'bin', 'python')
params = ' '.join([
'-v {0}'.format(self.__variables['variable']),
'--visiblePS',
'-c {0}'.format(self.__variables['centre_of_mass_energy']),
'-p {0}'.format(self.__variables['json_path']),
'-o {0}'.format(DPS_PLOTDIR),
])
if self.__variables['show_generator_ratio']: # temporary solution
params += ' --show-generator-ratio'
command = '{python} {script} {params}'.format(
python=python,
script=dps_script_path,
params=params,
)
conda_activate = os.path.join(conda_path, 'bin', 'activate')
conda_env = 'source {0} ntp'.format(conda_activate)
command = '{0} && {1}'.format(conda_env, command)
from hepshell.interpreter import call
call(command, logger=LOG, shell=True)
return True
def run(self, args, variables):
self.__prepare(args, variables)
vo = self.__variables['vo']
hours = str(self.__variables['hours'])
command = ['voms-proxy-init', '-voms', vo, '-hours', hours]
from hepshell.interpreter import call
call(command, logger=LOG, shell=True, redirect=False)
return True
def __extract_tarball(self, tarball):
commands = [
'cd {CMSSW_BASE}', 'tar xzf {tarball}', 'cd {CMSSW_SRC}',
'echo "{tarball}" > .tarball_setup'
]
all_in_one = ' && '.join(commands)
all_in_one = all_in_one.format(
CMSSW_BASE=CMSSW_BASE,
CMSSW_SRC=CMSSW_SRC,
tarball=tarball,
)
from hepshell.interpreter import call
call(all_in_one, logger=LOG, shell=True)
def run(self, args, variables):
input_file = os.path.abspath(args[0])
LOG.info('Using simulation config: {0}'.format(input_file))
self._replace_output_dir(input_file)
LOG.info('Snapshot at: {0}'.format(self.__input_file))
commands = ['cd {LZ_LUXSIM_PATH}', './{LUX_SIM_EXE} {input_file}']
all_commands = ' && '.join(commands)
all_commands = all_commands.format(
LZ_LUXSIM_PATH=LZ_LUXSIM_PATH,
LUX_SIM_EXE=LUX_SIM_EXE,
input_file=input_file,
)
from hepshell.interpreter import call
code, _, _ = call(all_commands, LOG, shell=True)
if code == 0:
# now find the actual output file (has a random number in the end)
output_file_prefix = self._get_parameter_from_input_file(
'/LUXSim/io/outputName', self.__input_file)
self.__output_file = self._find_output_file(output_file_prefix)
self.__text = 'Produced file {0}'.format(self.__output_file)
return True
else:
return False
def run(self, args, variables):
self.__prepare(args, variables)
campaign = self.__variables['campaign']
dataset = self.__variables['dataset']
from hepshell.interpreter import call
crab_config = CONFIG_PATH.format(
NTPROOT=NTPROOT,
campaign=campaign,
dataset=dataset,
)
if not os.path.exists(crab_config):
LOG.error(
'Requested config file ({0}) does not exist!'.format(crab_config))
return False
LOG.info('Submitting config file {0} to the grid'.format(crab_config))
commands = [
'cd {CMSSW_SRC}',
'source /cvmfs/cms.cern.ch/cmsset_default.sh',
'eval `/cvmfs/cms.cern.ch/common/scram runtime -sh`',
'crab submit {crab_config}',
]
all_in_one = ' && '.join(commands)
all_in_one = all_in_one.format(
CMSSW_SRC=CMSSW_SRC,
crab_config=crab_config
)
code, stdout, stderr = call(all_in_one, logger=LOG)
return True
def __run_analysisSoftware(self, dataset, mode, output_file):
commands = [
'cd {CMSSW_SRC}',
'source /cvmfs/cms.cern.ch/cmsset_default.sh',
'eval `/cvmfs/cms.cern.ch/common/scram runtime -sh`',
'sample="{dataset}" analysisMode="{mode}" BAT {PSET} {params}',
]
# BAT expects parameters to start with '--'
params = self.__extract_additional_parameters(
prefix='--',
skip=['isData', 'isReHLT', 'isTTbarMC'],
)
all_in_one = ' && '.join(commands)
all_in_one = all_in_one.format(
CMSSW_SRC=CMSSW_SRC,
dataset=dataset,
mode=mode,
PSET=PSET,
params=params,
)
print(params)
LOG.info("Executing BAT")
from hepshell.interpreter import call
code, _, _ = call([all_in_one],
LOG,
stdout_log_level=logging.INFO,
shell=True)
self.__move_output_file(output_file)
return code
def __run_hadd(self, output_file, files, compression):
'''
If the option -k is used, hadd will not exit on corrupt or
non-existant input files but skip the offending files instead.
'''
commands = [
'cd {CMSSW_SRC}',
'source /cvmfs/cms.cern.ch/cmsset_default.sh',
'eval `/cvmfs/cms.cern.ch/common/scram runtime -sh`',
'hadd -k -f{compression} {output_file} {files}',
]
all_in_one = ' && '.join(commands)
all_in_one = all_in_one.format(
CMSSW_SRC=CMSSW_SRC,
files=' '.join(files),
output_file=output_file,
compression=compression,
)
LOG.info("Merging files")
from hepshell.interpreter import call
code, _, _ = call([all_in_one],
LOG,
stdout_log_level=logging.INFO,
shell=True)
return code
def run(self, args, variables):
self.__prepare(args, variables)
from hepshell.interpreter import call
proxy = get_x509_proxy()
is_valid = os.path.isfile(proxy)
time_left = 0
if is_valid:
_, stdout, stderr = call('voms-proxy-info --timeleft',
logger=LOG,
shell=True)
raw_output = stdout if not stdout == '' else stderr
try:
time_left = int(raw_output)
except:
time_left = 0
msg = 'Proxy exists ({0}) but '.format(proxy)
msg + 'could not read time left on proxy.'
LOG.warning(msg)
LOG.debug("Command output: {0}".format(
stdout.encode('string-escape')))
self.__results['timeleft_in_seconds'] = time_left
self.__results['timeleft_in_minutes'] = time_left / 60
LOG.info('Time left on proxy: {0} min'.format(
self.__results['timeleft_in_minutes']))
else:
LOG.error('Could not find proxy {0}'.format(proxy))
return is_valid
def run(self, args, variables):
if not self.__can_run():
return False
from ..setup import CMSSW_SRC
self.__prepare(args, variables)
n_jobs = int(self.__variables['ncpu'])
commands = [
'cd {CMSSW_SRC}', 'source /cvmfs/cms.cern.ch/cmsset_default.sh',
'eval `/cvmfs/cms.cern.ch/common/scram runtime -sh`',
'scram b -j{n_jobs}'
]
all_in_one = ' && '.join(commands)
all_in_one = all_in_one.format(CMSSW_SRC=CMSSW_SRC, n_jobs=n_jobs)
from hepshell.interpreter import call
call(all_in_one, logger=LOG, shell=True)
return True
def runBACCARAT(input_file):
BAC_EXE = './BACCARATExecutable'
commands = [
'cd {BACCARAT_DIR}',
'{BAC_EXE} {input_file}',
]
all_in_one = ' && '.join(commands)
all_in_one = all_in_one.format(
BACCARAT_DIR=BACCARAT_DIR, BAC_EXE=BAC_EXE, input_file=input_file)
from hepshell.interpreter import call
code, stdout, stderr = call(
[all_in_one], logger, stdout_log_level=logging.INFO, shell=True)
return code, stdout, stderr
def run(self, args, variables):
self.__prepare(args, variables)
input_file = args[0]
LOG.debug('Reading {0}'.format(input_file))
output_format = self.__variables['format']
commands = [
'cd {CMSSW_SRC}',
'source /cvmfs/cms.cern.ch/cmsset_default.sh',
'eval `/cvmfs/cms.cern.ch/common/scram runtime -sh`',
'python {CURRENT_FOLDER}/{script}.py {input_file} {args}',
]
script = 'create_table'
if output_format.lower() == 'table':
script = 'create_table'
elif output_format.lower() == 'json':
script = 'create_json'
else:
self.__text = 'Unknown format "{0}"'.format(output_format)
return False
all_in_one = ' && '.join(commands)
all_in_one = all_in_one.format(
CMSSW_SRC=CMSSW_SRC,
CURRENT_FOLDER=CURRENT_FOLDER,
input_file=input_file,
script=script,
args=self.__variables['prefix'],
)
from hepshell.interpreter import call
code, stdout, stderr = call([all_in_one],
LOG,
stdout_log_level=logging.DEBUG,
shell=True)
if not code == 0:
LOG.error('An error occured: ' + stdout + stderr)
return False
self.__text = ''
import json
output = self.__clean_stdout(stdout)
text = json.loads(output)
self.__text = '\n'.join(text)
return True
def __run_cmssw(self):
commands = [
'cd {CMSSW_SRC}',
'source /cvmfs/cms.cern.ch/cmsset_default.sh',
'eval `/cvmfs/cms.cern.ch/common/scram runtime -sh`',
'cmsRun {PSET} {params}',
]
all_in_one = ' && '.join(commands)
all_in_one = all_in_one.format(
CMSSW_SRC=CMSSW_SRC, PSET=PSET, params=self.__extract_params())
LOG.info("Executing cmsRun")
from hepshell.interpreter import call
code, _, _ = call(
[all_in_one], LOG, stdout_log_level=logging.INFO, shell=True)
return code
def run(self, args, variables):
if not self.__can_run():
return False
self.__prepare(args, variables)
commands = [
'cd {CMSSW_SRC}',
'source /cvmfs/cms.cern.ch/cmsset_default.sh',
'eval `/cvmfs/cms.cern.ch/common/scram runtime -sh`',
]
from .. import DEPENDENCIES, CMSSW_SRC, CMSSW_VERSION
from hepshell.interpreter import call
rcs = []
for dep in DEPENDENCIES:
LOG.info('Setting up dependency "{0}"'.format(dep['name']))
provider = dep['provider']
source = dep['source']
destination = dep['destination']
command = ''
if provider == 'git':
command = 'git clone {source} {destination}'.format(
source=source, destination=destination)
elif provider == 'git-cms-merge-topic':
command = 'git-cms-merge-topic {source}'.format(source=source)
elif provider == 'pip':
command = 'pip install -U --user {source}'.format(
source=source)
else:
LOG.error('Unknown provider "{0}"'.format(provider))
return False
if 'setup-cmds' in dep:
additional_commands = ' && '.join(dep['setup-cmds'])
command = command + ' && ' + additional_commands
all_in_one = ' && '.join(commands)
all_in_one = all_in_one + ' && ' + command + ' \n'
all_in_one = all_in_one.format(
CMSSW_VERSION=CMSSW_VERSION,
CMSSW_SRC=CMSSW_SRC,
)
rc = call(all_in_one, logger=LOG, shell=True)
rcs.append(rc)
return all(rcs)
def run(self, args, variables):
self.__prepare(args, variables)
input_file = args[0]
if not input_file.startswith('/store'):
input_file = os.path.abspath(input_file)
else:
if not self.__has_valid_proxy():
self.__create_proxy()
input_file = 'root://{XROOTD_SERVER}/{input_file}'.format(
XROOTD_SERVER=XROOTD_SERVER, input_file=input_file)
commands = [
'cd {CMSSW_SRC}',
'source /cvmfs/cms.cern.ch/cmsset_default.sh',
'eval `/cvmfs/cms.cern.ch/common/scram runtime -sh`',
'python {CURRENT_FOLDER}/run.py {input_file}',
]
all_in_one = ' && '.join(commands)
all_in_one = all_in_one.format(CMSSW_SRC=CMSSW_SRC,
CURRENT_FOLDER=CURRENT_FOLDER,
input_file=input_file)
from hepshell.interpreter import call
code, stdout, stderr = call([all_in_one],
LOG,
stdout_log_level=logging.DEBUG,
shell=True)
if code == 0:
stdout = stdout.replace('Welcome to rootlogon.C', '')
self.__text = 'Found {0} events in file {1}'.format(
stdout, input_file)
else:
LOG.error('An error occured: ' + stderr)
return False
return True
