def main():
output_histogram = sys.argv[1]
input_histograms_list = sys.argv[2]
print('hadd_in_cluster.py %s %s' % (output_histogram, input_histograms_list))
if (not (output_histogram and input_histograms_list)):
print("Usage: hadd_in_cluster.py /full/path/ouput_histogram.root /full/path/input_histograms_list.txt")
return False
input_histograms = []
with file(input_histograms_list) as f:
input_histograms = f.read().strip().split("\n")
print("hadd_in_cluster.py will read %s and add them into %s." % (input_histograms, output_histogram))
m = sbatchManager()
m.setWorkingDir('/tmp/') # Why SBatch manager even needs to know this?
m.hadd_in_cluster(
input_histograms=input_histograms,
final_output_histogram=output_histogram
)
m.waitForJobs()
def main():
output_histogram = sys.argv[1]
input_histograms_list = sys.argv[2]
print('hadd_in_cluster.py %s %s' %
(output_histogram, input_histograms_list))
if (not (output_histogram and input_histograms_list)):
print(
"Usage: hadd_in_cluster.py /full/path/ouput_histogram.root /full/path/input_histograms_list.txt"
)
return False
input_histograms = []
with file(input_histograms_list) as f:
input_histograms = f.read().strip().split("\n")
print("hadd_in_cluster.py will read %s and add them into %s." %
(input_histograms, output_histogram))
pool_id = uuid.uuid4()
m = sbatchManager(pool_id)
m.setWorkingDir('/tmp/') # Why SBatch manager even needs to know this?
m.hadd_in_cluster(inputFiles=input_histograms, outputFile=output_histogram)
m.waitForJobs()
def setUp(self): self.manager = sbatchManager(pool_id = self.pool_id, verbose = self.verbose) self.manager.setWorkingDir(self.workingDir) self.manager.setLogFileDir(testDir) self.manager.queue = self.queue self.manager.poll_interval = self.poll_interval self.manager.log_completion = self.log_completion
def setUp(self):
self.manager = sbatchManager(pool_id=self.pool_id,
verbose=self.verbose)
self.manager.setWorkingDir(self.workingDir)
self.manager.queue = self.queue
self.manager.poll_interval = self.poll_interval
self.manager.log_completion = self.log_completion
self.manager.max_nof_greps = self.max_nof_greps
def setUp(self):
self.manager = sbatchManager(pool_id=self.pool_id,
verbose=self.verbose,
max_num_submittedJobs=50)
self.manager.setWorkingDir(self.workingDir)
self.manager.setLogFileDir(testDir)
self.manager.queue = self.queue
self.manager.poll_interval = self.poll_interval
self.manager.log_completion = self.log_completion
self.manager.max_nof_greps = self.max_nof_greps
def call_histogram_aggregation_on_cluster_node_spec():
# Prepare
run_cmd("rm -rf %(temp_dir)s/call_histogram_aggregation_on_cluster_node" % config)
run_cmd("mkdir -p %(temp_dir)s/call_histogram_aggregation_on_cluster_node/" % config)
# Add histograms and run task
pool_id = uuid.uuid4()
m = sbatchManager(pool_id)
m.setWorkingDir('%(cmssw_base)s/src/analysis2mu1b1j/analysis2mu1b1j/test' % config)
try:
m.hadd_in_cluster(
inputFiles=[
'%(fixtures_dir)s/histogram_1.root' % config,
'%(fixtures_dir)s/histogram_2.root' % config
],
outputFile='%(temp_dir)s/call_histogram_aggregation_on_cluster_node/result.root' % config
)
m.waitForJobs()
except:
return False
# Check result
root_result_file = '%(temp_dir)s/call_histogram_aggregation_on_cluster_node/result.root' % config
root_file_exists = os.path.isfile(root_result_file)
if not root_file_exists:
print('FAILED: HADD on cluster node failed - file is missing')
return False
histogram_metadata_file = root_result_file + '.metadata'
root_file_metadata_txt = run_cmd('cat %s' % histogram_metadata_file)
expected_metadata_txt = "events_count: 3629292.0"
if root_file_metadata_txt.find(expected_metadata_txt) == -1:
print('FAILED: Metadata "%s" is not correct, should be "%s"' % (root_file_metadata_txt, expected_metadata_txt))
return False
print('PASSED: HADD on cluster node worked')
return True
def execute_command_on_cluster_node_spec():
# Prepare
run_cmd("rm -rf /%(temp_dir)s/execute_command_on_cluster_node_spec/*" %
config)
# Run task
pool_id = uuid.uuid4()
m = sbatchManager(pool_id)
m.setWorkingDir('%(cmssw_base)s/src/analysis2mu1b1j/analysis2mu1b1j/test' %
config)
m.submit_job_version2(
task_name='creating_result.txt', # BUG: Task name can't include space
command='''
export TEST_DIR=%(temp_dir)s/execute_command_on_cluster_node_spec/
mkdir -p $TEST_DIR
echo "Worked" > $TEST_DIR/result.txt
''' % config,
output_dir='%(temp_dir)s/execute_command_on_cluster_node_spec/' %
config)
# Check the result
try:
m.waitForJobs()
except:
got_exception = True
else:
got_exception = False
if got_exception:
return False
with file('%(temp_dir)s/execute_command_on_cluster_node_spec/result.txt' %
config) as f:
result = f.read().strip()
if result != 'Worked':
print(
"$TEST_DIR/ did not contain result.txt with content 'Worked'.")
print('FAILED: Execute on cluster node failed.')
return False
return True
def call_histogram_aggregation_on_cluster_node_spec():
# Prepare
run_cmd("rm -rf /home/%(user)s/tmp/call_histogram_aggregation_on_cluster_node" % config)
run_cmd("mkdir -p /home/%(user)s/tmp/call_histogram_aggregation_on_cluster_node/" % config)
# Add histograms and run task
m = sbatchManager()
m.setWorkingDir('/home/%(user)s/VHbbNtuples_7_6_x/CMSSW_7_6_3/src/analysis2mu1b1j/analysis2mu1b1j/test' % config)
m.hadd_in_cluster(
inputFiles=[
'/home/%(user)s/VHbbNtuples_7_6_x/CMSSW_7_6_3/src/tthAnalysis/HiggsToTauTau/specification/fixtures/histogram_1.root' % config,
'/home/%(user)s/VHbbNtuples_7_6_x/CMSSW_7_6_3/src/tthAnalysis/HiggsToTauTau/specification/fixtures/histogram_2.root' % config
],
outputFile='/home/%(user)s/tmp/call_histogram_aggregation_on_cluster_node/result.root' % config
)
m.waitForJobs()
# Check result
root_result_file = '/home/%(user)s/tmp/call_histogram_aggregation_on_cluster_node/result.root' % config
result_successful = os.path.isfile(root_result_file)
# Output result
if result_successful:
print('HADD on cluster node worked')
else:
print('HADD on cluster node failed')
return result_successful
def execute_command_on_cluster_node_spec():
# Prepare
run_cmd("rm -rf /home/%(user)s/tmp/execute_command_on_cluster_node_spec/*" % config)
# Run task
m = sbatchManager()
m.setWorkingDir('/home/%(user)s/VHbbNtuples_7_6_x/CMSSW_7_6_3/src/analysis2mu1b1j/analysis2mu1b1j/test' % config)
m.submit_job_version2(
task_name = 'creating_result.txt', # BUG: Task name can't include space
command = '''
export TEST_DIR=/home/%(user)s/tmp/execute_command_on_cluster_node_spec/
mkdir -p $TEST_DIR
echo "Worked" > $TEST_DIR/result.txt
''' % config,
output_dir = '/home/%(user)s/tmp/execute_command_on_cluster_node_spec/' % config
)
m.waitForJobs()
# Check the result
with file('/home/%(user)s/tmp/execute_command_on_cluster_node_spec/result.txt' % config) as f:
result = f.read().strip()
if result == 'Worked':
print('Execute on cluster node passed.')
return True
print("$TEST_DIR/ did not contain result.txt with content 'Worked'.")
print('Execute on cluster node failed.')
return False
def call_histogram_aggregation_on_cluster_node_with_invalid_input_spec():
# Prepare
run_cmd(
"rm -rf %(temp_dir)s/call_histogram_aggregation_on_cluster_node_with_invalid_input"
% config)
run_cmd(
"mkdir -p %(temp_dir)s/call_histogram_aggregation_on_cluster_node_with_invalid_input/"
% config)
# Add histograms and run task
pool_id = uuid.uuid4()
m = sbatchManager(pool_id)
m.setWorkingDir('%(cmssw_base)s/src/analysis2mu1b1j/analysis2mu1b1j/test' %
config)
try:
m.hadd_in_cluster(
inputFiles=[
'%(fixtures_dir)s/histogram_1.root' % config,
'%(fixtures_dir)s/histogram_broken.root' % config
],
outputFile=
'%(temp_dir)s/call_histogram_aggregation_on_cluster_node_with_invalid_input/result.root'
% config)
m.waitForJobs()
except sbatchManagerRuntimeError:
got_exception = True
except:
got_exception = False # Got wrong exception
else:
got_exception = False # Didn't get any exceptions, although we should've
# Check result
root_result_file = '%(temp_dir)s/call_histogram_aggregation_on_cluster_node_with_invalid_input/result.root' % config
root_file_does_not_exist = not os.path.isfile(root_result_file)
result_successful = root_file_does_not_exist and got_exception
# Output result
if result_successful:
print(
'PASSED: HADD on cluster node with invalid input was interrupted as expected'
)
else:
error_reasons = ""
if not root_file_does_not_exist:
error_reasons = 'root file should not exist; '
if not got_exception:
error_reasons = error_reasons + "didn't catch sbatchManagerRuntimeError; "
print(
'FAILED: HADD on cluster node failed silently and this is bad: ' +
error_reasons)
return result_successful
