def main(): output_histogram = sys.argv[1] input_histograms_list = sys.argv[2] print('hadd_in_cluster.py %s %s' % (output_histogram, input_histograms_list)) if (not (output_histogram and input_histograms_list)): print("Usage: hadd_in_cluster.py /full/path/ouput_histogram.root /full/path/input_histograms_list.txt") return False input_histograms = [] with file(input_histograms_list) as f: input_histograms = f.read().strip().split("\n") print("hadd_in_cluster.py will read %s and add them into %s." % (input_histograms, output_histogram)) m = sbatchManager() m.setWorkingDir('/tmp/') # Why SBatch manager even needs to know this? m.hadd_in_cluster( input_histograms=input_histograms, final_output_histogram=output_histogram ) m.waitForJobs()
def main(): output_histogram = sys.argv[1] input_histograms_list = sys.argv[2] print('hadd_in_cluster.py %s %s' % (output_histogram, input_histograms_list)) if (not (output_histogram and input_histograms_list)): print( "Usage: hadd_in_cluster.py /full/path/ouput_histogram.root /full/path/input_histograms_list.txt" ) return False input_histograms = [] with file(input_histograms_list) as f: input_histograms = f.read().strip().split("\n") print("hadd_in_cluster.py will read %s and add them into %s." % (input_histograms, output_histogram)) pool_id = uuid.uuid4() m = sbatchManager(pool_id) m.setWorkingDir('/tmp/') # Why SBatch manager even needs to know this? m.hadd_in_cluster(inputFiles=input_histograms, outputFile=output_histogram) m.waitForJobs()
def setUp(self): self.manager = sbatchManager(pool_id = self.pool_id, verbose = self.verbose) self.manager.setWorkingDir(self.workingDir) self.manager.setLogFileDir(testDir) self.manager.queue = self.queue self.manager.poll_interval = self.poll_interval self.manager.log_completion = self.log_completion
def setUp(self): self.manager = sbatchManager(pool_id=self.pool_id, verbose=self.verbose) self.manager.setWorkingDir(self.workingDir) self.manager.queue = self.queue self.manager.poll_interval = self.poll_interval self.manager.log_completion = self.log_completion self.manager.max_nof_greps = self.max_nof_greps
def setUp(self): self.manager = sbatchManager(pool_id=self.pool_id, verbose=self.verbose, max_num_submittedJobs=50) self.manager.setWorkingDir(self.workingDir) self.manager.setLogFileDir(testDir) self.manager.queue = self.queue self.manager.poll_interval = self.poll_interval self.manager.log_completion = self.log_completion self.manager.max_nof_greps = self.max_nof_greps
def call_histogram_aggregation_on_cluster_node_spec(): # Prepare run_cmd("rm -rf %(temp_dir)s/call_histogram_aggregation_on_cluster_node" % config) run_cmd("mkdir -p %(temp_dir)s/call_histogram_aggregation_on_cluster_node/" % config) # Add histograms and run task pool_id = uuid.uuid4() m = sbatchManager(pool_id) m.setWorkingDir('%(cmssw_base)s/src/analysis2mu1b1j/analysis2mu1b1j/test' % config) try: m.hadd_in_cluster( inputFiles=[ '%(fixtures_dir)s/histogram_1.root' % config, '%(fixtures_dir)s/histogram_2.root' % config ], outputFile='%(temp_dir)s/call_histogram_aggregation_on_cluster_node/result.root' % config ) m.waitForJobs() except: return False # Check result root_result_file = '%(temp_dir)s/call_histogram_aggregation_on_cluster_node/result.root' % config root_file_exists = os.path.isfile(root_result_file) if not root_file_exists: print('FAILED: HADD on cluster node failed - file is missing') return False histogram_metadata_file = root_result_file + '.metadata' root_file_metadata_txt = run_cmd('cat %s' % histogram_metadata_file) expected_metadata_txt = "events_count: 3629292.0" if root_file_metadata_txt.find(expected_metadata_txt) == -1: print('FAILED: Metadata "%s" is not correct, should be "%s"' % (root_file_metadata_txt, expected_metadata_txt)) return False print('PASSED: HADD on cluster node worked') return True
def execute_command_on_cluster_node_spec(): # Prepare run_cmd("rm -rf /%(temp_dir)s/execute_command_on_cluster_node_spec/*" % config) # Run task pool_id = uuid.uuid4() m = sbatchManager(pool_id) m.setWorkingDir('%(cmssw_base)s/src/analysis2mu1b1j/analysis2mu1b1j/test' % config) m.submit_job_version2( task_name='creating_result.txt', # BUG: Task name can't include space command=''' export TEST_DIR=%(temp_dir)s/execute_command_on_cluster_node_spec/ mkdir -p $TEST_DIR echo "Worked" > $TEST_DIR/result.txt ''' % config, output_dir='%(temp_dir)s/execute_command_on_cluster_node_spec/' % config) # Check the result try: m.waitForJobs() except: got_exception = True else: got_exception = False if got_exception: return False with file('%(temp_dir)s/execute_command_on_cluster_node_spec/result.txt' % config) as f: result = f.read().strip() if result != 'Worked': print( "$TEST_DIR/ did not contain result.txt with content 'Worked'.") print('FAILED: Execute on cluster node failed.') return False return True
def call_histogram_aggregation_on_cluster_node_spec(): # Prepare run_cmd("rm -rf /home/%(user)s/tmp/call_histogram_aggregation_on_cluster_node" % config) run_cmd("mkdir -p /home/%(user)s/tmp/call_histogram_aggregation_on_cluster_node/" % config) # Add histograms and run task m = sbatchManager() m.setWorkingDir('/home/%(user)s/VHbbNtuples_7_6_x/CMSSW_7_6_3/src/analysis2mu1b1j/analysis2mu1b1j/test' % config) m.hadd_in_cluster( inputFiles=[ '/home/%(user)s/VHbbNtuples_7_6_x/CMSSW_7_6_3/src/tthAnalysis/HiggsToTauTau/specification/fixtures/histogram_1.root' % config, '/home/%(user)s/VHbbNtuples_7_6_x/CMSSW_7_6_3/src/tthAnalysis/HiggsToTauTau/specification/fixtures/histogram_2.root' % config ], outputFile='/home/%(user)s/tmp/call_histogram_aggregation_on_cluster_node/result.root' % config ) m.waitForJobs() # Check result root_result_file = '/home/%(user)s/tmp/call_histogram_aggregation_on_cluster_node/result.root' % config result_successful = os.path.isfile(root_result_file) # Output result if result_successful: print('HADD on cluster node worked') else: print('HADD on cluster node failed') return result_successful
def execute_command_on_cluster_node_spec(): # Prepare run_cmd("rm -rf /home/%(user)s/tmp/execute_command_on_cluster_node_spec/*" % config) # Run task m = sbatchManager() m.setWorkingDir('/home/%(user)s/VHbbNtuples_7_6_x/CMSSW_7_6_3/src/analysis2mu1b1j/analysis2mu1b1j/test' % config) m.submit_job_version2( task_name = 'creating_result.txt', # BUG: Task name can't include space command = ''' export TEST_DIR=/home/%(user)s/tmp/execute_command_on_cluster_node_spec/ mkdir -p $TEST_DIR echo "Worked" > $TEST_DIR/result.txt ''' % config, output_dir = '/home/%(user)s/tmp/execute_command_on_cluster_node_spec/' % config ) m.waitForJobs() # Check the result with file('/home/%(user)s/tmp/execute_command_on_cluster_node_spec/result.txt' % config) as f: result = f.read().strip() if result == 'Worked': print('Execute on cluster node passed.') return True print("$TEST_DIR/ did not contain result.txt with content 'Worked'.") print('Execute on cluster node failed.') return False
def call_histogram_aggregation_on_cluster_node_with_invalid_input_spec(): # Prepare run_cmd( "rm -rf %(temp_dir)s/call_histogram_aggregation_on_cluster_node_with_invalid_input" % config) run_cmd( "mkdir -p %(temp_dir)s/call_histogram_aggregation_on_cluster_node_with_invalid_input/" % config) # Add histograms and run task pool_id = uuid.uuid4() m = sbatchManager(pool_id) m.setWorkingDir('%(cmssw_base)s/src/analysis2mu1b1j/analysis2mu1b1j/test' % config) try: m.hadd_in_cluster( inputFiles=[ '%(fixtures_dir)s/histogram_1.root' % config, '%(fixtures_dir)s/histogram_broken.root' % config ], outputFile= '%(temp_dir)s/call_histogram_aggregation_on_cluster_node_with_invalid_input/result.root' % config) m.waitForJobs() except sbatchManagerRuntimeError: got_exception = True except: got_exception = False # Got wrong exception else: got_exception = False # Didn't get any exceptions, although we should've # Check result root_result_file = '%(temp_dir)s/call_histogram_aggregation_on_cluster_node_with_invalid_input/result.root' % config root_file_does_not_exist = not os.path.isfile(root_result_file) result_successful = root_file_does_not_exist and got_exception # Output result if result_successful: print( 'PASSED: HADD on cluster node with invalid input was interrupted as expected' ) else: error_reasons = "" if not root_file_does_not_exist: error_reasons = 'root file should not exist; ' if not got_exception: error_reasons = error_reasons + "didn't catch sbatchManagerRuntimeError; " print( 'FAILED: HADD on cluster node failed silently and this is bad: ' + error_reasons) return result_successful