def call_analyze_slice_condor( module_name, analysis_name, tree, grl, files, ranges, process_number, directory, result_queue, error_queue, logger_queue, keep, ): analysis_framework = os.getenv('ANALYSISFRAMEWORK') analysis_home = os.getenv('ANALYSISHOME') sys.stdout = logpatch(logger_queue,'Process number {0}: '.format(process_number),'') sys.stderr = logpatch(logger_queue,'Process number {0}: '.format(process_number),'') num_processes = len(ranges) if num_processes>1: condor_dir = '{directory}/condor_{0:0>{1}}'.format(process_number,int(log(num_processes-1,10))+1,directory=directory) else: condor_dir = '{directory}/condor'.format(directory=directory) os.mkdir(condor_dir) os.chdir(condor_dir) output_name = os.path.abspath('output.root') #cleanup function always called no matter form of exit def cleanup(logger_text,error_text,output_name): #output name will be None if there is some problem if logger_text: print logger_text if error_text: output_name = None error=error_text else: error=None result_queue.put(output_name) #error will be None if there is NO problem if error: error_queue.put(error) sys.exit() files_text = 'files.text' with open(files_text,'w') as f: for file_ in files: f.write(file_+'\n') result_file_name = 'result.out' error_file_name = 'error.out' logger_file_name = 'logger.out' #setup condor files with open('{0}/condor/default_condor.submit'.format(analysis_framework)) as f: with open('condor.submit','w') as f_out: f_out.write(f.read()) with open('{0}/condor/default_condor_executable.sh'.format(analysis_framework)) as f: with open('condor_executable.sh','w') as f_out: f_out.write(f.read().format( analysis_framework = analysis_framework, analysis_home = analysis_home, module_name = module_name, analysis_name = analysis_name, tree = tree, grl = grl, files = files_text, keep = keep, start = ranges[process_number][0], end = ranges[process_number][1], output_name = output_name, process_number = process_number, error_file_name = error_file_name, logger_file_name = logger_file_name, )) #submit condor job try: print call('condor_submit condor.submit').strip() except Exception: error_text = 'Error occured in initialization\n'+traceback.format_exc() cleanup('',error_text,output_name) #attach to monitoring files error_file = None logger_file = None while(1): if not error_file and os.path.exists(error_file_name): error_file = open(error_file_name,'r+') if not logger_file and os.path.exists(logger_file_name): logger_file = open(logger_file_name,'r+') if None not in [ error_file, logger_file, ]: break if os.path.exists('done'): cleanup('','An unknown error has prevented log file creation before job finished.',output_name) sleep(1) #monitor job while not os.path.exists('done'): logger_text = logger_file.read() if logger_text: print logger_text.strip() error_text = error_file.read() if error_text: cleanup(logger_file.read(),error_text,output_name) sleep(0.5) cleanup(logger_file.read(),error_file.read(),output_name)
def call_grid( module_name, analysis_name, grid_jsons, tree='physics', grl= None, num_processes=1, keep=False, merge=False, jobsize=1, help=False, ): import os import sys import atexit import shutil from time import sleep, time import string import random from common.external import call import tarfile import json import itertools #parse all first because we change directory and finds json problems grid_datas = [] for grid_json in grid_jsons: with open(grid_json) as f: grid_datas.append(json.load(f)) analysis_home = os.getenv('ANALYSISHOME') analysis_framework = os.getenv('ANALYSISFRAMEWORK') analysis_constructor = __import__(module_name,globals(),locals(),[analysis_name]).__dict__[analysis_name] analysis_instance = analysis_constructor() if help: sys.exit(2) while True: directory = '/tmp/'+''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(10)) try: os.mkdir(directory) except OSError as error: if error.errno != 17: raise continue break print 'Created temporary directory {0}'.format(directory) cwd = os.getcwd() atexit.register(os.chdir,cwd) atexit.register(shutil.rmtree,os.path.abspath(directory)) os.chdir(directory) #create tarball of working directory print 'Creating tarball' tarball = tarfile.open('send.tar.gz','w:gz') os.chdir(analysis_framework+'/../') tarball.add(os.path.basename(analysis_framework)) os.chdir(analysis_home+'/../') tarball.add(os.path.basename(analysis_home)) tarball.close() os.chdir(directory) exclude_sites = [ 'ANALY_INFN-NAPOLI', 'ANALY_ARC', 'ANALY_INFN-FRASCATI', 'ANALY_FZK_SHORT', #'ANALY_MWT2_SL6', ] args = [] for i,(k,g) in enumerate(itertools.groupby(sys.argv,lambda x:x=='-')): if not i: continue g=list(g) args += g[:] for grid_data in grid_datas: grl = grid_data.get('GRL') grid_command = 'unset tmp; unset tmpdir; source analysis-framework/setup.sh; source {analysis_home}/setup.sh; analyze.py -m {module} -a {analysis} -i \`echo %IN | sed \'s/,/ /g\'\` -o skim.root -p {processes} -n {tree}{keep}{grl} {args}'.format( module=module_name, analysis=analysis_name, tree=tree, processes=num_processes, analysis_home=os.path.basename(analysis_home), keep=' --keep' if keep else '', grl = ' -g {0}'.format(' '.join(grl)) if grl else '', args = ' '.join(args) ) make_command = 'unset tmp; unset tmpdir; source analysis-framework/setup.sh; source {analysis_home}/setup.sh; python {analysis_home}/make_externals.py'.format( analysis_home=os.path.basename(analysis_home), ) prun_command = 'prun --bexec="{make_command}" --exec "{grid_command}" --rootVer="5.34.07" --cmtConfig="x86_64-slc5-gcc43-opt" --outputs="skim.root" --inDsTxt=input_datasets.txt --outDS={output_name} --inTarBall=send.tar.gz {jobsize}{exclude_sites}{ngb} --useContElementBoundary{merge}' for output_name,input_datasets in grid_data.get('datasets').items(): if 'steering' in grid_data: output_name = output_name.format(**grid_data['steering']) with open('input_datasets.txt','w') as f: for input_dataset in input_datasets: f.write(input_dataset+'\n') final_prun_command = prun_command.format( grid_command=grid_command, make_command=make_command, output_name=output_name, merge=' --mergeOutput' if merge else '', jobsize='--nFilesPerJob='+str(jobsize) if jobsize else '', exclude_sites=' --excludedSite="'+','.join(exclude_sites)+'"' if exclude_sites else '', ngb=' --nGBPerJob=MAX' if not jobsize else '', ) print final_prun_command print call(final_prun_command,verbose=True).strip()