def checkFile(job): from Analysis.Tools.helpers import checkRootFile, deepCheckRootFile filename = job if filename.startswith('root://'): if not (checkRootFile(filename, checkForObjects=["Events"]) and deepCheckRootFile(filename)): return str(filename) else: if not (os.path.exists(filename) and os.stat(filename).st_size > 0 and checkRootFile(filename, checkForObjects=["Events"]) and deepCheckRootFile(filename)): return str(filename)
def checkRootFile( file ): logger.info("Checking root file: %s"%file) from Analysis.Tools.helpers import checkRootFile, deepCheckRootFile, deepCheckWeight valid = checkRootFile( file, checkForObjects=["Events"] ) and deepCheckRootFile( file ) and deepCheckWeight( file ) if valid: logger.info("Check done!") else: logger.info("Corrupt root file: %s"%file) return valid
def checkFile(file): if args.log: logger.info("Checking filepath: %s" % file) corrupt = False if args.check: corrupt = not checkRootFile(file, checkForObjects=["Events"]) if args.deepcheck and not corrupt: corrupt = not deepCheckRootFile(file) if args.checkWeight and not corrupt: corrupt = not deepCheckWeight(file) if corrupt: if file.startswith("root://hephyse.oeaw.ac.at/"): file = file.split("root://hephyse.oeaw.ac.at/")[1] logger.info("File corrupt: %s" % file) if args.remove: logger.info("Removing file: %s" % file) os.system("/usr/bin/rfrm -f %s" % file)
if args.toCBEEos: if not args.target.startswith("/eos"): raise ValueError( "The target path needs to start with the /eos path when using --toCBEEos!" ) source = args.redirector + args.file target = args.target + args.file if args.noCheck: if os.path.exists(target): print "Found file %s. No check. Skip." % target sys.exit(0) else: from Analysis.Tools.helpers import checkRootFile, deepCheckRootFile if os.path.exists(target) and checkRootFile( target, checkForObjects=["Events"]) and deepCheckRootFile(target): print "Found file %s. Checked it. Skip." % target sys.exit(0) executable = ["echo", "xrdcp"] if args.noCopy else [which("xrdcp")] cmd = executable + [ "-p", "-f", source, clip_redirector + target if args.toCBEEos else target ] subprocess.Popen(cmd) time.sleep(float(args.sleep))
if options.small: output.reduceFiles( to = 1 ) for i,s in enumerate(masspoints[job]): #cut = "GenSusyMStop=="+str(s[0])+"&&GenSusyMNeutralino=="+str(s[1]) #FIXME logger.info("Going to write masspoint mStop %i mNeu %i", s[0], s[1]) cut = "Max$(GenPart_mass*(abs(GenPart_pdgId)==1000006))=="+str(s[0])+"&&Max$(GenPart_mass*(abs(GenPart_pdgId)==1000022))=="+str(s[1]) logger.debug("Using cut %s", cut) if options.T2tt: signal_prefix = 'T2tt_' elif options.T2bW: signal_prefix = 'T2bW_' elif options.T2bt: signal_prefix = 'T2bt_' elif options.T8bbstausnu: signal_prefix = 'T8bbstausnu_XCha%s_XStau%s_'%(x_cha,x_stau) elif options.T8bbllnunu: signal_prefix = 'T8bbllnunu_XCha%s_XSlep%s_'%(x_cha,x_slep) else: logger.info("Model isn't specified") signalFile = os.path.join(signalDir, signal_prefix + str(s[0]) + '_' + str(s[1]) + '.root' ) #signalFile = os.path.join(signalDir, 'T2tt_'+str(s[0])+'_'+str(s[1])+'.root' ) logger.debug("Ouput file will be %s", signalFile) if os.path.exists(signalFile) and deepCheckRootFile(signalFile): c = ROOT.TChain("Events") c.Add(signalFile) if c.GetEntries()==0: options.overwrite = True # :-) if not (os.path.exists(signalFile) and deepCheckRootFile(signalFile)) or options.overwrite: outF = ROOT.TFile.Open(signalFile, "RECREATE") t = output.chain.CopyTree(cut) nEvents = t.GetEntries() outF.Write() outF.Close() logger.info( "Number of events %i", nEvents) inF = ROOT.TFile.Open(signalFile, "READ") try: u = inF.Get("Events")
#!/usr/bin/env python import os, sys from Analysis.Tools.helpers import checkRootFile, deepCheckRootFile def get_parser(): ''' Argument parser for post-processing module. ''' import argparse argParser = argparse.ArgumentParser() argParser.add_argument('f') args = argParser.parse_args() if os.path.exists(args.f) and checkRootFile( args.f, checkForObjects=["Events"]) and deepCheckRootFile(args.f): print "passed:", args.f else: print "failed:", args.f
nanoAOD_list = list( set([ int(f.rstrip('.root').split('_')[-2]) for f in nanoAODFiles ])) except ValueError: #possibly 'SPLIT1' nanoAOD_list = [0] if len(allRootFiles) > 0: rootFiles = [ os.path.join(dirPath, filename) for filename in allRootFiles ] if args.check != 'None': if args.check == 'deep': check_f = lambda f: checkRootFile( prefix + f, ["Events"]) and deepCheckRootFile(prefix + f) elif args.check == 'normal': check_f = lambda f: checkRootFile(prefix + f, ["Events"]) len_before = len(rootFiles) rootFiles = filter(check_f, rootFiles) if len_before > len(rootFiles): logger.warning("Path: %s : %i/%i files failed %s check", dirPath, len_before - len(rootFiles), len_before, args.check) elif len_before == len(rootFiles): logger.debug("Sample %s: All %i files passed %s check", sample, len(rootFiles), args.check) else: rootFiles = [] logger.warning("Path does not exist or no files found: %s", dirPath)
if not os.path.exists( output_directory ): try: os.makedirs( output_directory ) logger.info( "Created output directory %s.", output_directory ) except: logger.info( "Directory %s already exists.", output_directory ) pass # checking overwrite or file exists sel = "&&".join(skimConds) nEvents = sample.getYieldFromDraw(weightString="1", selectionString=sel)['val'] if not options.overwrite: if os.path.isfile(targetFilePath): logger.info( "Output file %s found.", targetFilePath) if checkRootFile( targetFilePath, checkForObjects=["Events"] ) and deepCheckRootFile( targetFilePath ) and deepCheckWeight( targetFilePath ): logger.info( "File already processed. Source: File check ok!" ) # Everything is fine, no overwriting logger.info( "Checking the normalization of the sample." ) existingSample = Sample.fromFiles( "existing", targetFilePath, treeName = "Events" ) nEventsExist = existingSample.getYieldFromDraw(weightString="1")['val'] if nEvents == nEventsExist: logger.info( "File already processed. Normalization file check ok! Skipping." ) # Everything is fine, no overwriting sys.exit(0) else: logger.info( "Target events not equal to processing sample events! Is: %s, should be: %s!"%(nEventsExist, nEvents) ) logger.info( "Removing file from target." ) os.remove( targetFilePath ) logger.info( "Reprocessing." ) else: logger.info( "File corrupt. Removing file from target." ) os.remove( targetFilePath )
''' import argparse argParser = argparse.ArgumentParser(description = "Argument parser for nanoPostProcessing") argParser.add_argument('--check', action='store_true', help="check root files?") argParser.add_argument('--deepcheck', action='store_true', help="check events of root files?") argParser.add_argument('--remove', action='store_true', help="remove corrupt root files?") argParser.add_argument('--log', action='store_true', help="print each filename?") return argParser args = get_parser().parse_args() if not (args.check or args.deepcheck): sys.exit(0) # check Root Files from Analysis.Tools.helpers import checkRootFile, deepCheckRootFile for file in Run2017.files: if args.log: logger.info( "Checking filepath: %s"%file ) corrupt = False if args.check: corrupt = not checkRootFile(file, checkForObjects=["Events"]) if args.deepcheck and not corrupt: corrupt = not deepCheckRootFile(file) if corrupt: if file.startswith("root://hephyse.oeaw.ac.at/"): file = file.split("root://hephyse.oeaw.ac.at/")[1] logger.info( "File corrupt: %s"%file ) if args.remove: logger.info( "Removing file: %s"%file ) os.system( "/usr/bin/rfrm -f %s"%file )
shutil.rmtree(tmp_output_directory) try: #Avoid trouble with race conditions in multithreading os.makedirs(tmp_output_directory) logger.info( "Created output directory %s.", tmp_output_directory ) except: pass target_outfilename = os.path.join(storage_directory, sample.name + '.root') filename, ext = os.path.splitext( os.path.join(tmp_output_directory, sample.name + '.root') ) outfilename = filename+ext if not options.overwrite: if os.path.isfile(target_outfilename): logger.info( "Output file %s found.", target_outfilename) if checkRootFile( target_outfilename, checkForObjects=["Events"] ) and deepCheckRootFile( target_outfilename ) and deepCheckWeight( target_outfilename ): logger.info( "File already processed. Source: File check ok! Skipping." ) # Everything is fine, no overwriting sys.exit(0) else: logger.info( "File corrupt. Removing file from target." ) os.remove( target_outfilename ) logger.info( "Reprocessing." ) else: logger.info( "Sample not processed yet." ) logger.info( "Processing." ) else: logger.info( "Overwriting.") # relocate original sample.copy_files( os.path.join(tmp_output_directory, "input") )
'weight': w } for w in LHE_weights[1:10]]) tmp_dir = ROOT.gDirectory output_filename = os.path.join(output_directory, sample.name + '.root') _logger.add_fileHandler(output_filename.replace('.root', '.log'), args.logLevel) _logger_rt.add_fileHandler(output_filename.replace('.root', '_rt.log'), args.logLevel) if os.path.exists(output_filename) and checkRootFile( output_filename, checkForObjects=[ "Events" ]) and deepCheckRootFile(output_filename) and not args.overwrite: logger.info("File %s found. Quit.", output_filename) sys.exit(0) # FWLite reader if this is an EDM file reader = sample.fwliteReader(products=products) output_file = ROOT.TFile(output_filename, 'recreate') output_file.cd() maker = TreeMaker(sequence=[filler], variables=[TreeVariable.fromString(x) for x in variables], treeName="Events") tmp_dir.cd() counter = 0