def hadd(self, sources, target, **kwargs): """Hadd files. Create intermediate target file if needed.""" target = self.expandpath(target, here=True) verb = kwargs.get('verb', self.verbosity) tmpdir = kwargs.get( 'tmpdir', target.startswith(self.parent) and self.cpurl != '') htarget = target if tmpdir: if not isinstance(tmpdir, str): tmpdir = self.tmpdir tmpdir = ensuredir(tmpdir, verb=verb) htarget = os.path.join(tmpdir, os.path.basename(target)) if isinstance(sources, basestring): sources = [sources] source = "" for i, file in enumerate(sources, 1): source += self.expandpath(file, url=self.fileurl) + ' ' source = source.strip() if verb >= 2: print ">>> %-10s = %r" % ('sources', sources) print ">>> %-10s = %r" % ('source', source) print ">>> %-10s = %r" % ('target', target) print ">>> %-10s = %r" % ('htarget', htarget) out = self.execute("%s %s %s" % (self.haddcmd, htarget, source), verb=verb) if tmpdir: cpout = self.cp(htarget, target, verb=verb) rmfile(htarget) return out
def hadd(self, sources, target, **kwargs): """Hadd files. Create intermediate target file if needed.""" target = self.expandpath(target, here=True) dryrun = kwargs.get('dry', False) verb = kwargs.get('verb', self.verbosity) fileurl = kwargs.get('url', self.fileurl) maxopen = kwargs.get('maxopenfiles', 0) # maximum number of files opened via -n option tmpdir = kwargs.get( 'tmpdir', target.startswith(self.parent) and self.cpurl != '') htarget = target if tmpdir: # create temporary dir for hadd target, and copy after if not isinstance(tmpdir, str): tmpdir = self.tmpdir tmpdir = ensuredir(tmpdir, verb=verb) htarget = os.path.join(tmpdir, os.path.basename(target)) if isinstance(sources, basestring): sources = [sources] source = "" for i, file in enumerate(sources, 1): fname = os.path.basename(file) if '$PATH' in file and fileurl and isglob( fname): # expand glob pattern parent = os.path.dirname(file) files = self.getfiles(parent, filter=fname, url=fileurl) source += ' '.join(files) + ' ' else: source += self.expandpath(file, url=fileurl) + ' ' source = source.strip() if verb >= 2: print ">>> %-10s = %r" % ('sources', sources) print ">>> %-10s = %r" % ('source', source) print ">>> %-10s = %r" % ('target', target) print ">>> %-10s = %r" % ('htarget', htarget) print ">>> %-10s = %r" % ('maxopen', maxopen) haddcmd = self.haddcmd if maxopen >= 1: haddcmd += " -n %s" % (maxopen) out = self.execute("%s %s %s" % (haddcmd, htarget, source), dry=dryrun, verb=verb) if tmpdir: # copy hadd target and remove temporary file cpout = self.cp(htarget, target, dry=dryrun, verb=verb) if not dryrun: rmfile(htarget) return out
print ">>> %-12s = %s" % ('doJEC', doJEC) print ">>> %-12s = %s" % ('doJECSys', doJECSys) print ">>> %-12s = %s" % ('cwd', os.getcwd()) print '-' * 80 # RUN p = PostProcessor(outdir, infiles, cut=None, branchsel=None, outputbranchsel=branchsel, noOut=False, modules=modules, jsonInput=json, postfix=postfix, maxEntries=maxevts, prefetch=prefetch) p.run() # COPY if copydir and outdir != copydir: from TauFW.PicoProducer.storage.utils import getstorage from TauFW.common.tools.file import rmfile store = getstorage(copydir, verb=2) store.cp(outfiles) print ">>> Removing %s..." % (outfiles) rmfile(outfiles) # DONE print ">>> skimjob.py done after %.1f seconds" % (time.time() - time0)
# GET MODULE module = getmodule(modname)(outfname, **kwargs) modules.append(module) # RUN p = PostProcessor(outdir, infiles, cut=None, branchsel=None, firstEntry=firstevt, maxEntries=maxevts, jsonInput=json, modules=modules, noOut=True, prefetch=prefetch) p.run() # COPY if copydir and outdir != copydir: print(">>> %-12s = %s" % ('cwd', os.getcwd())) print(">>> %-12s = %s" % ('ls', os.listdir(outdir))) from TauFW.PicoProducer.storage.utils import getstorage from TauFW.common.tools.file import rmfile store = getstorage(copydir, verb=2) store.cp(outfname) print(">>> Removing %r..." % (outfname)) rmfile(outfname) # DONE print(">>> picojob.py done after %.1f seconds" % (time.time() - time0))
print(">>> %-12s = %s" % ('cwd', os.getcwd())) print(">>> %-12s = %s" % ('preselection', presel)) print('-' * 80) # RUN p = PostProcessor(outdir, infiles, cut=presel, branchsel=None, outputbranchsel=branchsel, firstEntry=firstevt, maxEntries=maxevts, jsonInput=json, modules=modules, postfix=postfix, noOut=False, prefetch=prefetch) p.run() # COPY if copydir and outdir != copydir: from TauFW.PicoProducer.storage.utils import getstorage from TauFW.common.tools.file import rmfile store = getstorage(copydir, verb=2) store.cp(outfiles) print(">>> Removing %s..." % (outfiles)) rmfile(outfiles, verb=2) # DONE print(">>> skimjob.py done after %.1f seconds" % (time.time() - time0))