def hadd(self, sources, target, **kwargs):
"""Hadd files. Create intermediate target file if needed."""
target = self.expandpath(target, here=True)
verb = kwargs.get('verb', self.verbosity)
tmpdir = kwargs.get(
'tmpdir',
target.startswith(self.parent) and self.cpurl != '')
htarget = target
if tmpdir:
if not isinstance(tmpdir, str):
tmpdir = self.tmpdir
tmpdir = ensuredir(tmpdir, verb=verb)
htarget = os.path.join(tmpdir, os.path.basename(target))
if isinstance(sources, basestring):
sources = [sources]
source = ""
for i, file in enumerate(sources, 1):
source += self.expandpath(file, url=self.fileurl) + ' '
source = source.strip()
if verb >= 2:
print ">>> %-10s = %r" % ('sources', sources)
print ">>> %-10s = %r" % ('source', source)
print ">>> %-10s = %r" % ('target', target)
print ">>> %-10s = %r" % ('htarget', htarget)
out = self.execute("%s %s %s" % (self.haddcmd, htarget, source),
verb=verb)
if tmpdir:
cpout = self.cp(htarget, target, verb=verb)
rmfile(htarget)
return out
def hadd(self, sources, target, **kwargs):
"""Hadd files. Create intermediate target file if needed."""
target = self.expandpath(target, here=True)
dryrun = kwargs.get('dry', False)
verb = kwargs.get('verb', self.verbosity)
fileurl = kwargs.get('url', self.fileurl)
maxopen = kwargs.get('maxopenfiles',
0) # maximum number of files opened via -n option
tmpdir = kwargs.get(
'tmpdir',
target.startswith(self.parent) and self.cpurl != '')
htarget = target
if tmpdir: # create temporary dir for hadd target, and copy after
if not isinstance(tmpdir, str):
tmpdir = self.tmpdir
tmpdir = ensuredir(tmpdir, verb=verb)
htarget = os.path.join(tmpdir, os.path.basename(target))
if isinstance(sources, basestring):
sources = [sources]
source = ""
for i, file in enumerate(sources, 1):
fname = os.path.basename(file)
if '$PATH' in file and fileurl and isglob(
fname): # expand glob pattern
parent = os.path.dirname(file)
files = self.getfiles(parent, filter=fname, url=fileurl)
source += ' '.join(files) + ' '
else:
source += self.expandpath(file, url=fileurl) + ' '
source = source.strip()
if verb >= 2:
print ">>> %-10s = %r" % ('sources', sources)
print ">>> %-10s = %r" % ('source', source)
print ">>> %-10s = %r" % ('target', target)
print ">>> %-10s = %r" % ('htarget', htarget)
print ">>> %-10s = %r" % ('maxopen', maxopen)
haddcmd = self.haddcmd
if maxopen >= 1:
haddcmd += " -n %s" % (maxopen)
out = self.execute("%s %s %s" % (haddcmd, htarget, source),
dry=dryrun,
verb=verb)
if tmpdir: # copy hadd target and remove temporary file
cpout = self.cp(htarget, target, dry=dryrun, verb=verb)
if not dryrun:
rmfile(htarget)
return out
print ">>> %-12s = %s" % ('doJEC', doJEC)
print ">>> %-12s = %s" % ('doJECSys', doJECSys)
print ">>> %-12s = %s" % ('cwd', os.getcwd())
print '-' * 80
# RUN
p = PostProcessor(outdir,
infiles,
cut=None,
branchsel=None,
outputbranchsel=branchsel,
noOut=False,
modules=modules,
jsonInput=json,
postfix=postfix,
maxEntries=maxevts,
prefetch=prefetch)
p.run()
# COPY
if copydir and outdir != copydir:
from TauFW.PicoProducer.storage.utils import getstorage
from TauFW.common.tools.file import rmfile
store = getstorage(copydir, verb=2)
store.cp(outfiles)
print ">>> Removing %s..." % (outfiles)
rmfile(outfiles)
# DONE
print ">>> skimjob.py done after %.1f seconds" % (time.time() - time0)
# GET MODULE
module = getmodule(modname)(outfname, **kwargs)
modules.append(module)
# RUN
p = PostProcessor(outdir,
infiles,
cut=None,
branchsel=None,
firstEntry=firstevt,
maxEntries=maxevts,
jsonInput=json,
modules=modules,
noOut=True,
prefetch=prefetch)
p.run()
# COPY
if copydir and outdir != copydir:
print(">>> %-12s = %s" % ('cwd', os.getcwd()))
print(">>> %-12s = %s" % ('ls', os.listdir(outdir)))
from TauFW.PicoProducer.storage.utils import getstorage
from TauFW.common.tools.file import rmfile
store = getstorage(copydir, verb=2)
store.cp(outfname)
print(">>> Removing %r..." % (outfname))
rmfile(outfname)
# DONE
print(">>> picojob.py done after %.1f seconds" % (time.time() - time0))
print(">>> %-12s = %s" % ('cwd', os.getcwd()))
print(">>> %-12s = %s" % ('preselection', presel))
print('-' * 80)
# RUN
p = PostProcessor(outdir,
infiles,
cut=presel,
branchsel=None,
outputbranchsel=branchsel,
firstEntry=firstevt,
maxEntries=maxevts,
jsonInput=json,
modules=modules,
postfix=postfix,
noOut=False,
prefetch=prefetch)
p.run()
# COPY
if copydir and outdir != copydir:
from TauFW.PicoProducer.storage.utils import getstorage
from TauFW.common.tools.file import rmfile
store = getstorage(copydir, verb=2)
store.cp(outfiles)
print(">>> Removing %s..." % (outfiles))
rmfile(outfiles, verb=2)
# DONE
print(">>> skimjob.py done after %.1f seconds" % (time.time() - time0))
