def dasgoclient(query, **kwargs):
"""Help function to call dasgoclient and retrieve data set information."""
try:
verbosity = kwargs.get('verb', 0)
instance = kwargs.get('instance', "")
limit = kwargs.get('limit', 0)
option = kwargs.get('opts', "")
if instance:
query += " instance=%s" % (instance)
dascmd = 'dasgoclient --query="%s"' % (query)
if limit > 0:
dascmd += " --limit=%d" % (limit)
if option:
dascmd += " " + option.strip()
LOG.verb(repr(dascmd), verbosity)
cmdout = execute(dascmd, verb=verbosity - 1)
except CalledProcessError as e:
print
LOG.error(
"Failed to call 'dasgoclient' command. Please make sure:\n"
" 1) 'dasgoclient' command exists.\n"
" 2) You have a valid VOMS proxy. Use 'voms-proxy-init -voms cms -valid 200:0' or 'source utils/setupVOMS.sh'.\n"
" 3) The DAS dataset in '%s' exists!\n" % (dascmd))
raise e
return cmdout
def execute(self, cmd, **kwargs):
kwargs.setdefault('verb', self.verbosity)
return execute(cmd, **kwargs)
maxEntries=maxevts,
jsonInput=json,
modules=modules,
postfix=postfix,
noOut=False,
prefetch=prefetch)
p.run()
# REDUCE FILE SIZE
# Temporary solution to reduce file size
# https://hypernews.cern.ch/HyperNews/CMS/get/physTools/3734/1.html
# https://github.com/cms-nanoAOD/nanoAOD-tools/issues/249
print(">>> Reduce file size...")
import glob
from TauFW.common.tools.utils import execute
execute("ls -hlt %s" % (outfiles), verb=2)
for outfile in glob.glob(outfiles):
ftmp = outfile.replace(".root", "_tmp.root")
execute("haddnano.py %s %s" % (ftmp, outfile), verb=2) # reduce file size
execute("ls -hlt %s %s" % (ftmp, outfile), verb=2)
execute("mv %s %s" % (ftmp, outfile), verb=2)
execute("ls -hlt %s" % (outfiles), verb=2)
# COPY
if copydir and outdir != copydir:
from TauFW.PicoProducer.storage.utils import getstorage
from TauFW.common.tools.file import rmfile
store = getstorage(copydir, verb=2)
store.cp(outfiles)
print(">>> Removing %s..." % (outfiles))
rmfile(outfiles, verb=2)
def execute(self, cmd, dry=False, fatal=True, **kwargs):
verbosity = kwargs.get('verb', self.verbosity)
return execute(cmd, dry=dry, fatal=fatal, verb=verbosity)