import re
pat = re.compile(r'(?P<scope>user.[a-zA-Z]+)\..*')
m = pat.match(did)
if m:
scope = m.group('scope')
else:
import os
scope = 'user.%s' % os.path.expandvars('$USER')
try:
files = [d['name'] for d in didClient.list_files(scope, did)]
from TuningTools import GridJobFilter
ffilter = GridJobFilter()
jobFilters = ffilter(files)
mainLogger.info('Found following filters: %r', jobFilters)
jobFileCollection = select(files,
jobFilters,
popListInCaseOneItem=False)
if len(jobFileCollection) > 1:
nFilesCollection = [len(l) for l in jobFileCollection]
else:
nFilesCollection = [len(l) for l in jobFileCollection]
mainLogger.info("A total of %r files were found.", nFilesCollection)
except DataIdentifierNotFound, e:
raise RuntimeError(
"Could not retrieve number of files on informed data DID. Rucio error:\n%s"
% str(e))
except ImportError, e:
raise ImportError(
"rucio environment was not set, please set rucio and try again. Full error:\n%s"
% str(e))
scope = parsedDataDS
else:
import re
pat = re.compile(r'(?P<scope>user.[a-zA-Z]+)\..*')
m = pat.match(did)
if m:
scope = m.group('scope')
else:
scope = user_scope
try:
files = [d['name'] for d in didClient.list_files(scope, did)]
from TuningTools import GridJobFilter
ffilter = GridJobFilter()
jobFilters = ffilter( files )
mainLogger.info('Found following filters: %r', jobFilters)
jobFileCollection = select( files, jobFilters )
nFilesCollection = [len(l) for l in jobFileCollection]
mainLogger.info("A total of %r files were found.", nFilesCollection )
except DataIdentifierNotFound, e:
mainLogger.fatal("Could not retrieve number of files on informed data DID. Rucio error:\n%s" % str(e))
except ImportError, e:
mainLogger.fatal("rucio environment was not set, please set rucio and try again. Full error:\n%s" % str(e))
args.setMergeExec("""source ./setrootcore.sh --grid;
{fileMerging}
-i %IN
-o %OUT
{OUTPUT_LEVEL}
""".format(
fileMerging = r"\\\$ROOTCOREBIN/user_scripts/TuningTools/standalone/fileMerging.py" ,
OUTPUT_LEVEL = conditionalOption("--output-level", args.output_level ) if args.output_level is not LoggingLevel.INFO else '',
pprint(args.inputFiles)
## Treat special arguments
if len( args.inputFiles ) == 1:
args.inputFiles = csvStr2List( args.inputFiles[0] )
args.inputFiles = expandFolders( args.inputFiles )
mainLogger.verbose("All input files are:")
if mainLogger.isEnabledFor( LoggingLevel.VERBOSE ):
pprint(args.inputFiles)
if args.binFilters is not NotSet:
try:
args.binFilters = str_to_class( "TuningTools.CrossValidStat", args.binFilters )
args.binFilters = getFilters( args.binFilters, args.inputFiles,
printf = mainLogger.info )
except TypeError:
args.binFilters = csvStr2List( args.binFilters )
args.inputFiles = select( args.inputFiles, args.binFilters )
if len(args.binFilters) is 1:
args.inputFiles = [args.inputFiles]
else:
args.inputFiles = [args.inputFiles]
import re
searchFormat = re.compile(r'.*\.(tar.gz|tgz|pic)(\.[0-9]*)?$')
for idx, fileCollection in enumerate(args.inputFiles):
m = searchFormat.match( fileCollection[0] )
# Treat output file name:
try:
cOutputName = appendToFileName( args.outputFile, args.binFilters[idx] )
except (IndexError, TypeError,):
if len(args.inputFiles) > 1:
cOutputName = appendToFileName( args.outputFile, str(idx) )
scope = parsedDataDS
else:
import re
pat = re.compile(r'(?P<scope>user.[a-zA-Z]+)\..*')
m = pat.match(did)
if m:
scope = m.group('scope')
else:
scope = user_scope
try:
files = [d['name'] for d in didClient.list_files(scope, did)]
from TuningTools import GridJobFilter
ffilter = GridJobFilter()
jobFilters = ffilter( files )
mainLogger.info('Found following filters: %r', jobFilters)
jobFileCollection = select( files, jobFilters )
nFilesCollection = [len(l) for l in jobFileCollection]
mainLogger.info("A total of %r files were found.", nFilesCollection )
except DataIdentifierNotFound, e:
mainLogger.fatal("Could not retrieve number of files on informed data DID. Rucio error:\n%s" % str(e))
except ImportError, e:
mainLogger.fatal("rucio environment was not set, please set rucio and try again. Full error:\n%s" % str(e))
args.setMergeExec("""source ./setrootcore.sh --grid --no-color;
{fileMerging}
-i %IN
-o %OUT
{OUTPUT_LEVEL}
""".format(
fileMerging = r"\\\$ROOTCOREBIN/user_scripts/TuningTools/standalone/fileMerging.py" ,
OUTPUT_LEVEL = conditionalOption("--output-level", args.output_level ) if args.output_level is not LoggingLevel.INFO else '',
if len(args.inputFiles) == 1:
args.inputFiles = csvStr2List(args.inputFiles[0])
args.inputFiles = expandFolders(args.inputFiles)
mainLogger.verbose("All input files are:")
if mainLogger.isEnabledFor(LoggingLevel.VERBOSE):
pprint(args.inputFiles)
if args.binFilters is not NotSet:
try:
args.binFilters = str_to_class("TuningTools.CrossValidStat",
args.binFilters)
args.binFilters = getFilters(args.binFilters,
args.inputFiles,
printf=mainLogger.info)
except (TypeError, AttributeError):
args.binFilters = csvStr2List(args.binFilters)
args.inputFiles = select(args.inputFiles, args.binFilters)
if len(args.binFilters) is 1:
args.inputFiles = [args.inputFiles]
else:
args.inputFiles = [args.inputFiles]
import re
searchFormat = re.compile(r'.*\.(tar.gz|tgz|pic)(\.[0-9]*)?$')
for idx, fileCollection in enumerate(args.inputFiles):
m = searchFormat.match(fileCollection[0])
# Treat output file name:
try:
cOutputName = appendToFileName(args.outputFile, args.binFilters[idx])
except (
IndexError,
