import re pat = re.compile(r'(?P<scope>user.[a-zA-Z]+)\..*') m = pat.match(did) if m: scope = m.group('scope') else: import os scope = 'user.%s' % os.path.expandvars('$USER') try: files = [d['name'] for d in didClient.list_files(scope, did)] from TuningTools import GridJobFilter ffilter = GridJobFilter() jobFilters = ffilter(files) mainLogger.info('Found following filters: %r', jobFilters) jobFileCollection = select(files, jobFilters, popListInCaseOneItem=False) if len(jobFileCollection) > 1: nFilesCollection = [len(l) for l in jobFileCollection] else: nFilesCollection = [len(l) for l in jobFileCollection] mainLogger.info("A total of %r files were found.", nFilesCollection) except DataIdentifierNotFound, e: raise RuntimeError( "Could not retrieve number of files on informed data DID. Rucio error:\n%s" % str(e)) except ImportError, e: raise ImportError( "rucio environment was not set, please set rucio and try again. Full error:\n%s" % str(e))
scope = parsedDataDS else: import re pat = re.compile(r'(?P<scope>user.[a-zA-Z]+)\..*') m = pat.match(did) if m: scope = m.group('scope') else: scope = user_scope try: files = [d['name'] for d in didClient.list_files(scope, did)] from TuningTools import GridJobFilter ffilter = GridJobFilter() jobFilters = ffilter( files ) mainLogger.info('Found following filters: %r', jobFilters) jobFileCollection = select( files, jobFilters ) nFilesCollection = [len(l) for l in jobFileCollection] mainLogger.info("A total of %r files were found.", nFilesCollection ) except DataIdentifierNotFound, e: mainLogger.fatal("Could not retrieve number of files on informed data DID. Rucio error:\n%s" % str(e)) except ImportError, e: mainLogger.fatal("rucio environment was not set, please set rucio and try again. Full error:\n%s" % str(e)) args.setMergeExec("""source ./setrootcore.sh --grid; {fileMerging} -i %IN -o %OUT {OUTPUT_LEVEL} """.format( fileMerging = r"\\\$ROOTCOREBIN/user_scripts/TuningTools/standalone/fileMerging.py" , OUTPUT_LEVEL = conditionalOption("--output-level", args.output_level ) if args.output_level is not LoggingLevel.INFO else '',
pprint(args.inputFiles) ## Treat special arguments if len( args.inputFiles ) == 1: args.inputFiles = csvStr2List( args.inputFiles[0] ) args.inputFiles = expandFolders( args.inputFiles ) mainLogger.verbose("All input files are:") if mainLogger.isEnabledFor( LoggingLevel.VERBOSE ): pprint(args.inputFiles) if args.binFilters is not NotSet: try: args.binFilters = str_to_class( "TuningTools.CrossValidStat", args.binFilters ) args.binFilters = getFilters( args.binFilters, args.inputFiles, printf = mainLogger.info ) except TypeError: args.binFilters = csvStr2List( args.binFilters ) args.inputFiles = select( args.inputFiles, args.binFilters ) if len(args.binFilters) is 1: args.inputFiles = [args.inputFiles] else: args.inputFiles = [args.inputFiles] import re searchFormat = re.compile(r'.*\.(tar.gz|tgz|pic)(\.[0-9]*)?$') for idx, fileCollection in enumerate(args.inputFiles): m = searchFormat.match( fileCollection[0] ) # Treat output file name: try: cOutputName = appendToFileName( args.outputFile, args.binFilters[idx] ) except (IndexError, TypeError,): if len(args.inputFiles) > 1: cOutputName = appendToFileName( args.outputFile, str(idx) )
scope = parsedDataDS else: import re pat = re.compile(r'(?P<scope>user.[a-zA-Z]+)\..*') m = pat.match(did) if m: scope = m.group('scope') else: scope = user_scope try: files = [d['name'] for d in didClient.list_files(scope, did)] from TuningTools import GridJobFilter ffilter = GridJobFilter() jobFilters = ffilter( files ) mainLogger.info('Found following filters: %r', jobFilters) jobFileCollection = select( files, jobFilters ) nFilesCollection = [len(l) for l in jobFileCollection] mainLogger.info("A total of %r files were found.", nFilesCollection ) except DataIdentifierNotFound, e: mainLogger.fatal("Could not retrieve number of files on informed data DID. Rucio error:\n%s" % str(e)) except ImportError, e: mainLogger.fatal("rucio environment was not set, please set rucio and try again. Full error:\n%s" % str(e)) args.setMergeExec("""source ./setrootcore.sh --grid --no-color; {fileMerging} -i %IN -o %OUT {OUTPUT_LEVEL} """.format( fileMerging = r"\\\$ROOTCOREBIN/user_scripts/TuningTools/standalone/fileMerging.py" , OUTPUT_LEVEL = conditionalOption("--output-level", args.output_level ) if args.output_level is not LoggingLevel.INFO else '',
if len(args.inputFiles) == 1: args.inputFiles = csvStr2List(args.inputFiles[0]) args.inputFiles = expandFolders(args.inputFiles) mainLogger.verbose("All input files are:") if mainLogger.isEnabledFor(LoggingLevel.VERBOSE): pprint(args.inputFiles) if args.binFilters is not NotSet: try: args.binFilters = str_to_class("TuningTools.CrossValidStat", args.binFilters) args.binFilters = getFilters(args.binFilters, args.inputFiles, printf=mainLogger.info) except (TypeError, AttributeError): args.binFilters = csvStr2List(args.binFilters) args.inputFiles = select(args.inputFiles, args.binFilters) if len(args.binFilters) is 1: args.inputFiles = [args.inputFiles] else: args.inputFiles = [args.inputFiles] import re searchFormat = re.compile(r'.*\.(tar.gz|tgz|pic)(\.[0-9]*)?$') for idx, fileCollection in enumerate(args.inputFiles): m = searchFormat.match(fileCollection[0]) # Treat output file name: try: cOutputName = appendToFileName(args.outputFile, args.binFilters[idx]) except ( IndexError,