def filterPaths(paths, grid=False): oDict = dict() import re from RingerCore import checkExtension if grid is True: #pat = re.compile(r'.*user.[a-zA-Z0-9]+.(?P<jobID>[0-9]+)\..*$') pat = re.compile( r'.*crossValStat_(?P<jobID>[0-9]+)(_monitoring)?\..*$') jobIDs = sorted( list( set([ pat.match(f).group('jobID') for f in paths if pat.match(f) is not None ]))) for jobID in jobIDs: oDict[jobID] = dict() for xname in paths: if jobID in xname and checkExtension(xname, '.root'): oDict[jobID]['root'] = xname if jobID in xname and checkExtension(xname, '.pic|.pic.gz|.pic.tgz'): oDict[jobID]['pic'] = xname else: #pat = re.compile(r'.*crossValStat_(?P<jobID>[0-9]+)(_monitoring)?\..*$') pat = re.compile( r'.+(?P<binID>et(?P<etBinIdx>\d+).eta(?P<etaBinIdx>\d+))\..+$') # jobIDs = sorted(list(set([pat.match(f).group('jobID') for f in paths if pat.match(f) is not None]))) jobIDs = sorted( list( set([ pat.match(f).group('binID') for f in paths if pat.match(f) is not None ]))) if not len(jobIDs): oDict['unique'] = {'root': '', 'pic': ''} for xname in paths: if xname.endswith('.root'): oDict['unique']['root'] = xname if '.pic' in xname: oDict['unique']['pic'] = xname else: for jobID in jobIDs: oDict[jobID] = dict() for xname in paths: if jobID in xname and checkExtension(xname, '.root'): oDict[jobID]['root'] = xname if jobID in xname and checkExtension( xname, '.pic|.pic.gz|.pic.tgz'): oDict[jobID]['pic'] = xname return oDict
def filterPaths(paths, grid=False): oDict = dict() if grid is True: import re from RingerCore import checkExtension pat = re.compile(r'.*user.[a-zA-Z0-9]+.(?P<jobID>[0-9]+)\..*$') jobIDs = sorted(list(set([pat.match(f).group('jobID') for f in paths if pat.match(f) is not None]))) for jobID in jobIDs: oDict[jobID] = dict() for xname in paths: if jobID in xname and checkExtension( xname, '.root'): oDict[jobID]['root'] = xname if jobID in xname and checkExtension( xname, '.pic|.pic.gz'): oDict[jobID]['pic'] = xname else: oDict['unique'] = {'root':'','pic':''} for xname in paths: if xname.endswith('.root'): oDict['unique']['root'] = xname if '.pic' in xname: oDict['unique']['pic'] = xname return oDict
pprint(args.inputFiles) ## Treat special arguments if len(args.inputFiles) == 1: args.inputFiles = csvStr2List(args.inputFiles[0]) args.inputFiles = expandFolders(args.inputFiles) mainLogger.verbose("All input files are:") if mainLogger.isEnabledFor(LoggingLevel.VERBOSE): pprint(args.inputFiles) for inFile in progressbar(args.inputFiles, len(args.inputFiles), logger=mainLogger, prefix="Processing files "): # Treat output file name: from RingerCore import checkExtension, changeExtension, load, save if checkExtension(inFile, "tgz|tar.gz|pic"): cOutputName = changeExtension(inFile, '.mat') if args.change_output_folder: import os.path cOutputName = os.path.join( os.path.abspath(args.change_output_folder), os.path.basename(cOutputName)) data = load(inFile, useHighLevelObj=False) from scipy.io import savemat try: savemat(cOutputName, data) except ImportError: self._logger.fatal( ("Cannot save matlab file, it seems that scipy is not " "available."), ImportError) mainLogger.info("Successfully created matlab file: %s", cOutputName)
## Treat special arguments if len(args.inputFiles) == 1: args.inputFiles = csvStr2List(args.inputFiles[0]) args.inputFiles = expandFolders(args.inputFiles) mainLogger.verbose("All input files are:") if mainLogger.isEnabledFor(LoggingLevel.VERBOSE): pprint(args.inputFiles) for inFile in progressbar(args.inputFiles, len(args.inputFiles), logger=mainLogger, prefix="Processing files "): # Treat output file name: from RingerCore import checkExtension, changeExtension, save if checkExtension(inFile, "root"): cOutputName = changeExtension(inFile, '.npz') if args.change_output_folder: import os.path cOutputName = os.path.join( os.path.abspath(args.change_output_folder), os.path.basename(cOutputName)) f = ROOT.TFile(inFile, 'r') data = {} for keyName, obj in getall(f): mainLogger.debug("Reading key: %s", keyName) shortKey = keyName.split('/')[-1] if not issubclass(type(obj), ROOT.TH1): mainLogger.verbose("Ignoring key: %s", shortKey) continue hist = obj
if mainLogger.isEnabledFor( LoggingLevel.DEBUG ): from pprint import pprint pprint(args.inputFiles) ## Treat special arguments if len( args.inputFiles ) == 1: args.inputFiles = csvStr2List( args.inputFiles[0] ) args.inputFiles = expandFolders( args.inputFiles ) mainLogger.verbose("All input files are:") if mainLogger.isEnabledFor( LoggingLevel.VERBOSE ): pprint(args.inputFiles) for inFile in progressbar(args.inputFiles, len(args.inputFiles), logger = mainLogger, prefix = "Processing files "): # Treat output file name: from RingerCore import checkExtension, changeExtension, load, save if checkExtension( inFile, "tgz|tar.gz|pic" ): cOutputName = changeExtension( inFile, '.mat' ) if args.change_output_folder: import os.path cOutputName = os.path.join( os.path.abspath(args.change_output_folder) , os.path.basename(cOutputName) ) data = load( inFile, useHighLevelObj = False ) from scipy.io import savemat try: savemat( cOutputName, data ) except ImportError: self._logger.fatal(("Cannot save matlab file, it seems that scipy is not " "available."), ImportError) mainLogger.info("Successfully created matlab file: %s", cOutputName) else: mainLogger.error("Cannot transform files '%s' to matlab." % inFile) # end of (for fileCollection)
except AttributeError, e: mainLogger.debug("Ignore reading object of type %s.", type(d)) ## Treat special arguments if len( args.inputFiles ) == 1: args.inputFiles = csvStr2List( args.inputFiles[0] ) args.inputFiles = expandFolders( args.inputFiles ) mainLogger.verbose("All input files are:") if mainLogger.isEnabledFor( LoggingLevel.VERBOSE ): pprint(args.inputFiles) for inFile in progressbar(args.inputFiles, len(args.inputFiles), logger = mainLogger, prefix = "Processing files "): # Treat output file name: from RingerCore import checkExtension, changeExtension, save if checkExtension( inFile, "root" ): cOutputName = changeExtension( inFile, '.npz' ) if args.change_output_folder: import os.path cOutputName = os.path.join( os.path.abspath(args.change_output_folder) , os.path.basename(cOutputName) ) f = ROOT.TFile( inFile, 'r' ) data = {} for keyName, obj in getall(f): mainLogger.debug("Reading key: %s", keyName) shortKey = keyName.split('/')[-1] if not issubclass(type(obj), ROOT.TH1): mainLogger.verbose("Ignoring key: %s", shortKey ) continue hist = obj if all( [wanted not in shortKey for wanted in args.filter_keys] ): mainLogger.debug("key <%s> does not match any filter", shortKey )