Beispiel #1
0
def main():
    """main options for program"""
    # command line options
    parser = OptionParser()
    parser.add_option("--file", "-f", help="ROOT input file", dest="file")
    parser.add_option("--tree", "-t", help="Tree to parse, Default Event", dest="treeName")
    parser.add_option("--sizeCutoff", "-s", dest="read_cutoff", help="Read size cutoff, Default 100")
    parser.add_option("--output_directory", "-o", dest="output", help="The name of the output file")
    parser.add_option("--regex", "-r", dest="fileNameRegex", help="Regular expression to filter files")
    parser.add_option(
        "--window_size", "-w", dest="window_size", help="Size of the displayed data graphic in mb Default:20"
    )
    parser.add_option("--begin_read", "-b", dest="beginning", help="Beeginning read to display in the file graphic")
    parser.add_option(
        "--raw_only",
        dest="raw_only",
        help="Create only the raw read graphics and not overlayed file usage",
        action="store_true",
    )
    parser.add_option(
        "--even_check",
        "-c",
        dest="check_fun_file",
        help="File to import that contains checkEvent method to validate any event",
    )
    parser.add_option("--top_n", "-n", dest="topN", help="Color only the top N branches in the file")
    parser.add_option("--prefix", "-p", dest="prefix", help="Prefix to append to files to search for file on system")
    (options, args) = parser.parse_args()

    #
    #   Handle command line options
    #
    #
    # If now file we have an error
    if options.file is None:
        print "File root file is required"
        parser.print_help()
        exit(0)
    if not options.file.upper().endswith(".ROOT"):
        print "Root file required"
        exit()
    try:
        with open(options.file) as f:
            pass
    except IOError as e:
        print "file does not exits"
        exit()

    # require the file name and tree
    if options.treeName is None:
        print "Using default tree of Events"
        treeName = "Events"
    else:
        treeName = options.treeName

    # output file name
    if options.output != None:
        outDir = options.output + "/"
    else:
        outDir = "./"

    # regular expression
    if options.fileNameRegex != None:
        frex = re.compile(options.fileNameRegex)
    else:
        frex = None

    # size expression
    if options.window_size != None:
        window_size = int(options.window_size)
    else:
        window_size = 20

    if options.read_cutoff != None:
        size_cutoff = int(options.read_cutoff)
    else:
        size_cutoff = 50

    if options.beginning:
        begin = int(options.beginning)
    else:
        begin = 0

    if options.raw_only:
        raw_only = True
    else:
        raw_only = False

    if options.check_fun_file:
        check_fun_file = __import__(options.check_fun_file)
    else:
        check_fun_file = None

    topN = None
    # color only top ten files
    if options.topN:

        topN = int(options.topN)
        if topN > 30:
            topN = 30
    else:
        topN = 30

    # prefix for the file name
    prefix = "/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms"
    if options.prefix:
        prefix = options.prefix

    #
    #   End Command Line Stuff
    #

    # Load the needed c++ classes into here
    ROOT.gROOT.ProcessLine("gErrorIgnoreLevel = 2500;")
    if os.path.exists("XrdFar"):
        ROOT.gSystem.Load("XrdFar/XrdFar")

    tf = ROOT.TFile(options.file)

    if not os.path.exists("XrdFar"):
        tf.MakeProject("XrdFar", "*", "new++")

    # Loop through all the events within the file
    for event in tf.XrdFar:
        if fileShouldBeRead(event, frex, size_cutoff, check_fun_file):
            print "\n\n"
            print "Starting on File: %s" % event.F.mName
            # get a bunch of needed info now that it has passed
            offsets = list(event.I.mOffsetVec)
            lengths = list(event.I.mLengthVec)
            name = event.F.mName
            uname = event.U.mServerUsername
            server = event.S.mHost
            readSize = event.F.mRTotalMB
            data = []
            idx = name.rfind("/")
            shortName = name[idx + 1 :]
            idx = name.rfind("/", 0, idx)
            # Alsor removing /store from the front of them
            title = name[7:idx]
            file = prefix + name

            # trasform the data
            for i in range(len(offsets)):
                val = transformData(offsets[i], lengths[i], i)
                data = data + val
            arr = createArray(data)
            points = markPoints(data)
            usage, limits = fixFileUsage(arr, size=window_size, start=begin)
            try:
                print "\tCreating Usage Graph"
                HCCPlot.plotUsage(
                    arr, name=title, outname=outDir + shortName + "_rawreads", points=points, subtitle=uname
                )
            except Exception, e:
                print "Error Plotting Layout: %s" % str(e)

            if not raw_only:
                t1 = time()
                print "\tParsing layout"
                fileLayout = HCCRootParser.parseFileTool2(file, treeName, None, False, topN)
                t2 = time()
                print "\t\tParse Time: " + str(t2 - t1)
                tenBig = HCCTool1.getTenBig(fileLayout, topN)
                fileLayout = HCCPlot.transformByteToMB(fileLayout)
                fileLayout = cutToRange(fileLayout, limits)
                colorMap = HCCTool1.createColorMap(fileLayout, False, tenBig)

                try:
                    print "\tPlotting layout overlay"
                    HCCPlot.plotFileLayoutTool2(
                        fileLayout,
                        False,
                        outDir + shortName + ".png",
                        colorMap,
                        tenBig,
                        limits=limits,
                        title=title,
                        fileUsage=usage,
                    )
                except Exception, e:
                    print "Error Plotting Layout: %s" % str(e)
Beispiel #2
0
def main():
    '''main options for program'''
    #command line options
    parser = OptionParser()
    parser.add_option('--file', '-f', help='ROOT input file',dest='file')
    parser.add_option('--tree', '-t', help='Tree to parse, Default Event', dest='treeName')
    parser.add_option('--sizeCutoff', '-s',dest='read_cutoff', help='Read size cutoff, Default 100')
    parser.add_option('--output_directory','-o', dest = 'output', help='The name of the output file')
    parser.add_option('--regex', '-r', dest='fileNameRegex', help='Regular expression to filter files')
    parser.add_option('--window_size', '-w', dest='window_size', help='Size of the displayed data graphic in mb Default:20')
    parser.add_option('--begin_read', '-b', dest='beginning', help='Beeginning read to display in the file graphic')
    parser.add_option('--raw_only', dest='raw_only', help='Create only the raw read graphics and not overlayed file usage', action='store_true')
    parser.add_option('--even_check', '-c', dest='check_fun_file', help='File to import that contains checkEvent method to validate any event')
    parser.add_option('--top_n','-n', dest = 'topN' , help='Color only the top N branches in the file')
    parser.add_option('--prefix','-p', dest = 'prefix' , help='Prefix to append to files to search for file on system')
    (options,  args)= parser.parse_args()


    #
    #   Handle command line options
    #
    #
    #If now file we have an error
    if options.file is None:
        print "File root file is required"
        parser.print_help()
        exit(0)
    if not options.file.upper().endswith('.ROOT'):
        print 'Root file required'
        exit()
    try:
        with open(options.file) as f: pass
    except IOError as e:
        print 'file does not exits'
        exit()
    
    #require the file name and tree
    if options.treeName is None:
        print "Using default tree of Events"
        treeName = 'Events'
    else:   
        treeName = options.treeName
    
    #output file name 
    if options.output != None:
        outDir = options.output + '/'
    else:
        outDir = './'
        
    #regular expression
    if options.fileNameRegex != None:
        frex = re.compile(options.fileNameRegex)
    else:
        frex = None
  
    #size expression
    if options.window_size != None:
        window_size = int(options.window_size)
    else:
        window_size = 20
        
    if options.read_cutoff != None:
        size_cutoff = int(options.read_cutoff)
    else:
        size_cutoff = 50
        
    if options.beginning:
        begin = int(options.beginning)
    else:
        begin = 0
        
    if options.raw_only:
        raw_only = True
    else:
        raw_only = False
     
    if options.check_fun_file:
        check_fun_file = __import__(options.check_fun_file)
    else:
        check_fun_file = None
        
    topN = None
    #color only top ten files
    if options.topN:
        
        topN = int(options.topN)
        if topN > 30:
            topN = 30
    else:
        topN = 30 

    #prefix for the file name
    prefix = '/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms'
    if options.prefix:
        prefix = options.prefix
        
     #
     #   End Command Line Stuff 
     #
     
     #Load the needed c++ classes into here  
    ROOT.gROOT.ProcessLine("gErrorIgnoreLevel = 2500;")
    if os.path.exists("XrdFar"):
        ROOT.gSystem.Load("XrdFar/XrdFar")

    tf = ROOT.TFile(options.file)
    
    if not os.path.exists("XrdFar"):
        tf.MakeProject("XrdFar", "*", "new++")
        
    #Loop through all the events within the file
    for event in tf.XrdFar:
        if fileShouldBeRead(event, frex, size_cutoff, check_fun_file):
            print '\n\n'
            print 'Starting on File: %s' % event.F.mName
            #get a bunch of needed info now that it has passed
            offsets = list(event.I.mOffsetVec) 
            lengths = list(event.I.mLengthVec)
            name = event.F.mName
            uname = event.U.mServerUsername
            server = event.S.mHost
            readSize = event.F.mRTotalMB
            data = []
            idx = name.rfind('/')
            shortName = name[idx+1:]
            idx = name.rfind('/',0,idx)
            #Alsor removing /store from the front of them
            title = name[7:idx]
            file = prefix + name
            
            #trasform the data 
            for i in range(len(offsets)):
                val = transformData(offsets[i], lengths[i], i)
                data = data +  val
            arr = createArray(data)
            points = markPoints(data)
            usage, limits = fixFileUsage(arr,size = window_size, start = begin)
            try:
                print '\tCreating Usage Graph'
                HCCPlot.plotUsage(arr, name = title, outname = outDir + shortName+'_rawreads', points = points, subtitle = uname )
            except Exception, e:
                print "Error Plotting Layout: %s" % str(e)


            if not raw_only:
                t1 = time()
                print '\tParsing layout'
                fileLayout = HCCRootParser.parseFile(file, treeName, None, False, topN)
                t2 = time();
                print '\tTime: ' + str(t2-t1)
                tenBig= HCCTool1.getTenBig(fileLayout, topN)
                fileLayout = HCCPlot.transformByteToMB(fileLayout)
                fileLayout = cutToRange(fileLayout, limits)
                colorMap = HCCTool1.createColorMap(fileLayout, False, tenBig)
                
                try:
                    print "\tPlotting layout overlay"
                    HCCPlot.plotFileLayout(fileLayout, False,outDir + shortName + '.png',  colorMap, tenBig,limits = limits, title = title, fileUsage = usage)
                except Exception, e:
                    print "Error Plotting Layout: %s" % str(e)
Beispiel #3
0
def main():
    """main options for program"""
    # command line options
    parser = OptionParser()
    parser.add_option("--file", "-f", help="ROOT input file", dest="file")
    parser.add_option("--tree", "-t", help="Tree to parse", dest="treeName")
    parser.add_option(
        "--firstN", "-m", help="Display only the first N buckets of the file on the graph", dest="maxBranch"
    )
    parser.add_option(
        "--display", "-d", dest="display", action="store_true", help="Show interactive matplotlib display"
    )
    parser.add_option(
        "--top_level", "-b", dest="topLevel", action="store_true", help="Color by only the top level branch"
    )
    parser.add_option("--output_file", "-o", dest="output", help="The name of the output file")
    parser.add_option("--top_n", "-n", dest="topN", help="Color only the top N branches in the file")
    parser.add_option(
        "--eventFilter",
        "-e",
        dest="events",
        help="Plot only up to this event number or this range of events in the display",
    )
    parser.add_option("--list_trees", "-l", dest="list", action="store_true", help="List the available trees in file")
    parser.add_option("--branch_regex", "-r", dest="branchRegex", help="Regular expression to filter out branches")
    (options, args) = parser.parse_args()

    # If now file we have an error
    if options.file is None:
        print "File root file is required"
        parser.print_help()
        exit(0)
    if not options.file.upper().endswith(".ROOT"):
        print "Root file required"
        exit()
    try:
        with open(options.file) as f:
            pass
    except IOError as e:
        print "file does not exits"
        exit()

    # if list just list trees and exit
    if options.list:
        HCCRootParser.listFileTrees(options.file)
        exit(0)

    # require the file name and tree
    if options.treeName is None:
        print "Tree Name is required"
        parser.print_help()
        exit(-1)

    numOfBranches = None
    if options.maxBranch:
        numOfBranches = int(options.maxBranch)

    # output file name
    if options.output != None:
        outName = options.output
    else:
        outName = getName(options.file)

    # interactive display
    if options.display:
        display = True
    else:
        display = False

    if options.topLevel:
        topLevel = True
    else:
        topLevel = False

    topN = None
    # color only top ten files
    if options.topN:

        topN = int(options.topN)
        if topN > 30:
            topN = 30
    else:
        topN = 30

    # regular expression
    if options.branchRegex != None:
        brex = re.compile(options.branchRegex)
    else:
        brex = None

    # get data
    data = HCCRootParser.parseFile(options.file, options.treeName, brex, topLevel)

    if options.events:
        data = filterByEvents(data, options.events)

    tenBig = getTenBig(data, topN)

    if numOfBranches != None:
        data = truncateData(data, numOfBranches)

    # no need to make graph if blank data
    if len(data) == 0:
        print "No Branches in that tree"
        exit(0)

    data = HCCPlot.transformByteToMB(data)

    # plot
    if brex != None:
        colorMap = {}
        HCCPlot.plotFileLayout(data, display, outName, colorMap, tenBig)
    else:
        colorMap = createColorMap(data, False, tenBig)
        HCCPlot.plotFileLayout(data, display, outName, colorMap, tenBig)
Beispiel #4
0
def main():
    '''main options for program'''
    #command line options
    parser = OptionParser()
    parser.add_option('--file', '-f', help='ROOT input file',dest='file')
    parser.add_option('--tree', '-t', help='Tree to parse', dest='treeName')
    parser.add_option('--firstN', '-m', help='Display only the first N buckets of the file on the graph', dest='maxBranch')
    parser.add_option('--display', '-d', dest='display', action='store_true' ,help='Show interactive matplotlib display')
    parser.add_option('--top_level', '-b', dest='topLevel', action='store_true' ,help='Color by only the top level branch')
    parser.add_option('--output_file','-o', dest = 'output', help='The name of the output file')
    parser.add_option('--top_n','-n', dest = 'topN' , help='Color only the top N branches in the file')
    parser.add_option('--eventFilter','-e' ,dest ='events', help='Plot only up to this event number or this range of events in the display')
    parser.add_option('--list_trees', '-l', dest='list', action='store_true', help='List the available trees in file')
    parser.add_option('--branch_regex', '-r', dest='branchRegex', help='Regular expression to filter out branches')
    (options,  args)= parser.parse_args()


    #If now file we have an error
    if options.file is None:
        print "File root file is required"
        parser.print_help()
        exit(0)
    if not options.file.upper().endswith('.ROOT'):
        print 'Root file required'
        exit()
    try:
        with open(options.file) as f: pass
    except IOError as e:
        print 'file does not exits'
        exit()
    

    #if list just list trees and exit
    if(options.list):
        HCCRootParser.listFileTrees(options.file)
        exit(0)


    #require the file name and tree
    if options.treeName is None:
        print "Tree Name is required"
        parser.print_help()
        exit(-1)

    numOfBranches = None
    if options.maxBranch:
        numOfBranches = int(options.maxBranch)

    
    #output file name 
    if options.output != None:
        outName = options.output
    else:
        outName = getName(options.file)

    #interactive display
    if options.display:
        display = True
    else:
        display = False

    if options.topLevel:
        topLevel = True
    else:
        topLevel  = False 
    
    topN = None
    #color only top ten files
    if options.topN:
        
        topN = int(options.topN)
        if topN > 30:
            topN = 30
    else:
        topN = 30 

    #regular expression
    if options.branchRegex != None:
        brex = re.compile(options.branchRegex)
    else:
        brex = None

    print 'parsing data'
    #get data
    data = HCCRootParser.parseFile(options.file, options.treeName, brex, topLevel) 

    if options.events:
        data = filterByEvents(data, options.events)

    tenBig= getTenBig(data, topN)

    if (numOfBranches != None):
       data =  truncateData(data, numOfBranches)

    #no need to make graph if blank data
    if len(data) == 0:
        print 'No Branches in that tree'
        exit(0)

    data = HCCPlot.transformByteToMB(data)

    print 'plotting'
    #plot
    if brex != None:
        colorMap = {} 
        HCCPlot.plotFileLayout(data, display, outName, colorMap, tenBig)
    else:   
        colorMap = createColorMap(data, False, tenBig)
        HCCPlot.plotFileLayout(data, display, outName, colorMap, tenBig)