Exemplo n.º 1
0
def do_main(args):
    '''
    '''
    fsrc = LMTConfig.process_configuration(args)
    (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
    if args.mds == True:
        query = "SELECT TIMESTAMP_INFO.TS_ID,TIMESTAMP,PCT_CPU FROM TIMESTAMP_INFO,MDS_DATA WHERE "
        query += "TIMESTAMP_INFO.TS_ID=MDS_DATA.TS_ID AND "
    else:
        query = "SELECT TIMESTAMP_INFO.TS_ID,TIMESTAMP,PCT_CPU FROM TIMESTAMP_INFO,OSS_DATA,OSS_INFO WHERE "
        query += "OSS_DATA.OSS_ID=OSS_INFO.OSS_ID and OSS_INFO.HOSTNAME='"
        query += args.oss + "' AND "
        query += "TIMESTAMP_INFO.TS_ID=OSS_DATA.TS_ID AND "
    query += "TIMESTAMP_INFO.TIMESTAMP >= '"
    query += beginTimestamp.timestr
    query += "' AND TIMESTAMP_INFO.TIMESTAMP <= '"
    query += endTimestamp.timestr
    query += "'"
    try:
        cursor = fsrc['conn'].cursor(MySQLdb.cursors.DictCursor)
        if args.verbose == True:
            print "\t%s" % query
        cursor.execute (query)
    except MySQLdb.Error, e:
        cursor.close()
        print "CPU.get_ops_data_from_db: Error %d: %s\n%s" % (e.args[0], e.args[1], query)
        return(None)
Exemplo n.º 2
0
def do_main(args):
    """
    It looks like it is possible to get an incomplet coverage of the set of time
    steps if you only get rate and brw_stats data for one OST. I should fix this
    in the base modules.
    """
    fsrc = LMTConfig.process_configuration(args)
    sie = Timestamp.calc_sie(args.time)
    beginTimestamp = Timestamp.Timestamp(sie - 5)
    endTimestamp   = Timestamp.Timestamp(sie)
    beginTimestamp.no_later_than(fsrc['conn'])
    endTimestamp.no_earlier_than(fsrc['conn'])
    if beginTimestamp.ts_id >= endTimestamp.ts_id:
        print "observation: failed to get time stamp range for %s" % args.time
        exit(1)
    Steps = TimeSteps.TimeSteps()
    Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
    bulk = Bulk.Bulk(fsrc['name'])
    bulk.getOSSs(fsrc['conn'])
    bulk.setSteps(Steps)
    bulk.getData()
    # FIXME
    # I am not handling masked values in the CPU data
    # Should be an easy fix
    bulk.getCPU()
    brwfs = BrwFS.BrwFS(fsrc['name'])
    brwfs.getOSSs(fsrc['conn'])
    # we'll query for all of them even when we only want one
    # because it make the processing more uniform
    brwfs.getBrwStats(fsrc['conn'], stat=args.stat)
    brwfs.getData(beginTimestamp, endTimestamp, stat=args.stat)
    return(bulk, brwfs)
Exemplo n.º 3
0
def do_main(args):
    """
    args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
                   It will fill in todays year, month and day if left out.
                   hh:mm:ss will default to 00:00:00 as portions are left out.
         config - (file) The lmtrc config file telling how to get to the DB
         end - (string) As above giving the end of the data to be gathered.
         FS - (string) The dbname entry in the config file for the file system of interest.
         index - (int) The index of the file system of interest in the config file
         oss - (string) The hostname of the OSS to be analyzed.
         verbose - (boolean) Turn on debugging output
         version - (boolean) print the version string and exit
    """
    # do_main will:
    # - process_configuration
    # - get the oss in question
    # - get the OSTs on it
    # - Process timestamps
    # - get the data including CPU utilization
    # - return the oss
    fsrc = LMTConfig.process_configuration(args)
    try:
        cursor = fsrc['conn'].cursor(MySQLdb.cursors.DictCursor)
        query = "SELECT * FROM OSS_INFO"
        cursor.execute (query)
        rows = cursor.fetchall()
    except MySQLdb.Error, e:
        cursor.close()
        print "OSS: Error %d: %s" % (e.args[0], e.args[1])
        return
Exemplo n.º 4
0
def do_main(args):
    """
    args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
                   It will fill in todays year, month and day if left out.
                   hh:mm:ss will default to 00:00:00 as portions are left out.
         config - (file) The lmtrc config file telling how to get to the DB
         end - (string) As above giving the end of the data to be gathered.
         fs - (string) The dbname entry in the config file for the file system of interest.
         index - (int) The index of the file system of interest in the config file
         verbose - (boolean) Turn on debugging output
         version - (boolean) print the version string and exit
    """
    # do_main will:
    # - process_configuration
    # - get all the OSSs which gets their OSTs as well
    # - Process timestamps
    # - get the data including CPU utilization if asked
    # - return the bulk_aggregate object
    fsrc = LMTConfig.process_configuration(args)
    bulk = Bulk.Bulk(fsrc['name'])
    if args.verbose == True:
        bulk.debug()
    bulk.getOSSs(fsrc['conn'])
    (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
    Steps = TimeSteps.TimeSteps()
    Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
    bulk.setSteps(Steps)
    if (bulk.Steps == None) or (bulk.Steps.steps() == 0):
        print "bulk_aggregate: Warning - No steps from FS %s" % bulk.name
        return(None)
    bulk.getData()
    bulk.getCPU()
    return(bulk)
Exemplo n.º 5
0
def do_main(args):
    """
    args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
                   It will fill in todays year, month and day if left out.
                   hh:mm:ss will default to 00:00:00 as portions are left out.
         config - (file) The lmtrc config file telling how to get to the DB
         end - (string) As above giving the end of the data to be gathered.
         fs - (string) The dbname entry in the config file for the file system of interest.
         index - (int) The index of the file system of interest in the config file
         verbose - (boolean) Turn on debugging output
         version - (boolean) print the version string and exit
    """
    fsrc = LMTConfig.process_configuration(args)
    mds = MDS.MDS(host=fsrc["host"], fs=fsrc["name"])
    if args.verbose == True:
        mds.debug()
        mds.debug(module="Operation")
        # mds.debug(module="Timestamp")
    mds.opsFromDB(fsrc["conn"])
    (begin_ts, end_ts) = Timestamp.process_timestamps(args, fsrc)
    mds.getData(begin=begin_ts, end=end_ts)
    if mds.haveData == False:
        print "got no data from %s to %s" % (args.begin, args.end)
        return None
    mds.getCPU()
    return mds
Exemplo n.º 6
0
def do_main(args):
    """
    args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
                   It will fill in todays year, month and day if left out.
                   hh:mm:ss will default to 00:00:00 as portions are left out.
         both - Add together the read and write rates for calculation
         config - (file) The lmtrc config file telling how to get to the DB
         cpu - (boolean) Do the CPU utilization calculations (always set this true)
         end - (string) As above giving the end of the data to be gathered.
         fs - (string) The dbname entry in the config file for the file system of interest.
         index - (int) The index of the file system of interest in the config file
         oss - (string) The hostname of the OSS to be analyzed.
         ost - (string) The name of the OST to be analyzed.
         read - Caclulate for the read data rate
         verbose - (boolean) Turn on debugging output
         version - (boolean) print the version string and exit
         write - Calculate for the write data rate
    """
    fsrc = LMTConfig.process_configuration(args)
    bulk = Bulk.Bulk(fsrc['dbname'])
    if args.verbose == True:
        bulk.debug()
    bulk.getOSSs(fsrc['conn'])
    (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
    Steps = TimeSteps.TimeSteps()
    Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
    bulk.setSteps(Steps)
    if (bulk.Steps == None) or (bulk.Steps.steps() == 0):
        print "data_moved.do_main(): Warning - No steps from FS %s" % bulk.name
        sys.exit(1)
    bulk.getData()
    if args.cpu == True:
        bulk.getCPU()
    return(bulk)
Exemplo n.º 7
0
def do_main(args):
    """
    It looks like it is possible to get an incomplet coverage of the set of time
    steps if you only get rate and brw_stats data for one OST. I should fix this
    in the base modules.
    """
    fsrc = LMTConfig.process_configuration(args)
    (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
    bulk = Bulk.Bulk(fsrc['name'])
    bulk.getOSSs(fsrc['conn'])
    Steps = TimeSteps.TimeSteps()
    Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
    bulk.setSteps(Steps)
    bulk.getData()
    bulk.getCPU()
    brwfs = BrwFS.BrwFS(fsrc['name'])
    brwfs.getOSSs(fsrc['conn'])
    brwfs.setSteps(Steps)
    if args.stat == 'BRW_IOSIZE':
        iosize_bins = np.array([4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576])
        brwfs.getBrwStats(fsrc['conn'], args.stat, bins=iosize_bins)
    else:
        brwfs.getBrwStats(fsrc['conn'], args.stat)
    brwfs.getData(args.stat)
    return(bulk, brwfs)
Exemplo n.º 8
0
def doMain(args):
    """
    """
    fsrc = LMTConfig.process_configuration(args)
    args.host = fsrc['host']
    #beginSie = Timestamp.calc_sie(args.begin)
    #endSie = Timestamp.calc_sie(args.end)
    beginSie = int(time.mktime(time.strptime(args.begin, "%Y-%m-%d %H:%M:%S" )))
    endSie = int(time.mktime(time.strptime(args.end, "%Y-%m-%d %H:%M:%S" )))
    fs = FS.FS(fsrc['name'])
    fs.getInfo(fsrc['conn'])
    return(fs, beginSie, endSie)
Exemplo n.º 9
0
def do_main(args):
    """
    args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
                   It will fill in todays year, month and day if left out.
                   hh:mm:ss will default to 00:00:00 as portions are left out.
         config - (file) The lmtrc config file telling how to get to the DB
         cpu - (boolean) Get CPU utilization data and potentially plot it
         end - (string) As above giving the end of the data to be gathered.
         FS - (string) The dbname entry in the config file for the file system of interest.
         index - (int) The index of the file system of interest in the config file
         metadata - (boolean) Do metadata rather than bulk I/O data
         plot - (string) The name of the file to which the graph should be saved.
                  'noplot' is allowed if you just want a report.
         report - (boolean) Print out summary info about the analyzed operations
         verbose - (boolean) Turn on debugging output
         version - (boolean) print the version string and exit
         x_correlate - (boolean) plot the ops versus the CPU utilization rather than
                         the ops versus time.
         ybound - (float) Use the given value as the maximum of the y-acis

    You can work with a data set interactively by entering ipython at the command line
    and carrying out this sequence of preparatory steps:
from pyLMT import FS
from pyLMT.tests import test_FS
args = FS.process_args()
args.begin = '2012-01-12'
args.end = '2012-01-13'
args.cpu = True
args.FS = 'filesystem_scratch'
fs = test_FS.do_main(args)

    It may take as much as half an hour to load up 24 hours worth of data,
    but once you have it in hand you can explore the data set interactively
    without having to go back and query the LMT DB over and over.
    """
    # do_main will:
    # - process_configuration
    # - get the OSSs and metadata ops
    # - Process timestamps
    # - get the bulk data
    # - get the metadata
    # - return the FS object
    fsrc = LMTConfig.process_configuration(args)
    fs = FS.FS(fsrc['name'])
    if args.verbose == True:
        fs.debug()
    fs.getInfo(fsrc['conn'])
    (begin_ts, end_ts) = Timestamp.process_timestamps(args, fsrc)
    fs.getData(begin_ts,
               end_ts)
    if (args.cpu == True) or (args.x_correlate == True):
        fs.getCPU()
    return(fs)
Exemplo n.º 10
0
def do_main(args):
    """
    """
    fsrc = LMTConfig.process_configuration(args)
    try:
        cursor = fsrc['conn'].cursor(MySQLdb.cursors.DictCursor)
        query = "SELECT * FROM OST_INFO"
        cursor.execute (query)
        rows = cursor.fetchall()
    except MySQLdb.Error, e:
        cursor.close()
        print "OST: Error %d: %s" % (e.args[0], e.args[1])
        return(None)
Exemplo n.º 11
0
def do_main(args):
    """
    args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
                   It will fill in todays year, month and day if left out.
                   hh:mm:ss will default to 00:00:00 as portions are left out.
         both - Add together the read and wtites in the plot
         config - (file) The lmtrc config file telling how to get to the DB
         cpu - plot CPU utilization
         end - (string) As above giving the end of the data to be gathered.
         fs - (string) The dbname entry in the config file for the file system of interest.
         index - (int) The index of the file system of interest in the config file
         lines - (boolean) Graph using lines (no matter how many data points)
         mask - (key=value:keys in {mincpu, maxcpu, minval, maxval}) mask values outside the given range
         plot - (string) The name of the file to which the graph should be saved.
                  'noplot' is allowed if you just want a report.
         plotSdevs (boolean) Calculate and plot the standard deviation across OSTs at each timestep
         report - (boolean) Print out summary info about the analyzed operations
         read - Plot read data rate
         spectrum - Plot the cross-correlation spectrum
         frac - Filter out samples of the spectrum below this fraction of the max
         verbose - (boolean) Turn on debugging output
         version - (boolean) print the version string and exit
         write - Plot the write rate
         x_correlate - (boolean) plot the ops versus the CPU utilization rather than
                         the ops versus time.
         ybound - (float) Use the given value as the maximum of the y-acis
    """
    # do_main will:
    # - process_configuration
    # - get all the OSSs which gets their OSTs as well
    # - Process timestamps
    # - get the data including CPU utilization if asked
    # - return the bulk object
    fsrc = LMTConfig.process_configuration(args)
    if fsrc is None:
        return None
    B = Bulk.Bulk(fsrc['name'])
    if args.verbose == True:
        B.debug()
    B.getOSSs(fsrc['conn'])
    (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
    Steps = TimeSteps.TimeSteps()
    Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
    B.setSteps(Steps)
    if (B.Steps is None) or (B.Steps.steps() == 0):
        print "bulk: Warning - No steps from FS %s" % B.name
        sys.exit(1)
    B.getData()
    if (args.cpu == True) or (args.x_correlate == True):
        B.getCPU()
    return(B)
Exemplo n.º 12
0
Arquivo: mds.py Projeto: dani-lbnl/lmt
def do_main(args):
    """
    args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
                   It will fill in todays year, month and day if left out.
                   hh:mm:ss will default to 00:00:00 as portions are left out.
         config - (file) The lmtrc config file telling how to get to the DB
         cpu - plot CPU utilization
         end - (string) As above giving the end of the data to be gathered.
         extra - (int) If None then ignore, otherwise pad the graph before and after
                 after the interval with this much extra, but don't inlcude in
                 average calculation.
         fs - (string) The dbname entry in the config file for the file system of interest.
         index - (int) The index of the file system of interest in the config file
         mask - (key=value:keys in {mincpu, maxcpu, minval, maxval}) mask values outside the given range
         plot - (string) The name of the file to which the graph should be saved.
                  'noplot' is allowed if you just want a report.
         report - (boolean) Print out summary info about the analyzed operations
         show_ops - (boolean) List the available operations
         show_steps - (boolean) List the steps that are in the -M <mask> region
         spectrum - (boolean)  Produce a graph of the Ops rate versus CPU utilization spectrum
         verbose - (boolean) Turn on debugging output
         version - (boolean) print the version string and exit
         x_correlate - (boolean) plot the ops versus the CPU utilization rather than
                         the ops versus time.
         ybound - (float) Use the given value as the maximum of the y-acis
    """
    fsrc = LMTConfig.process_configuration(args)
    mds = MDS.MDS(host=fsrc['host'], fs=fsrc['name'])
    if args.verbose == True:
        mds.debug()
        mds.debug(module="Operation")
        #mds.debug(module="Timestamp")
    mds.opsFromDB(fsrc['conn'])
    if args.show_ops == True:
        mds.showOps()
        return
    (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
    Steps = TimeSteps.TimeSteps()
    Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
    mds.setSteps(Steps)
    mds.getData()
    if mds.haveData == False:
        print "got no data from %s to %s" % (args.begin, args.end)
        return(None)
    if (args.cpu == True) or (args.x_correlate == True):
        mds.getCPU()
    if not args.extra is None:
        if (args.extra <= 0) or (args.extra >= 100):
            print "mds: extra should be an integer between 0 and 100"
            args.extra = None
    return(mds)
Exemplo n.º 13
0
Arquivo: ost.py Projeto: dani-lbnl/lmt
def do_main(args):
    """
    """
    fsrc = LMTConfig.process_configuration(args)
    (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
    try:
        cursor = fsrc['conn'].cursor(MySQLdb.cursors.DictCursor)
        query = "SELECT * FROM OST_INFO"
        cursor.execute (query)
        rows = cursor.fetchall()
    except MySQLdb.Error, e:
        cursor.close()
        print "ost: Error %d: %s" % (e.args[0], e.args[1])
        sys.exit(1)
Exemplo n.º 14
0
Arquivo: oss.py Projeto: dani-lbnl/lmt
def do_main(args):
    """
    args.apportion - A seconds in epoch value at which to show the amount of
                     I/O from each OST.
         begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
                   It will fill in todays year, month and day if left out.
                   hh:mm:ss will default to 00:00:00 as portions are left out.
         both - Add together the read and writes in the plot
         config - (file) The lmtrc config file telling how to get to the DB
         cpu - plot CPU utilization
         end - (string) As above giving the end of the data to be gathered.
         fs - (string) The dbname entry in the config file for the file system of interest.
         index - (int) The index of the file system of interest in the config file
         lines - (boolean) Graph using lines (no matter how many data points)
         mask - (key=value:keys in {mincpu, maxcpu, minval, maxval}) mask values outside the given range
         oss - (string) The hostname of the OSS to be analyzed.
         plot - (string) The name of the file to which the graph should be saved.
                  'noplot' is allowed if you just want a report.
         plotSdevs (boolean) Calculate and plot the standard deviation across OSTs at each timestep
         report - (boolean) Print out summary info about the analyzed operations
         read - Plot read data rate
         show_osss - (boolean) Just list the available OSS host names to choose
                       from for the -o <oss> argument, then exit.
         show_osts - (boolean) Just list the OSTs on the OSS -o <oss>
         show_steps - (boolean) List the steps that are in the -M <mask> region
         spectrum - (boolean) Produce a graph of the rate versus CPU utilization spectrum
         verbose - (boolean) Turn on debugging output
         version - (boolean) print the version string and exit
         write - Plot the write rate
         x_correlate - (boolean) plot the ops versus the CPU utilization rather than
                         the ops versus time.
         ybound - (float) Use the given value as the maximum of the y-acis
    """
    # do_main will:
    # - process_configuration
    # - get the oss in question with early terminiation if it's just a show_osss
    # - get the OSTs on it with early termination if it's just a show_osts
    # - Process timestamps
    # - get the data including CPU utilization if asked
    # - return the oss
    fsrc = LMTConfig.process_configuration(args)
    try:
        cursor = fsrc['conn'].cursor(MySQLdb.cursors.DictCursor)
        query = "SELECT * FROM OSS_INFO"
        cursor.execute (query)
        rows = cursor.fetchall()
    except MySQLdb.Error, e:
        cursor.close()
        print "oss: Error %d: %s" % (e.args[0], e.args[1])
        return
Exemplo n.º 15
0
def get_fsrc(args):
    fsrc = None
    if args.host == 'franklin':
        if args.filesys == 'scratch':
            args.fs = 'filesystem_franklin_scratch_20090420'
            fsrc = LMTConfig.process_configuration(args)
        elif args.filesys == 'scratch2':
            args.fs = 'filesystem_franklin_scratch2_20090420'
            fsrc = LMTConfig.process_configuration(args)
        ymax = 18000
    elif args.host == 'hopper':
        if args.filesys == 'scratch':
            args.fs = 'filesystem_scratch'
            fsrc = LMTConfig.process_configuration(args)
            if fsrc is None:
                return('fsrc is None')
            else:
                return(fsrc)
        elif args.filesys == 'scratch2':
            args.fs = 'filesystem_scratch2'
            fsrc = LMTConfig.process_configuration(args)
        ymax = 50000
    return(fsrc)
Exemplo n.º 16
0
def do_main(args):
    """
    args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
                   It will fill in todays year, month and day if left out.
                   hh:mm:ss will default to 00:00:00 as portions are left out.
         config - (file) The lmtrc config file telling how to get to the DB
         end - (string) As above giving the end of the data to be gathered.
         fs - (string) The dbname entry in the config file for the file system of interest.
         index - (int) The index of the file system of interest in the config file
         plot - (string) The name of the file to which the graph should be saved.
                  'noplot' is allowed if you just want a report.
         report - (boolean) Print out summary info about the analyzed operations
         read - Plot read data rate
         stat - Show histograms only for this statistic
         verbose - (boolean) Turn on debugging output
         version - (boolean) print the version string and exit
         write - Plot the write rate
    """
    # do_main will:
    # - process_configuration
    # - get the oss in question
    # - get the OSTs on it
    # - Process timestamps
    # - get the data
    # - return the oss
    fsrc = LMTConfig.process_configuration(args)
    brwfs = BrwFS.BrwFS(fsrc['dbname'])
    brwfs.getOSSs(fsrc['conn'])
    if args.verbose == True:
        brwfs.debug()
        brwfs.debug(module="BrwFS")
        #oss.debug(module="Timestamp")
    (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
    Steps = TimeSteps.TimeSteps()
    Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
    brwfs.setSteps(Steps)
    brwfs.getBrwStats(conn=fsrc['conn'], stat=args.stat)
    brwfs.getData(stat=args.stat)
    if (brwfs.Bins is None) or (len(brwfs.Bins) == 0):
        print "test_BrwFS: Warning - No HistBins objects from OSS %s" % oss.name
        return
    if (brwfs.Steps is None) or (brwfs.Steps.steps() == 0):
        print "test_BrwFS: Warning - No steps from OSS %s" % oss.name
        return
    return(brwfs)
Exemplo n.º 17
0
def do_main(args):
    '''
    '''
    fsrc = LMTConfig.process_configuration(args)
    (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
    try:
        cursor = fsrc['conn'].cursor(MySQLdb.cursors.DictCursor)
        query = "select TS_ID,TIMESTAMP from TIMESTAMP_INFO where TIMESTAMP >= '"
        query += beginTimestamp.timestr
        query += "' and TIMESTAMP <= '"
        query += endTimestamp.timestr
        query += "'"
        cursor.execute (query)
        rows = cursor.fetchall()
    except MySQLdb.Error, e:
        cursor.close()
        print "Counter: Error %d: %s" % (e.args[0], e.args[1])
        sys.exit(1)
Exemplo n.º 18
0
def doMain(args):
    """
    It looks like it is possible to get an incomplete coverage of the set of time
    steps if you only get rate and brw_stats data for one OST. I should fix this
    in the base modules.
    """
    fsrc = LMTConfig.process_configuration(args)
    (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
    Steps = TimeSteps.TimeSteps()
    Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
    fs = FS.FS(fsrc['name'])
    fs.getInfo(fsrc['conn'])
    fs.setSteps(Steps)
    brwfs = BrwFS.BrwFS(fsrc['name'])
    brwfs.getOSSs(fsrc['conn'])
    iosize_bins = np.array([4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576])
    brwfs.setSteps(Steps)
    brwfs.getBrwStats(fsrc['conn'], stat="BRW_IOSIZE", bins=iosize_bins)
    return(fs, brwfs)
Exemplo n.º 19
0
def do_main(args):
    """
    args.both - Add together the read and write rates for calculation
         config - (file) The lmtrc config file telling how to get to the DB
         cpu - (boolean) Do the CPU utilization calculations (always set this true)
         fs - (string) The dbname entry in the config file for the file system of
                       interest.
         index - (int) The index of the file system of interest in the config file
         oss - (string) The hostname of the OSS to be analyzed.
         read - Caclulate for the read data rate
         seconds - (int) The length of time over which to average the observations.
         threshold - (int) Sound the alarm if the average exceeds this value.
                           default - 40: GiB for rates, or pct. CPU utilization
         verbose - (boolean) Turn on debugging output
         version - (boolean) print the version string and exit
         write - Calculate for the write data rate
    """
    fsrc = LMTConfig.process_configuration(args)
    bulk = Bulk.Bulk(fsrc['dbname'])
    if args.verbose == True:
        bulk.debug()
    bulk.getOSSs(fsrc['conn'])
    now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
    # Now we want to generate the begin/end pair for this day
    end_sie = Timestamp.calc_sie(now)
    # This will be upset by savings time changes
    begin_sie = end_sie - args.seconds
    args.begin = Timestamp.format_timestamp(begin_sie)
    args.end = Timestamp.format_timestamp(end_sie)
    (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
    Steps = TimeSteps.TimeSteps()
    Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
    bulk.setSteps(Steps)
    if (bulk.Steps == None) or (bulk.Steps.steps() == 0):
        print "pyalarm.do_main(): Warning - No steps from FS %s" % bulk.name
        return(None)
    bulk.getData()
    if args.cpu == True:
        bulk.getCPU()
    return(bulk)
Exemplo n.º 20
0
def do_main(args):
    """
    It looks like it is possible to get an incomplet coverage of the set of time
    steps if you only get rate and brw_stats data for one OST. I should fix this
    in the base modules.
    """
    fsrc = LMTConfig.process_configuration(args)
    (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
    Steps = TimeSteps.TimeSteps()
    Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
    thisost = None
    thisbo  = None
    bulk = Bulk.Bulk(fsrc['name'])
    bulk.getOSSs(fsrc['conn'])
    bulk.setSteps(Steps)
    bulk.getData()
    bulk.getCPU()
    brwfs = BrwFS.BrwFS(fsrc['name'])
    brwfs.getOSSs(fsrc['conn'])
    brwfs.getBrwStats(fsrc['conn'], args.stat)
    brwfs.getData(beginTimestamp, endTimestamp, args.stat)
    return(bulk, brwfs)
Exemplo n.º 21
0
def do_main(args):
    """
    args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
                   It will fill in todays year, month and day if left out.
                   hh:mm:ss will default to 00:00:00 as portions are left out.
         both - Add together the read and wtites in the plot
         config - (file) The lmtrc config file telling how to get to the DB
         end - (string) As above giving the end of the data to be gathered.
         fs - (string) The dbname entry in the config file for the file system of interest.
         index - (int) The index of the file system of interest in the config file
         plot - (string) The name of the file to which the graph should be saved.
                  'noplot' is allowed if you just want a report.
         report - (boolean) Print out summary info about the analyzed operations
         read - Plot read data rate
         verbose - (boolean) Turn on debugging output
         version - (boolean) print the version string and exit
         write - Plot the write rate
         ybound - (float) Use the given value as the maximum of the y-acis
    """
    # do_main will:
    # - process_configuration
    # - get all the OSSs which gets their OSTs as well
    # - Process timestamps
    # - get the data
    # - return the osts object
    fsrc = LMTConfig.process_configuration(args)
    bulk = Bulk.Bulk(fsrc['name'])
    if args.verbose == True:
        bulk.debug()
    bulk.getOSSs(fsrc['conn'])
    (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
    Steps = TimeSteps.TimeSteps()
    Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
    bulk.setSteps(Steps)
    if (bulk.Steps is None) or (bulk.Steps.steps() == 0):
        print "osts: Warning - No steps from FS %s" % bulk.name
        sys.exit(1)
    bulk.getData()
    return(bulk)
Exemplo n.º 22
0
def do_main(args):
    """
    args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
                   It will fill in todays year, month and day if left out.
                   hh:mm:ss will default to 00:00:00 as portions are left out.
         config - (file) The lmtrc config file telling how to get to the DB
         cpu - plot CPU utilization
         end - (string) As above giving the end of the data to be gathered.
         fs - (string) The dbname entry in the config file for the file system of interest.
         index - (int) The index of the file system of interest in the config file
         mask - (key=value:keys in {mincpu, maxcpu, minval, maxval}) mask values outside the given range
         plot - (string) The name of the file to which the graph should be saved.
                  'noplot' is allowed if you just want a report.
         report - (boolean) Print out summary info about the analyzed operations
         spectrum - Plot the cross-correlation spectrum
         verbose - (boolean) Turn on debugging output
         version - (boolean) print the version string and exit
         x_correlate - (boolean) plot the ops versus the CPU utilization rather than
                         the ops versus time.
         ybound - (float) Use the given value as the maximum of the y-acis
    """
    fsrc = LMTConfig.process_configuration(args)
    mds = MDS.MDS(host=fsrc['host'],fs=fsrc['name'])
    if args.verbose == True:
        mds.debug()
        mds.debug(module="Operation")
    #mds.getOSSs(fsrc['conn'])
    (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
    Steps = TimeSteps.TimeSteps()
    Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
    mds.setSteps(Steps)
    mds.getQuickData(conn=fsrc['conn'])
    if (mds.Steps == None) or (mds.Steps.steps() == 0):
        print "mds_quick: Warning - No steps from FS %s" % mds.name
        return(None)
    mds.getCPU()
    return(mds)
Exemplo n.º 23
0
def do_main(args):
    """
    args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
                   It will fill in todays year, month and day if left out.
                   hh:mm:ss will default to 00:00:00 as portions are left out.
         config - (file) The lmtrc config file telling how to get to the DB
         end - (string) As above giving the end of the data to be gathered.
         fs - (string) The dbname entry in the config file for the file system of interest.
         index - (int) The index of the file system of interest in the config file
         oss - (string) The hostname of the OSS to be analyzed.
         plot - (string) The name of the file to which the graph should be saved.
                  'noplot' is allowed if you just want a report.
         report - (boolean) Print out summary info about the analyzed operations
         read - Plot read data rate
         stat - Show histograms only for this statistic
         verbose - (boolean) Turn on debugging output
         version - (boolean) print the version string and exit
         write - Plot the write rate
    """
    # do_main will:
    # - process_configuration
    # - get the oss in question
    # - get the OSTs on it
    # - Process timestamps
    # - get the data
    # - return the oss
    fsrc = LMTConfig.process_configuration(args)
    try:
        cursor = fsrc['conn'].cursor(MySQLdb.cursors.DictCursor)
        query = "SELECT * FROM OSS_INFO"
        cursor.execute (query)
        rows = cursor.fetchall()
    except MySQLdb.Error, e:
        cursor.close()
        print "test_BrwOSS: Error %d: %s" % (e.args[0], e.args[1])
        return
Exemplo n.º 24
0
            print "MDS: Error - No ops have been specified"
        else:
            args.mds_op = [args.hilite_op]
    if (args.x_correlate == True) and ((len(args.mds_op) != 1) or (args.mds_op[0] == "cpu")):
        print "MDS: If you want a cross correlation (-x) you need to provide exactly one other operation (not -m cpu)"
        sys.exit()
    if args.x_correlate == True:
        args.mds_op += ["cpu"]
    return(args)

#*******************************************************************************

print "This code is out of date. Revise before using."
sys.exit(1)
args = process_args()
fsrc = LMTConfig.process_configuration(args)
mds = MDS.MDS(host=fsrc['host'], fs=fsrc['name'])
if args.verbose == True:
    mds.debug()
    mds.debug(module="Operation")
    #mds.debug(module="Timestamp")
mds.opsFromDB(fsrc['conn'])
(beginTimestamp, endTimestamp) = Timestamp.day(args, fsrc)
Steps = TimeSteps.TimeSteps()
Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
mds.setSteps(Steps)
mds.getData()
if mds.haveData == False:
    print "got no data from %s to %s" % (args.begin, args.end)
    sys.exit(0)
if args.report == True:
Exemplo n.º 25
0
def do_main(args):
    """
    args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
                   It will fill in todays year, month and day if left out.
                   hh:mm:ss will default to 00:00:00 as portions are left out.
         config - (file) The lmtrc config file telling how to get to the DB
         end - (string) As above giving the end of the data to be gathered.
         fs - (string) The dbname entry in the config file for the file system of interest.
         index - (int) The index of the file system of interest in the config file
         plot - (string) The name of the file to which the graph should be saved.
                  'noplot' is allowed if you just want a report.
         progress - Print the name of each OST as you work through the list
         report - (boolean) Print out summary info about the analyzed operations
         read - Plot read data rate
         stat - Show histograms only for this statistic
         verbose - (boolean) Turn on debugging output
         version - (boolean) print the version string and exit
         write - Plot the write rate
    """
    # do_main will:
    # - process_configuration
    # - get the oss in question
    # - get the OSTs on it
    # - Process timestamps
    # - get the data
    # - return the oss
    bins = None
    if args.stat == "BRW_IOSIZE":
        bins = np.array([4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576])
    fsrc = LMTConfig.process_configuration(args)
    sie = Timestamp.calc_sie(args.begin)
    beginTimestamp = Timestamp.Timestamp(sie)
    endTimestamp   = Timestamp.Timestamp(sie + 5)
    beginTimestamp.no_later_than(fsrc['conn'])
    endTimestamp.no_earlier_than(fsrc['conn'])
    if beginTimestamp.ts_id >= endTimestamp.ts_id:
        print "observation: failed to get time stamp range for %s" % args.time
        exit(1)
    brwBegin = BrwFS.BrwFS(fsrc['dbname'])
    brwBegin.getOSSs(fsrc['conn'])
    if args.verbose == True:
        brwBegin.debug()
        brwBegin.debug(module="BrwFS")
        #oss.debug(module="Timestamp")
    Steps = TimeSteps.TimeSteps()
    Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
    brwBegin.setSteps(Steps)
    brwBegin.getBrwStats(conn=fsrc['conn'], stat=args.stat, bins=bins)
    brwBegin.getDataSlice(stat=args.stat)
    if (brwBegin.Bins is None) or (len(brwBegin.Bins) == 0):
        print "test_BrwFS: Warning - No HistBins objects from OSS %s" % oss.name
        return
    if (brwBegin.Steps is None) or (brwBegin.Steps.steps() == 0):
        print "test_BrwFS: Warning - No steps from OSS %s" % oss.name
        return
    sie = Timestamp.calc_sie(args.end)
    beginTimestamp = Timestamp.Timestamp(sie)
    endTimestamp   = Timestamp.Timestamp(sie + 5)
    beginTimestamp.no_later_than(fsrc['conn'])
    endTimestamp.no_earlier_than(fsrc['conn'])
    if beginTimestamp.ts_id >= endTimestamp.ts_id:
        print "observation: failed to get time stamp range for %s" % args.time
        return(None)
    brwEnd = BrwFS.BrwFS(fsrc['dbname'])
    brwEnd.getOSSs(fsrc['conn'])
    if args.verbose == True:
        brwEnd.debug()
        brwEnd.debug(module="BrwFS")
        #oss.debug(module="Timestamp")
    Steps = TimeSteps.TimeSteps()
    Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
    brwEnd.setSteps(Steps)
    brwEnd.getBrwStats(conn=fsrc['conn'], stat=args.stat, bins=bins)
    brwEnd.getDataSlice(stat=args.stat)
    if (brwEnd.Bins is None) or (len(brwEnd.Bins) == 0):
        print "test_BrwFS: Warning - No HistBins objects from OSS %s" % oss.name
        return
    if (brwEnd.Steps is None) or (brwEnd.Steps.steps() == 0):
        print "test_BrwFS: Warning - No steps from OSS %s" % oss.name
        return
    return(brwBegin, brwEnd)