Пример #1
0
 def getDst(self):
     if self.dst != None:
         return(self.dst)
     if (self.begin == None) or (self.end == None):
         return(None)
     self.dst = Timestamp.dst(self.begin)
     if (self.dst != Timestamp.dst(self.end.sie)) and (self.Debug == True):
         self.DebugMessages += "Timestamp.dst(): Warning - The interval spans a change to or from daylight savings time."
     return(self.dst)
Пример #2
0
def do_main(args):
    '''
    '''
    fsrc = LMTConfig.process_configuration(args)
    (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
    if args.mds == True:
        query = "SELECT TIMESTAMP_INFO.TS_ID,TIMESTAMP,PCT_CPU FROM TIMESTAMP_INFO,MDS_DATA WHERE "
        query += "TIMESTAMP_INFO.TS_ID=MDS_DATA.TS_ID AND "
    else:
        query = "SELECT TIMESTAMP_INFO.TS_ID,TIMESTAMP,PCT_CPU FROM TIMESTAMP_INFO,OSS_DATA,OSS_INFO WHERE "
        query += "OSS_DATA.OSS_ID=OSS_INFO.OSS_ID and OSS_INFO.HOSTNAME='"
        query += args.oss + "' AND "
        query += "TIMESTAMP_INFO.TS_ID=OSS_DATA.TS_ID AND "
    query += "TIMESTAMP_INFO.TIMESTAMP >= '"
    query += beginTimestamp.timestr
    query += "' AND TIMESTAMP_INFO.TIMESTAMP <= '"
    query += endTimestamp.timestr
    query += "'"
    try:
        cursor = fsrc['conn'].cursor(MySQLdb.cursors.DictCursor)
        if args.verbose == True:
            print "\t%s" % query
        cursor.execute (query)
    except MySQLdb.Error, e:
        cursor.close()
        print "CPU.get_ops_data_from_db: Error %d: %s\n%s" % (e.args[0], e.args[1], query)
        return(None)
Пример #3
0
def do_main(args):
    """
    It looks like it is possible to get an incomplet coverage of the set of time
    steps if you only get rate and brw_stats data for one OST. I should fix this
    in the base modules.
    """
    fsrc = LMTConfig.process_configuration(args)
    sie = Timestamp.calc_sie(args.time)
    beginTimestamp = Timestamp.Timestamp(sie - 5)
    endTimestamp   = Timestamp.Timestamp(sie)
    beginTimestamp.no_later_than(fsrc['conn'])
    endTimestamp.no_earlier_than(fsrc['conn'])
    if beginTimestamp.ts_id >= endTimestamp.ts_id:
        print "observation: failed to get time stamp range for %s" % args.time
        exit(1)
    Steps = TimeSteps.TimeSteps()
    Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
    bulk = Bulk.Bulk(fsrc['name'])
    bulk.getOSSs(fsrc['conn'])
    bulk.setSteps(Steps)
    bulk.getData()
    # FIXME
    # I am not handling masked values in the CPU data
    # Should be an easy fix
    bulk.getCPU()
    brwfs = BrwFS.BrwFS(fsrc['name'])
    brwfs.getOSSs(fsrc['conn'])
    # we'll query for all of them even when we only want one
    # because it make the processing more uniform
    brwfs.getBrwStats(fsrc['conn'], stat=args.stat)
    brwfs.getData(beginTimestamp, endTimestamp, stat=args.stat)
    return(bulk, brwfs)
Пример #4
0
def do_main(args):
    """
    args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
                   It will fill in todays year, month and day if left out.
                   hh:mm:ss will default to 00:00:00 as portions are left out.
         config - (file) The lmtrc config file telling how to get to the DB
         end - (string) As above giving the end of the data to be gathered.
         fs - (string) The dbname entry in the config file for the file system of interest.
         index - (int) The index of the file system of interest in the config file
         verbose - (boolean) Turn on debugging output
         version - (boolean) print the version string and exit
    """
    fsrc = LMTConfig.process_configuration(args)
    mds = MDS.MDS(host=fsrc["host"], fs=fsrc["name"])
    if args.verbose == True:
        mds.debug()
        mds.debug(module="Operation")
        # mds.debug(module="Timestamp")
    mds.opsFromDB(fsrc["conn"])
    (begin_ts, end_ts) = Timestamp.process_timestamps(args, fsrc)
    mds.getData(begin=begin_ts, end=end_ts)
    if mds.haveData == False:
        print "got no data from %s to %s" % (args.begin, args.end)
        return None
    mds.getCPU()
    return mds
Пример #5
0
def do_main(args):
    """
    args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
                   It will fill in todays year, month and day if left out.
                   hh:mm:ss will default to 00:00:00 as portions are left out.
         both - Add together the read and write rates for calculation
         config - (file) The lmtrc config file telling how to get to the DB
         cpu - (boolean) Do the CPU utilization calculations (always set this true)
         end - (string) As above giving the end of the data to be gathered.
         fs - (string) The dbname entry in the config file for the file system of interest.
         index - (int) The index of the file system of interest in the config file
         oss - (string) The hostname of the OSS to be analyzed.
         ost - (string) The name of the OST to be analyzed.
         read - Caclulate for the read data rate
         verbose - (boolean) Turn on debugging output
         version - (boolean) print the version string and exit
         write - Calculate for the write data rate
    """
    fsrc = LMTConfig.process_configuration(args)
    bulk = Bulk.Bulk(fsrc['dbname'])
    if args.verbose == True:
        bulk.debug()
    bulk.getOSSs(fsrc['conn'])
    (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
    Steps = TimeSteps.TimeSteps()
    Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
    bulk.setSteps(Steps)
    if (bulk.Steps == None) or (bulk.Steps.steps() == 0):
        print "data_moved.do_main(): Warning - No steps from FS %s" % bulk.name
        sys.exit(1)
    bulk.getData()
    if args.cpu == True:
        bulk.getCPU()
    return(bulk)
Пример #6
0
def do_main(args):
    """
    args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
                   It will fill in todays year, month and day if left out.
                   hh:mm:ss will default to 00:00:00 as portions are left out.
         config - (file) The lmtrc config file telling how to get to the DB
         end - (string) As above giving the end of the data to be gathered.
         fs - (string) The dbname entry in the config file for the file system of interest.
         index - (int) The index of the file system of interest in the config file
         verbose - (boolean) Turn on debugging output
         version - (boolean) print the version string and exit
    """
    # do_main will:
    # - process_configuration
    # - get all the OSSs which gets their OSTs as well
    # - Process timestamps
    # - get the data including CPU utilization if asked
    # - return the bulk_aggregate object
    fsrc = LMTConfig.process_configuration(args)
    bulk = Bulk.Bulk(fsrc['name'])
    if args.verbose == True:
        bulk.debug()
    bulk.getOSSs(fsrc['conn'])
    (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
    Steps = TimeSteps.TimeSteps()
    Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
    bulk.setSteps(Steps)
    if (bulk.Steps == None) or (bulk.Steps.steps() == 0):
        print "bulk_aggregate: Warning - No steps from FS %s" % bulk.name
        return(None)
    bulk.getData()
    bulk.getCPU()
    return(bulk)
Пример #7
0
def do_main(args):
    """
    It looks like it is possible to get an incomplet coverage of the set of time
    steps if you only get rate and brw_stats data for one OST. I should fix this
    in the base modules.
    """
    fsrc = LMTConfig.process_configuration(args)
    (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
    bulk = Bulk.Bulk(fsrc['name'])
    bulk.getOSSs(fsrc['conn'])
    Steps = TimeSteps.TimeSteps()
    Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
    bulk.setSteps(Steps)
    bulk.getData()
    bulk.getCPU()
    brwfs = BrwFS.BrwFS(fsrc['name'])
    brwfs.getOSSs(fsrc['conn'])
    brwfs.setSteps(Steps)
    if args.stat == 'BRW_IOSIZE':
        iosize_bins = np.array([4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576])
        brwfs.getBrwStats(fsrc['conn'], args.stat, bins=iosize_bins)
    else:
        brwfs.getBrwStats(fsrc['conn'], args.stat)
    brwfs.getData(args.stat)
    return(bulk, brwfs)
Пример #8
0
def do_main(args):
    """
    args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
                   It will fill in todays year, month and day if left out.
                   hh:mm:ss will default to 00:00:00 as portions are left out.
         end - (string) As above giving the end of the data to be gathered.
         file - (string) The h5lmt file to query
         plot - (string) The name of the file to which the graph should be saved.
                  'noplot' is allowed if you just want a report.
         verbose - (boolean) Turn on debugging output
         version - (boolean) print the version string and exit
                         the ops versus time.
    """
    fsFile = h5py.File(args.file, 'r')
    b_sie = Timestamp.calc_sie(args.begin)
    e_sie = Timestamp.calc_sie(args.end)
    return(b_sie, e_sie, fsFile)
Пример #9
0
def doMain(args):
    fsFile = h5py.File(args.file, 'r')
    fsStepsGroup = fsFile['FSStepsGroup']
    fsStepsDataSet = fsStepsGroup['FSStepsDataSet']
    day=fsStepsDataSet.attrs['day']
    if args.begin is None:
        begin_str = "%s 00:00:00" % day
    else:
        begin_str = "%s %s" % (day, args.begin)
    if args.end is None:
        midnight_before = datetime.datetime.strptime(day, "%Y-%m-%d")
        midnight_after = midnight_before + datetime.timedelta(days=1)
        end_str = midnight_after.strftime("%Y-%m-%d 00:00:00")
    else:
        end_str = "%s %s" % (day, args.end)
    b_sie = Timestamp.calc_sie(begin_str)
    e_sie = Timestamp.calc_sie(end_str)
    return(b_sie, e_sie, fsFile)
Пример #10
0
def host_interval(host, begin, end):
    begin_str = Timestamp.format_timestamp(int(begin))
    if begin_str is None:
        return(bottle.template('bad_value', host=host, filesys='n/a', begin=begin, end=end, data='n/a', value='begin'))
    end_str = Timestamp.format_timestamp(int(end))
    if end_str is None:
        return(bottle.template('bad_value', host=host, filesys='n/a', begin=begin, end=end, data='n/a', value='end'))
    bulk_scratch = '/pylmt/%s/scratch/%s/%s/bulk' % (host, begin, end)
    bulk_scratch2 = '/pylmt/%s/scratch2/%s/%s/bulk' % (host, begin, end)
    metadata_scratch = '/pylmt/%s/scratch/%s/%s/metadata' % (host, begin, end)
    metadata_scratch2 = '/pylmt/%s/scratch2/%s/%s/metadata' % (host, begin, end)
    return(bottle.template('host_interval',
                    host=host,
                    begin=begin_str,
                    end=end_str,
                    bulk_scratch=bulk_scratch,
                    bulk_scratch2=bulk_scratch2,
                    metadata_scratch=metadata_scratch,
                    metadata_scratch2=metadata_scratch2))
Пример #11
0
def get_bulk(args):
    fsrc = get_fsrc(args)
    if type(fsrc) is str:
        return(fsrc)
    bulk = Bulk.Bulk(fsrc['name'])
    bulk.getOSSs(fsrc['conn'])
    (begin_ts, end_ts) = Timestamp.process_timestamps(args, fsrc)
    bulk.getQuickData(begin_ts,
                         end_ts,
                         conn=fsrc['conn'])
    return(bulk)
Пример #12
0
def do_main(args):
    """
    args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
                   It will fill in todays year, month and day if left out.
                   hh:mm:ss will default to 00:00:00 as portions are left out.
         config - (file) The lmtrc config file telling how to get to the DB
         cpu - (boolean) Get CPU utilization data and potentially plot it
         end - (string) As above giving the end of the data to be gathered.
         FS - (string) The dbname entry in the config file for the file system of interest.
         index - (int) The index of the file system of interest in the config file
         metadata - (boolean) Do metadata rather than bulk I/O data
         plot - (string) The name of the file to which the graph should be saved.
                  'noplot' is allowed if you just want a report.
         report - (boolean) Print out summary info about the analyzed operations
         verbose - (boolean) Turn on debugging output
         version - (boolean) print the version string and exit
         x_correlate - (boolean) plot the ops versus the CPU utilization rather than
                         the ops versus time.
         ybound - (float) Use the given value as the maximum of the y-acis

    You can work with a data set interactively by entering ipython at the command line
    and carrying out this sequence of preparatory steps:
from pyLMT import FS
from pyLMT.tests import test_FS
args = FS.process_args()
args.begin = '2012-01-12'
args.end = '2012-01-13'
args.cpu = True
args.FS = 'filesystem_scratch'
fs = test_FS.do_main(args)

    It may take as much as half an hour to load up 24 hours worth of data,
    but once you have it in hand you can explore the data set interactively
    without having to go back and query the LMT DB over and over.
    """
    # do_main will:
    # - process_configuration
    # - get the OSSs and metadata ops
    # - Process timestamps
    # - get the bulk data
    # - get the metadata
    # - return the FS object
    fsrc = LMTConfig.process_configuration(args)
    fs = FS.FS(fsrc['name'])
    if args.verbose == True:
        fs.debug()
    fs.getInfo(fsrc['conn'])
    (begin_ts, end_ts) = Timestamp.process_timestamps(args, fsrc)
    fs.getData(begin_ts,
               end_ts)
    if (args.cpu == True) or (args.x_correlate == True):
        fs.getCPU()
    return(fs)
Пример #13
0
def get_metadata(args):
    fsrc = get_fsrc(args)
    if fsrc is None:
        return(None)
    metadata = MDS.MDS(host=fsrc['host'], fs=fsrc['name'])
    metadata.opsFromDB(fsrc['conn'])
    (begin_ts, end_ts) = Timestamp.process_timestamps(args, fsrc)
    metadata.getQuickData(begin_ts,
                         end_ts,
                         conn=fsrc['conn'])
    metadata.getCPU()
    return(metadata)
Пример #14
0
def do_main(args):
    """
    args.both - Add together the read and write rates for calculation
         config - (file) The lmtrc config file telling how to get to the DB
         cpu - (boolean) Do the CPU utilization calculations (always set this true)
         fs - (string) The dbname entry in the config file for the file system of
                       interest.
         index - (int) The index of the file system of interest in the config file
         oss - (string) The hostname of the OSS to be analyzed.
         read - Caclulate for the read data rate
         seconds - (int) The length of time over which to average the observations.
         threshold - (int) Sound the alarm if the average exceeds this value.
                           default - 40: GiB for rates, or pct. CPU utilization
         verbose - (boolean) Turn on debugging output
         version - (boolean) print the version string and exit
         write - Calculate for the write data rate
    """
    fsrc = LMTConfig.process_configuration(args)
    bulk = Bulk.Bulk(fsrc['dbname'])
    if args.verbose == True:
        bulk.debug()
    bulk.getOSSs(fsrc['conn'])
    now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
    # Now we want to generate the begin/end pair for this day
    end_sie = Timestamp.calc_sie(now)
    # This will be upset by savings time changes
    begin_sie = end_sie - args.seconds
    args.begin = Timestamp.format_timestamp(begin_sie)
    args.end = Timestamp.format_timestamp(end_sie)
    (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
    Steps = TimeSteps.TimeSteps()
    Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
    bulk.setSteps(Steps)
    if (bulk.Steps == None) or (bulk.Steps.steps() == 0):
        print "pyalarm.do_main(): Warning - No steps from FS %s" % bulk.name
        return(None)
    bulk.getData()
    if args.cpu == True:
        bulk.getCPU()
    return(bulk)
Пример #15
0
def do_main(args):
    """
    args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
                   It will fill in todays year, month and day if left out.
                   hh:mm:ss will default to 00:00:00 as portions are left out.
         both - Add together the read and wtites in the plot
         config - (file) The lmtrc config file telling how to get to the DB
         cpu - plot CPU utilization
         end - (string) As above giving the end of the data to be gathered.
         fs - (string) The dbname entry in the config file for the file system of interest.
         index - (int) The index of the file system of interest in the config file
         lines - (boolean) Graph using lines (no matter how many data points)
         mask - (key=value:keys in {mincpu, maxcpu, minval, maxval}) mask values outside the given range
         plot - (string) The name of the file to which the graph should be saved.
                  'noplot' is allowed if you just want a report.
         plotSdevs (boolean) Calculate and plot the standard deviation across OSTs at each timestep
         report - (boolean) Print out summary info about the analyzed operations
         read - Plot read data rate
         spectrum - Plot the cross-correlation spectrum
         frac - Filter out samples of the spectrum below this fraction of the max
         verbose - (boolean) Turn on debugging output
         version - (boolean) print the version string and exit
         write - Plot the write rate
         x_correlate - (boolean) plot the ops versus the CPU utilization rather than
                         the ops versus time.
         ybound - (float) Use the given value as the maximum of the y-acis
    """
    # do_main will:
    # - process_configuration
    # - get all the OSSs which gets their OSTs as well
    # - Process timestamps
    # - get the data including CPU utilization if asked
    # - return the bulk object
    fsrc = LMTConfig.process_configuration(args)
    if fsrc is None:
        return None
    B = Bulk.Bulk(fsrc['name'])
    if args.verbose == True:
        B.debug()
    B.getOSSs(fsrc['conn'])
    (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
    Steps = TimeSteps.TimeSteps()
    Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
    B.setSteps(Steps)
    if (B.Steps is None) or (B.Steps.steps() == 0):
        print "bulk: Warning - No steps from FS %s" % B.name
        sys.exit(1)
    B.getData()
    if (args.cpu == True) or (args.x_correlate == True):
        B.getCPU()
    return(B)
Пример #16
0
def do_main(args):
    """
    args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
                   It will fill in todays year, month and day if left out.
                   hh:mm:ss will default to 00:00:00 as portions are left out.
         config - (file) The lmtrc config file telling how to get to the DB
         cpu - plot CPU utilization
         end - (string) As above giving the end of the data to be gathered.
         extra - (int) If None then ignore, otherwise pad the graph before and after
                 after the interval with this much extra, but don't inlcude in
                 average calculation.
         fs - (string) The dbname entry in the config file for the file system of interest.
         index - (int) The index of the file system of interest in the config file
         mask - (key=value:keys in {mincpu, maxcpu, minval, maxval}) mask values outside the given range
         plot - (string) The name of the file to which the graph should be saved.
                  'noplot' is allowed if you just want a report.
         report - (boolean) Print out summary info about the analyzed operations
         show_ops - (boolean) List the available operations
         show_steps - (boolean) List the steps that are in the -M <mask> region
         spectrum - (boolean)  Produce a graph of the Ops rate versus CPU utilization spectrum
         verbose - (boolean) Turn on debugging output
         version - (boolean) print the version string and exit
         x_correlate - (boolean) plot the ops versus the CPU utilization rather than
                         the ops versus time.
         ybound - (float) Use the given value as the maximum of the y-acis
    """
    fsrc = LMTConfig.process_configuration(args)
    mds = MDS.MDS(host=fsrc['host'], fs=fsrc['name'])
    if args.verbose == True:
        mds.debug()
        mds.debug(module="Operation")
        #mds.debug(module="Timestamp")
    mds.opsFromDB(fsrc['conn'])
    if args.show_ops == True:
        mds.showOps()
        return
    (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
    Steps = TimeSteps.TimeSteps()
    Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
    mds.setSteps(Steps)
    mds.getData()
    if mds.haveData == False:
        print "got no data from %s to %s" % (args.begin, args.end)
        return(None)
    if (args.cpu == True) or (args.x_correlate == True):
        mds.getCPU()
    if not args.extra is None:
        if (args.extra <= 0) or (args.extra >= 100):
            print "mds: extra should be an integer between 0 and 100"
            args.extra = None
    return(mds)
Пример #17
0
def do_main(args):
    """
    """
    fsrc = LMTConfig.process_configuration(args)
    (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
    try:
        cursor = fsrc['conn'].cursor(MySQLdb.cursors.DictCursor)
        query = "SELECT * FROM OST_INFO"
        cursor.execute (query)
        rows = cursor.fetchall()
    except MySQLdb.Error, e:
        cursor.close()
        print "ost: Error %d: %s" % (e.args[0], e.args[1])
        sys.exit(1)
Пример #18
0
def do_main(args):
    """
    args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
                   It will fill in todays year, month and day if left out.
                   hh:mm:ss will default to 00:00:00 as portions are left out.
         config - (file) The lmtrc config file telling how to get to the DB
         end - (string) As above giving the end of the data to be gathered.
         fs - (string) The dbname entry in the config file for the file system of interest.
         index - (int) The index of the file system of interest in the config file
         plot - (string) The name of the file to which the graph should be saved.
                  'noplot' is allowed if you just want a report.
         report - (boolean) Print out summary info about the analyzed operations
         read - Plot read data rate
         stat - Show histograms only for this statistic
         verbose - (boolean) Turn on debugging output
         version - (boolean) print the version string and exit
         write - Plot the write rate
    """
    # do_main will:
    # - process_configuration
    # - get the oss in question
    # - get the OSTs on it
    # - Process timestamps
    # - get the data
    # - return the oss
    fsrc = LMTConfig.process_configuration(args)
    brwfs = BrwFS.BrwFS(fsrc['dbname'])
    brwfs.getOSSs(fsrc['conn'])
    if args.verbose == True:
        brwfs.debug()
        brwfs.debug(module="BrwFS")
        #oss.debug(module="Timestamp")
    (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
    Steps = TimeSteps.TimeSteps()
    Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
    brwfs.setSteps(Steps)
    brwfs.getBrwStats(conn=fsrc['conn'], stat=args.stat)
    brwfs.getData(stat=args.stat)
    if (brwfs.Bins is None) or (len(brwfs.Bins) == 0):
        print "test_BrwFS: Warning - No HistBins objects from OSS %s" % oss.name
        return
    if (brwfs.Steps is None) or (brwfs.Steps.steps() == 0):
        print "test_BrwFS: Warning - No steps from OSS %s" % oss.name
        return
    return(brwfs)
Пример #19
0
def do_main(args):
    '''
    '''
    fsrc = LMTConfig.process_configuration(args)
    (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
    try:
        cursor = fsrc['conn'].cursor(MySQLdb.cursors.DictCursor)
        query = "select TS_ID,TIMESTAMP from TIMESTAMP_INFO where TIMESTAMP >= '"
        query += beginTimestamp.timestr
        query += "' and TIMESTAMP <= '"
        query += endTimestamp.timestr
        query += "'"
        cursor.execute (query)
        rows = cursor.fetchall()
    except MySQLdb.Error, e:
        cursor.close()
        print "Counter: Error %d: %s" % (e.args[0], e.args[1])
        sys.exit(1)
Пример #20
0
def doMain(args):
    """
    It looks like it is possible to get an incomplete coverage of the set of time
    steps if you only get rate and brw_stats data for one OST. I should fix this
    in the base modules.
    """
    fsrc = LMTConfig.process_configuration(args)
    (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
    Steps = TimeSteps.TimeSteps()
    Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
    fs = FS.FS(fsrc['name'])
    fs.getInfo(fsrc['conn'])
    fs.setSteps(Steps)
    brwfs = BrwFS.BrwFS(fsrc['name'])
    brwfs.getOSSs(fsrc['conn'])
    iosize_bins = np.array([4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576])
    brwfs.setSteps(Steps)
    brwfs.getBrwStats(fsrc['conn'], stat="BRW_IOSIZE", bins=iosize_bins)
    return(fs, brwfs)
Пример #21
0
def do_main(args):
    """
    It looks like it is possible to get an incomplet coverage of the set of time
    steps if you only get rate and brw_stats data for one OST. I should fix this
    in the base modules.
    """
    fsrc = LMTConfig.process_configuration(args)
    (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
    Steps = TimeSteps.TimeSteps()
    Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
    thisost = None
    thisbo  = None
    bulk = Bulk.Bulk(fsrc['name'])
    bulk.getOSSs(fsrc['conn'])
    bulk.setSteps(Steps)
    bulk.getData()
    bulk.getCPU()
    brwfs = BrwFS.BrwFS(fsrc['name'])
    brwfs.getOSSs(fsrc['conn'])
    brwfs.getBrwStats(fsrc['conn'], args.stat)
    brwfs.getData(beginTimestamp, endTimestamp, args.stat)
    return(bulk, brwfs)
Пример #22
0
def do_main(args):
    """
    args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
                   It will fill in todays year, month and day if left out.
                   hh:mm:ss will default to 00:00:00 as portions are left out.
         both - Add together the read and wtites in the plot
         config - (file) The lmtrc config file telling how to get to the DB
         end - (string) As above giving the end of the data to be gathered.
         fs - (string) The dbname entry in the config file for the file system of interest.
         index - (int) The index of the file system of interest in the config file
         plot - (string) The name of the file to which the graph should be saved.
                  'noplot' is allowed if you just want a report.
         report - (boolean) Print out summary info about the analyzed operations
         read - Plot read data rate
         verbose - (boolean) Turn on debugging output
         version - (boolean) print the version string and exit
         write - Plot the write rate
         ybound - (float) Use the given value as the maximum of the y-acis
    """
    # do_main will:
    # - process_configuration
    # - get all the OSSs which gets their OSTs as well
    # - Process timestamps
    # - get the data
    # - return the osts object
    fsrc = LMTConfig.process_configuration(args)
    bulk = Bulk.Bulk(fsrc['name'])
    if args.verbose == True:
        bulk.debug()
    bulk.getOSSs(fsrc['conn'])
    (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
    Steps = TimeSteps.TimeSteps()
    Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
    bulk.setSteps(Steps)
    if (bulk.Steps is None) or (bulk.Steps.steps() == 0):
        print "osts: Warning - No steps from FS %s" % bulk.name
        sys.exit(1)
    bulk.getData()
    return(bulk)
Пример #23
0
def do_main(args):
    """
    args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
                   It will fill in todays year, month and day if left out.
                   hh:mm:ss will default to 00:00:00 as portions are left out.
         config - (file) The lmtrc config file telling how to get to the DB
         cpu - plot CPU utilization
         end - (string) As above giving the end of the data to be gathered.
         fs - (string) The dbname entry in the config file for the file system of interest.
         index - (int) The index of the file system of interest in the config file
         mask - (key=value:keys in {mincpu, maxcpu, minval, maxval}) mask values outside the given range
         plot - (string) The name of the file to which the graph should be saved.
                  'noplot' is allowed if you just want a report.
         report - (boolean) Print out summary info about the analyzed operations
         spectrum - Plot the cross-correlation spectrum
         verbose - (boolean) Turn on debugging output
         version - (boolean) print the version string and exit
         x_correlate - (boolean) plot the ops versus the CPU utilization rather than
                         the ops versus time.
         ybound - (float) Use the given value as the maximum of the y-acis
    """
    fsrc = LMTConfig.process_configuration(args)
    mds = MDS.MDS(host=fsrc['host'],fs=fsrc['name'])
    if args.verbose == True:
        mds.debug()
        mds.debug(module="Operation")
    #mds.getOSSs(fsrc['conn'])
    (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
    Steps = TimeSteps.TimeSteps()
    Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
    mds.setSteps(Steps)
    mds.getQuickData(conn=fsrc['conn'])
    if (mds.Steps == None) or (mds.Steps.steps() == 0):
        print "mds_quick: Warning - No steps from FS %s" % mds.name
        return(None)
    mds.getCPU()
    return(mds)
Пример #24
0
def do_main(args):
    """
    args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
                   It will fill in todays year, month and day if left out.
                   hh:mm:ss will default to 00:00:00 as portions are left out.
         config - (file) The lmtrc config file telling how to get to the DB
         end - (string) As above giving the end of the data to be gathered.
         fs - (string) The dbname entry in the config file for the file system of interest.
         index - (int) The index of the file system of interest in the config file
         plot - (string) The name of the file to which the graph should be saved.
                  'noplot' is allowed if you just want a report.
         progress - Print the name of each OST as you work through the list
         report - (boolean) Print out summary info about the analyzed operations
         read - Plot read data rate
         stat - Show histograms only for this statistic
         verbose - (boolean) Turn on debugging output
         version - (boolean) print the version string and exit
         write - Plot the write rate
    """
    # do_main will:
    # - process_configuration
    # - get the oss in question
    # - get the OSTs on it
    # - Process timestamps
    # - get the data
    # - return the oss
    bins = None
    if args.stat == "BRW_IOSIZE":
        bins = np.array([4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576])
    fsrc = LMTConfig.process_configuration(args)
    sie = Timestamp.calc_sie(args.begin)
    beginTimestamp = Timestamp.Timestamp(sie)
    endTimestamp   = Timestamp.Timestamp(sie + 5)
    beginTimestamp.no_later_than(fsrc['conn'])
    endTimestamp.no_earlier_than(fsrc['conn'])
    if beginTimestamp.ts_id >= endTimestamp.ts_id:
        print "observation: failed to get time stamp range for %s" % args.time
        exit(1)
    brwBegin = BrwFS.BrwFS(fsrc['dbname'])
    brwBegin.getOSSs(fsrc['conn'])
    if args.verbose == True:
        brwBegin.debug()
        brwBegin.debug(module="BrwFS")
        #oss.debug(module="Timestamp")
    Steps = TimeSteps.TimeSteps()
    Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
    brwBegin.setSteps(Steps)
    brwBegin.getBrwStats(conn=fsrc['conn'], stat=args.stat, bins=bins)
    brwBegin.getDataSlice(stat=args.stat)
    if (brwBegin.Bins is None) or (len(brwBegin.Bins) == 0):
        print "test_BrwFS: Warning - No HistBins objects from OSS %s" % oss.name
        return
    if (brwBegin.Steps is None) or (brwBegin.Steps.steps() == 0):
        print "test_BrwFS: Warning - No steps from OSS %s" % oss.name
        return
    sie = Timestamp.calc_sie(args.end)
    beginTimestamp = Timestamp.Timestamp(sie)
    endTimestamp   = Timestamp.Timestamp(sie + 5)
    beginTimestamp.no_later_than(fsrc['conn'])
    endTimestamp.no_earlier_than(fsrc['conn'])
    if beginTimestamp.ts_id >= endTimestamp.ts_id:
        print "observation: failed to get time stamp range for %s" % args.time
        return(None)
    brwEnd = BrwFS.BrwFS(fsrc['dbname'])
    brwEnd.getOSSs(fsrc['conn'])
    if args.verbose == True:
        brwEnd.debug()
        brwEnd.debug(module="BrwFS")
        #oss.debug(module="Timestamp")
    Steps = TimeSteps.TimeSteps()
    Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
    brwEnd.setSteps(Steps)
    brwEnd.getBrwStats(conn=fsrc['conn'], stat=args.stat, bins=bins)
    brwEnd.getDataSlice(stat=args.stat)
    if (brwEnd.Bins is None) or (len(brwEnd.Bins) == 0):
        print "test_BrwFS: Warning - No HistBins objects from OSS %s" % oss.name
        return
    if (brwEnd.Steps is None) or (brwEnd.Steps.steps() == 0):
        print "test_BrwFS: Warning - No steps from OSS %s" % oss.name
        return
    return(brwBegin, brwEnd)
Пример #25
0
     cursor = fsrc['conn'].cursor(MySQLdb.cursors.DictCursor)
     query = "select TS_ID,TIMESTAMP from TIMESTAMP_INFO where TIMESTAMP >= '"
     query += beginTimestamp.timestr
     query += "' and TIMESTAMP <= '"
     query += endTimestamp.timestr
     query += "'"
     cursor.execute (query)
     rows = cursor.fetchall()
 except MySQLdb.Error, e:
     cursor.close()
     print "Counter: Error %d: %s" % (e.args[0], e.args[1])
     sys.exit(1)
 Series = Counter.Counter("random", "none")
 Steps = TimeSteps.TimeSteps()
 for row in rows:
     Steps.examine(row['TIMESTAMP'], Timestamp.calc_sie(row['TIMESTAMP']), row['TS_ID'])
 cursor.close()
 Steps.register()
 Series.setSteps(Steps)
 count = Steps.steps()
 data = np.random.random_sample(count)
 counter = 0
 for i in range(count):
     try:
         sie = Steps.getSie(index=i)
     except:
         print "Counter: Out of range index"
         sys.exit(1)
     counter += data[i]
     if data[i] > 0.97:
         counter = 0
Пример #26
0
        args.mds_op += ["cpu"]
    return(args)

#*******************************************************************************

print "This code is out of date. Revise before using."
sys.exit(1)
args = process_args()
fsrc = LMTConfig.process_configuration(args)
mds = MDS.MDS(host=fsrc['host'], fs=fsrc['name'])
if args.verbose == True:
    mds.debug()
    mds.debug(module="Operation")
    #mds.debug(module="Timestamp")
mds.opsFromDB(fsrc['conn'])
(beginTimestamp, endTimestamp) = Timestamp.day(args, fsrc)
Steps = TimeSteps.TimeSteps()
Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
mds.setSteps(Steps)
mds.getData()
if mds.haveData == False:
    print "got no data from %s to %s" % (args.begin, args.end)
    sys.exit(0)
if args.report == True:
    mds.report()
#print "%d steps" % mds.Steps.steps()
if args.x_correlate == True:
    if args.hilite_op == None:
        args.hilite_op = args.mds_op[0]
    if (args.plot == None) or (args.plot == "noplot"):
        MDS.do_xcorr(mds, plot=None, ybound=args.ybound, hilite=args.hilite_op)
Пример #27
0
def doMain(args):
    fsFile = h5py.File(args.file, 'r')
    b_sie = Timestamp.calc_sie(args.begin)
    e_sie = Timestamp.calc_sie(args.end)
    return(b_sie, e_sie, fsFile)
Пример #28
0
        return
    oss = None
    for row in rows:
        if row["HOSTNAME"] == args.oss:
            oss = OSS.OSS(fsrc['name'], args.oss)
            break
    cursor.close()
    if oss == None:
        print "oss_aggregate: %s not found (try -s)" % args.oss
        return(None)
    if args.verbose == True:
        oss.debug()
        oss.debug(module="OST")
        #oss.debug(module="Timestamp")
    oss.getOSTs(fsrc['conn'])
    (begin_ts, end_ts) = Timestamp.process_timestamps(args, fsrc)
    oss.getData(begin_ts,
              end_ts)
    if (oss.Steps == None) or (oss.Steps.steps() == 0):
        print "oss_aggregate: Warning - No steps from OSS %s" % oss.name
        return(None)
    oss.getCPU()
    return(oss)

#*******************************************************************************

def do_action(args, oss):
    insert = oss.insertHeader()
    if insert is None:
        print "oss_aggregate.do_action(): Error - Failed to determine ossID"
        return
Пример #29
0
def do_action(args, bulk):
    scale=1024.0*1024.0
    fig = plt.figure()
    ax = fig.add_subplot(111)
    steps = bulk.Steps.Steps
    dates = np.array(steps, dtype=np.float64)
    if Timestamp.dst(steps[0]) == True:
        tzAdjust = 7.0
    else:
        tzAdjust = 8.0
    dates = (dates - tzAdjust*60.0*60.0)/(24.0*60.0*60.0) + mpl.dates.date2num(datetime.date(1970,1,1))
    if len(steps) <= 500:
        format = '-'
    else:
        # the ',' make the point one pixel
        format = ','
    ymax = 0
    values = bulk.Read.Values/scale
    max = np.max(values)
    if max > ymax:
        ymax = max
    ax.plot_date(dates, values, fmt=format, xdate=True, ydate=False, color='r', label="LMT read", zorder=0, drawstyle='steps')
    #Graph.timeSeries(ax, steps, values, 'r', label='read',
    #                 Ave=False)
    values = bulk.Write.Values/scale
    max = np.max(values)
    if max > ymax:
        ymax = max
    ax.plot_date(dates, values, fmt=format, xdate=True, ydate=False, color='b', label="LMT write", zorder=0, drawstyle='steps')
    #Graph.timeSeries(ax, steps, values, 'b', label='write',
    #                 Ave=False)
    IORDates = np.zeros(10)
    IORRead = np.zeros(10)
    IORWrite = np.zeros(10)
    IORDates[0] = dates[0]
    IORDates[-1] = dates[-1]
    if ((not args.rbegin is None) and (not args.rend is None)):
        beginsie = float(steps[0]) + args.rbegin + args.skew
        beginDate = ((beginsie - tzAdjust*60.0*60.0)/(24.0*60.0*60.0) +
                     mpl.dates.date2num(datetime.date(1970,1,1)))
        endsie = float(steps[0]) + args.rend + args.skew
        endDate   = ((endsie - tzAdjust*60.0*60.0)/(24.0*60.0*60.0) +
                     mpl.dates.date2num(datetime.date(1970,1,1)))
        if endsie > beginsie:
            rate = args.bytes/(endsie - beginsie)
            for i in range(5):
                IORDates[i] = dates[0] + i*((beginDate - dates[0])/5)
            IORDates[5] = beginDate - 0.0000001
            IORDates[6] = beginDate
            IORDates[7] = endDate
            IORDates[8] = endDate + 0.0000001
            IORRead[6] = rate
            IORRead[7] = rate
            ax.plot_date(IORDates, IORRead, fmt='--', xdate=True, ydate=False, color='r', label="IOR read", zorder=0)
            if ymax < rate:
                ymax = rate
    if ((not args.wbegin is None) and (not args.wend is None)):
        beginsie = float(steps[0]) + args.wbegin + args.skew
        beginDate = ((beginsie - tzAdjust*60.0*60.0)/(24.0*60.0*60.0) +
                     mpl.dates.date2num(datetime.date(1970,1,1)))
        endsie = float(steps[0]) + args.wend + args.skew
        endDate   = ((endsie - tzAdjust*60.0*60.0)/(24.0*60.0*60.0) +
                     mpl.dates.date2num(datetime.date(1970,1,1)))
        if endsie > beginsie:
            rate = args.bytes/(endsie - beginsie)
            IORDates[1] = beginDate - 0.0000001
            IORDates[2] = beginDate
            IORDates[3] = endDate
            IORDates[4] = endDate + 0.0000001
            IORWrite[2] = rate
            IORWrite[3] = rate
            ax.plot_date(IORDates, IORWrite, fmt='--', xdate=True, ydate=False, color='b', label="IOR write", zorder=0)
            if ymax < rate:
                ymax = rate
    plt.xlabel('time')
    plt.ylabel(r'$MiB/sec$')
    plt.legend()
    plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right')
    dayStr = time.strftime("%Y-%m-%d", time.localtime(bulk.begin.sie))
    plt.title("%s %s aggregate I/O" % (dayStr, bulk.name))
    if args.ybound is None:
        args.ybound = ymax
    ax.set_ybound(lower = 0, upper = args.ybound)
    if args.plot is None:
        plt.show()
    else:
        plt.savefig(args.plot)
    plt.cla()
Пример #30
0
                break
    cursor.close()
    if args.show_osss == True:
        return
    if oss is None:
        print "OSS: %s not found (try -s)" % args.oss
        return
    if args.verbose == True:
        oss.debug()
        oss.debug(module="OST")
        #oss.debug(module="Timestamp")
    oss.getOSTs(fsrc['conn'])
    if args.show_osts == True:
        oss.showOSTs()
        return
    (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
    Steps = TimeSteps.TimeSteps()
    Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
    oss.setSteps(Steps)
    if (oss.Steps is None) or (oss.Steps.steps() == 0):
        print "OSS: Warning - No steps from OSS %s" % oss.name
        return
    oss.getData()
    if (args.cpu == True) or (args.x_correlate == True):
        oss.getCPU()
    return(oss)

#*******************************************************************************

def do_action(args, oss):
    if (args.show_osss == True) or (args.show_osts == True):