def do_main(args): ''' ''' fsrc = LMTConfig.process_configuration(args) (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc) if args.mds == True: query = "SELECT TIMESTAMP_INFO.TS_ID,TIMESTAMP,PCT_CPU FROM TIMESTAMP_INFO,MDS_DATA WHERE " query += "TIMESTAMP_INFO.TS_ID=MDS_DATA.TS_ID AND " else: query = "SELECT TIMESTAMP_INFO.TS_ID,TIMESTAMP,PCT_CPU FROM TIMESTAMP_INFO,OSS_DATA,OSS_INFO WHERE " query += "OSS_DATA.OSS_ID=OSS_INFO.OSS_ID and OSS_INFO.HOSTNAME='" query += args.oss + "' AND " query += "TIMESTAMP_INFO.TS_ID=OSS_DATA.TS_ID AND " query += "TIMESTAMP_INFO.TIMESTAMP >= '" query += beginTimestamp.timestr query += "' AND TIMESTAMP_INFO.TIMESTAMP <= '" query += endTimestamp.timestr query += "'" try: cursor = fsrc['conn'].cursor(MySQLdb.cursors.DictCursor) if args.verbose == True: print "\t%s" % query cursor.execute (query) except MySQLdb.Error, e: cursor.close() print "CPU.get_ops_data_from_db: Error %d: %s\n%s" % (e.args[0], e.args[1], query) return(None)
def do_main(args): """ args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format. It will fill in todays year, month and day if left out. hh:mm:ss will default to 00:00:00 as portions are left out. both - Add together the read and write rates for calculation config - (file) The lmtrc config file telling how to get to the DB cpu - (boolean) Do the CPU utilization calculations (always set this true) end - (string) As above giving the end of the data to be gathered. fs - (string) The dbname entry in the config file for the file system of interest. index - (int) The index of the file system of interest in the config file oss - (string) The hostname of the OSS to be analyzed. ost - (string) The name of the OST to be analyzed. read - Caclulate for the read data rate verbose - (boolean) Turn on debugging output version - (boolean) print the version string and exit write - Calculate for the write data rate """ fsrc = LMTConfig.process_configuration(args) bulk = Bulk.Bulk(fsrc['dbname']) if args.verbose == True: bulk.debug() bulk.getOSSs(fsrc['conn']) (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc) Steps = TimeSteps.TimeSteps() Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn']) bulk.setSteps(Steps) if (bulk.Steps == None) or (bulk.Steps.steps() == 0): print "data_moved.do_main(): Warning - No steps from FS %s" % bulk.name sys.exit(1) bulk.getData() if args.cpu == True: bulk.getCPU() return(bulk)
def do_main(args): """ args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format. It will fill in todays year, month and day if left out. hh:mm:ss will default to 00:00:00 as portions are left out. config - (file) The lmtrc config file telling how to get to the DB end - (string) As above giving the end of the data to be gathered. fs - (string) The dbname entry in the config file for the file system of interest. index - (int) The index of the file system of interest in the config file verbose - (boolean) Turn on debugging output version - (boolean) print the version string and exit """ # do_main will: # - process_configuration # - get all the OSSs which gets their OSTs as well # - Process timestamps # - get the data including CPU utilization if asked # - return the bulk_aggregate object fsrc = LMTConfig.process_configuration(args) bulk = Bulk.Bulk(fsrc['name']) if args.verbose == True: bulk.debug() bulk.getOSSs(fsrc['conn']) (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc) Steps = TimeSteps.TimeSteps() Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn']) bulk.setSteps(Steps) if (bulk.Steps == None) or (bulk.Steps.steps() == 0): print "bulk_aggregate: Warning - No steps from FS %s" % bulk.name return(None) bulk.getData() bulk.getCPU() return(bulk)
def do_main(args): """ args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format. It will fill in todays year, month and day if left out. hh:mm:ss will default to 00:00:00 as portions are left out. config - (file) The lmtrc config file telling how to get to the DB end - (string) As above giving the end of the data to be gathered. fs - (string) The dbname entry in the config file for the file system of interest. index - (int) The index of the file system of interest in the config file verbose - (boolean) Turn on debugging output version - (boolean) print the version string and exit """ fsrc = LMTConfig.process_configuration(args) mds = MDS.MDS(host=fsrc["host"], fs=fsrc["name"]) if args.verbose == True: mds.debug() mds.debug(module="Operation") # mds.debug(module="Timestamp") mds.opsFromDB(fsrc["conn"]) (begin_ts, end_ts) = Timestamp.process_timestamps(args, fsrc) mds.getData(begin=begin_ts, end=end_ts) if mds.haveData == False: print "got no data from %s to %s" % (args.begin, args.end) return None mds.getCPU() return mds
def do_main(args): """ It looks like it is possible to get an incomplet coverage of the set of time steps if you only get rate and brw_stats data for one OST. I should fix this in the base modules. """ fsrc = LMTConfig.process_configuration(args) (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc) bulk = Bulk.Bulk(fsrc['name']) bulk.getOSSs(fsrc['conn']) Steps = TimeSteps.TimeSteps() Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn']) bulk.setSteps(Steps) bulk.getData() bulk.getCPU() brwfs = BrwFS.BrwFS(fsrc['name']) brwfs.getOSSs(fsrc['conn']) brwfs.setSteps(Steps) if args.stat == 'BRW_IOSIZE': iosize_bins = np.array([4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576]) brwfs.getBrwStats(fsrc['conn'], args.stat, bins=iosize_bins) else: brwfs.getBrwStats(fsrc['conn'], args.stat) brwfs.getData(args.stat) return(bulk, brwfs)
def get_bulk(args): fsrc = get_fsrc(args) if type(fsrc) is str: return(fsrc) bulk = Bulk.Bulk(fsrc['name']) bulk.getOSSs(fsrc['conn']) (begin_ts, end_ts) = Timestamp.process_timestamps(args, fsrc) bulk.getQuickData(begin_ts, end_ts, conn=fsrc['conn']) return(bulk)
def get_metadata(args): fsrc = get_fsrc(args) if fsrc is None: return(None) metadata = MDS.MDS(host=fsrc['host'], fs=fsrc['name']) metadata.opsFromDB(fsrc['conn']) (begin_ts, end_ts) = Timestamp.process_timestamps(args, fsrc) metadata.getQuickData(begin_ts, end_ts, conn=fsrc['conn']) metadata.getCPU() return(metadata)
def do_main(args): """ args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format. It will fill in todays year, month and day if left out. hh:mm:ss will default to 00:00:00 as portions are left out. config - (file) The lmtrc config file telling how to get to the DB cpu - (boolean) Get CPU utilization data and potentially plot it end - (string) As above giving the end of the data to be gathered. FS - (string) The dbname entry in the config file for the file system of interest. index - (int) The index of the file system of interest in the config file metadata - (boolean) Do metadata rather than bulk I/O data plot - (string) The name of the file to which the graph should be saved. 'noplot' is allowed if you just want a report. report - (boolean) Print out summary info about the analyzed operations verbose - (boolean) Turn on debugging output version - (boolean) print the version string and exit x_correlate - (boolean) plot the ops versus the CPU utilization rather than the ops versus time. ybound - (float) Use the given value as the maximum of the y-acis You can work with a data set interactively by entering ipython at the command line and carrying out this sequence of preparatory steps: from pyLMT import FS from pyLMT.tests import test_FS args = FS.process_args() args.begin = '2012-01-12' args.end = '2012-01-13' args.cpu = True args.FS = 'filesystem_scratch' fs = test_FS.do_main(args) It may take as much as half an hour to load up 24 hours worth of data, but once you have it in hand you can explore the data set interactively without having to go back and query the LMT DB over and over. """ # do_main will: # - process_configuration # - get the OSSs and metadata ops # - Process timestamps # - get the bulk data # - get the metadata # - return the FS object fsrc = LMTConfig.process_configuration(args) fs = FS.FS(fsrc['name']) if args.verbose == True: fs.debug() fs.getInfo(fsrc['conn']) (begin_ts, end_ts) = Timestamp.process_timestamps(args, fsrc) fs.getData(begin_ts, end_ts) if (args.cpu == True) or (args.x_correlate == True): fs.getCPU() return(fs)
def do_main(args): """ args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format. It will fill in todays year, month and day if left out. hh:mm:ss will default to 00:00:00 as portions are left out. config - (file) The lmtrc config file telling how to get to the DB cpu - plot CPU utilization end - (string) As above giving the end of the data to be gathered. extra - (int) If None then ignore, otherwise pad the graph before and after after the interval with this much extra, but don't inlcude in average calculation. fs - (string) The dbname entry in the config file for the file system of interest. index - (int) The index of the file system of interest in the config file mask - (key=value:keys in {mincpu, maxcpu, minval, maxval}) mask values outside the given range plot - (string) The name of the file to which the graph should be saved. 'noplot' is allowed if you just want a report. report - (boolean) Print out summary info about the analyzed operations show_ops - (boolean) List the available operations show_steps - (boolean) List the steps that are in the -M <mask> region spectrum - (boolean) Produce a graph of the Ops rate versus CPU utilization spectrum verbose - (boolean) Turn on debugging output version - (boolean) print the version string and exit x_correlate - (boolean) plot the ops versus the CPU utilization rather than the ops versus time. ybound - (float) Use the given value as the maximum of the y-acis """ fsrc = LMTConfig.process_configuration(args) mds = MDS.MDS(host=fsrc['host'], fs=fsrc['name']) if args.verbose == True: mds.debug() mds.debug(module="Operation") #mds.debug(module="Timestamp") mds.opsFromDB(fsrc['conn']) if args.show_ops == True: mds.showOps() return (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc) Steps = TimeSteps.TimeSteps() Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn']) mds.setSteps(Steps) mds.getData() if mds.haveData == False: print "got no data from %s to %s" % (args.begin, args.end) return(None) if (args.cpu == True) or (args.x_correlate == True): mds.getCPU() if not args.extra is None: if (args.extra <= 0) or (args.extra >= 100): print "mds: extra should be an integer between 0 and 100" args.extra = None return(mds)
def do_main(args): """ args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format. It will fill in todays year, month and day if left out. hh:mm:ss will default to 00:00:00 as portions are left out. both - Add together the read and wtites in the plot config - (file) The lmtrc config file telling how to get to the DB cpu - plot CPU utilization end - (string) As above giving the end of the data to be gathered. fs - (string) The dbname entry in the config file for the file system of interest. index - (int) The index of the file system of interest in the config file lines - (boolean) Graph using lines (no matter how many data points) mask - (key=value:keys in {mincpu, maxcpu, minval, maxval}) mask values outside the given range plot - (string) The name of the file to which the graph should be saved. 'noplot' is allowed if you just want a report. plotSdevs (boolean) Calculate and plot the standard deviation across OSTs at each timestep report - (boolean) Print out summary info about the analyzed operations read - Plot read data rate spectrum - Plot the cross-correlation spectrum frac - Filter out samples of the spectrum below this fraction of the max verbose - (boolean) Turn on debugging output version - (boolean) print the version string and exit write - Plot the write rate x_correlate - (boolean) plot the ops versus the CPU utilization rather than the ops versus time. ybound - (float) Use the given value as the maximum of the y-acis """ # do_main will: # - process_configuration # - get all the OSSs which gets their OSTs as well # - Process timestamps # - get the data including CPU utilization if asked # - return the bulk object fsrc = LMTConfig.process_configuration(args) if fsrc is None: return None B = Bulk.Bulk(fsrc['name']) if args.verbose == True: B.debug() B.getOSSs(fsrc['conn']) (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc) Steps = TimeSteps.TimeSteps() Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn']) B.setSteps(Steps) if (B.Steps is None) or (B.Steps.steps() == 0): print "bulk: Warning - No steps from FS %s" % B.name sys.exit(1) B.getData() if (args.cpu == True) or (args.x_correlate == True): B.getCPU() return(B)
def do_main(args): """ """ fsrc = LMTConfig.process_configuration(args) (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc) try: cursor = fsrc['conn'].cursor(MySQLdb.cursors.DictCursor) query = "SELECT * FROM OST_INFO" cursor.execute (query) rows = cursor.fetchall() except MySQLdb.Error, e: cursor.close() print "ost: Error %d: %s" % (e.args[0], e.args[1]) sys.exit(1)
def do_main(args): """ args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format. It will fill in todays year, month and day if left out. hh:mm:ss will default to 00:00:00 as portions are left out. config - (file) The lmtrc config file telling how to get to the DB end - (string) As above giving the end of the data to be gathered. fs - (string) The dbname entry in the config file for the file system of interest. index - (int) The index of the file system of interest in the config file plot - (string) The name of the file to which the graph should be saved. 'noplot' is allowed if you just want a report. report - (boolean) Print out summary info about the analyzed operations read - Plot read data rate stat - Show histograms only for this statistic verbose - (boolean) Turn on debugging output version - (boolean) print the version string and exit write - Plot the write rate """ # do_main will: # - process_configuration # - get the oss in question # - get the OSTs on it # - Process timestamps # - get the data # - return the oss fsrc = LMTConfig.process_configuration(args) brwfs = BrwFS.BrwFS(fsrc['dbname']) brwfs.getOSSs(fsrc['conn']) if args.verbose == True: brwfs.debug() brwfs.debug(module="BrwFS") #oss.debug(module="Timestamp") (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc) Steps = TimeSteps.TimeSteps() Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn']) brwfs.setSteps(Steps) brwfs.getBrwStats(conn=fsrc['conn'], stat=args.stat) brwfs.getData(stat=args.stat) if (brwfs.Bins is None) or (len(brwfs.Bins) == 0): print "test_BrwFS: Warning - No HistBins objects from OSS %s" % oss.name return if (brwfs.Steps is None) or (brwfs.Steps.steps() == 0): print "test_BrwFS: Warning - No steps from OSS %s" % oss.name return return(brwfs)
def do_main(args): ''' ''' fsrc = LMTConfig.process_configuration(args) (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc) try: cursor = fsrc['conn'].cursor(MySQLdb.cursors.DictCursor) query = "select TS_ID,TIMESTAMP from TIMESTAMP_INFO where TIMESTAMP >= '" query += beginTimestamp.timestr query += "' and TIMESTAMP <= '" query += endTimestamp.timestr query += "'" cursor.execute (query) rows = cursor.fetchall() except MySQLdb.Error, e: cursor.close() print "Counter: Error %d: %s" % (e.args[0], e.args[1]) sys.exit(1)
def doMain(args): """ It looks like it is possible to get an incomplete coverage of the set of time steps if you only get rate and brw_stats data for one OST. I should fix this in the base modules. """ fsrc = LMTConfig.process_configuration(args) (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc) Steps = TimeSteps.TimeSteps() Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn']) fs = FS.FS(fsrc['name']) fs.getInfo(fsrc['conn']) fs.setSteps(Steps) brwfs = BrwFS.BrwFS(fsrc['name']) brwfs.getOSSs(fsrc['conn']) iosize_bins = np.array([4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576]) brwfs.setSteps(Steps) brwfs.getBrwStats(fsrc['conn'], stat="BRW_IOSIZE", bins=iosize_bins) return(fs, brwfs)
def do_main(args): """ args.both - Add together the read and write rates for calculation config - (file) The lmtrc config file telling how to get to the DB cpu - (boolean) Do the CPU utilization calculations (always set this true) fs - (string) The dbname entry in the config file for the file system of interest. index - (int) The index of the file system of interest in the config file oss - (string) The hostname of the OSS to be analyzed. read - Caclulate for the read data rate seconds - (int) The length of time over which to average the observations. threshold - (int) Sound the alarm if the average exceeds this value. default - 40: GiB for rates, or pct. CPU utilization verbose - (boolean) Turn on debugging output version - (boolean) print the version string and exit write - Calculate for the write data rate """ fsrc = LMTConfig.process_configuration(args) bulk = Bulk.Bulk(fsrc['dbname']) if args.verbose == True: bulk.debug() bulk.getOSSs(fsrc['conn']) now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) # Now we want to generate the begin/end pair for this day end_sie = Timestamp.calc_sie(now) # This will be upset by savings time changes begin_sie = end_sie - args.seconds args.begin = Timestamp.format_timestamp(begin_sie) args.end = Timestamp.format_timestamp(end_sie) (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc) Steps = TimeSteps.TimeSteps() Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn']) bulk.setSteps(Steps) if (bulk.Steps == None) or (bulk.Steps.steps() == 0): print "pyalarm.do_main(): Warning - No steps from FS %s" % bulk.name return(None) bulk.getData() if args.cpu == True: bulk.getCPU() return(bulk)
def do_main(args): """ args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format. It will fill in todays year, month and day if left out. hh:mm:ss will default to 00:00:00 as portions are left out. both - Add together the read and wtites in the plot config - (file) The lmtrc config file telling how to get to the DB end - (string) As above giving the end of the data to be gathered. fs - (string) The dbname entry in the config file for the file system of interest. index - (int) The index of the file system of interest in the config file plot - (string) The name of the file to which the graph should be saved. 'noplot' is allowed if you just want a report. report - (boolean) Print out summary info about the analyzed operations read - Plot read data rate verbose - (boolean) Turn on debugging output version - (boolean) print the version string and exit write - Plot the write rate ybound - (float) Use the given value as the maximum of the y-acis """ # do_main will: # - process_configuration # - get all the OSSs which gets their OSTs as well # - Process timestamps # - get the data # - return the osts object fsrc = LMTConfig.process_configuration(args) bulk = Bulk.Bulk(fsrc['name']) if args.verbose == True: bulk.debug() bulk.getOSSs(fsrc['conn']) (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc) Steps = TimeSteps.TimeSteps() Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn']) bulk.setSteps(Steps) if (bulk.Steps is None) or (bulk.Steps.steps() == 0): print "osts: Warning - No steps from FS %s" % bulk.name sys.exit(1) bulk.getData() return(bulk)
def do_main(args): """ It looks like it is possible to get an incomplet coverage of the set of time steps if you only get rate and brw_stats data for one OST. I should fix this in the base modules. """ fsrc = LMTConfig.process_configuration(args) (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc) Steps = TimeSteps.TimeSteps() Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn']) thisost = None thisbo = None bulk = Bulk.Bulk(fsrc['name']) bulk.getOSSs(fsrc['conn']) bulk.setSteps(Steps) bulk.getData() bulk.getCPU() brwfs = BrwFS.BrwFS(fsrc['name']) brwfs.getOSSs(fsrc['conn']) brwfs.getBrwStats(fsrc['conn'], args.stat) brwfs.getData(beginTimestamp, endTimestamp, args.stat) return(bulk, brwfs)
def do_main(args): """ args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format. It will fill in todays year, month and day if left out. hh:mm:ss will default to 00:00:00 as portions are left out. config - (file) The lmtrc config file telling how to get to the DB cpu - plot CPU utilization end - (string) As above giving the end of the data to be gathered. fs - (string) The dbname entry in the config file for the file system of interest. index - (int) The index of the file system of interest in the config file mask - (key=value:keys in {mincpu, maxcpu, minval, maxval}) mask values outside the given range plot - (string) The name of the file to which the graph should be saved. 'noplot' is allowed if you just want a report. report - (boolean) Print out summary info about the analyzed operations spectrum - Plot the cross-correlation spectrum verbose - (boolean) Turn on debugging output version - (boolean) print the version string and exit x_correlate - (boolean) plot the ops versus the CPU utilization rather than the ops versus time. ybound - (float) Use the given value as the maximum of the y-acis """ fsrc = LMTConfig.process_configuration(args) mds = MDS.MDS(host=fsrc['host'],fs=fsrc['name']) if args.verbose == True: mds.debug() mds.debug(module="Operation") #mds.getOSSs(fsrc['conn']) (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc) Steps = TimeSteps.TimeSteps() Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn']) mds.setSteps(Steps) mds.getQuickData(conn=fsrc['conn']) if (mds.Steps == None) or (mds.Steps.steps() == 0): print "mds_quick: Warning - No steps from FS %s" % mds.name return(None) mds.getCPU() return(mds)
break cursor.close() if args.show_osss == True: return if oss is None: print "OSS: %s not found (try -s)" % args.oss return if args.verbose == True: oss.debug() oss.debug(module="OST") #oss.debug(module="Timestamp") oss.getOSTs(fsrc['conn']) if args.show_osts == True: oss.showOSTs() return (beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc) Steps = TimeSteps.TimeSteps() Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn']) oss.setSteps(Steps) if (oss.Steps is None) or (oss.Steps.steps() == 0): print "OSS: Warning - No steps from OSS %s" % oss.name return oss.getData() if (args.cpu == True) or (args.x_correlate == True): oss.getCPU() return(oss) #******************************************************************************* def do_action(args, oss): if (args.show_osss == True) or (args.show_osts == True):
return oss = None for row in rows: if row["HOSTNAME"] == args.oss: oss = OSS.OSS(fsrc['name'], args.oss) break cursor.close() if oss == None: print "oss_aggregate: %s not found (try -s)" % args.oss return(None) if args.verbose == True: oss.debug() oss.debug(module="OST") #oss.debug(module="Timestamp") oss.getOSTs(fsrc['conn']) (begin_ts, end_ts) = Timestamp.process_timestamps(args, fsrc) oss.getData(begin_ts, end_ts) if (oss.Steps == None) or (oss.Steps.steps() == 0): print "oss_aggregate: Warning - No steps from OSS %s" % oss.name return(None) oss.getCPU() return(oss) #******************************************************************************* def do_action(args, oss): insert = oss.insertHeader() if insert is None: print "oss_aggregate.do_action(): Error - Failed to determine ossID" return