def do_main(args):
'''
'''
fsrc = LMTConfig.process_configuration(args)
(beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
if args.mds == True:
query = "SELECT TIMESTAMP_INFO.TS_ID,TIMESTAMP,PCT_CPU FROM TIMESTAMP_INFO,MDS_DATA WHERE "
query += "TIMESTAMP_INFO.TS_ID=MDS_DATA.TS_ID AND "
else:
query = "SELECT TIMESTAMP_INFO.TS_ID,TIMESTAMP,PCT_CPU FROM TIMESTAMP_INFO,OSS_DATA,OSS_INFO WHERE "
query += "OSS_DATA.OSS_ID=OSS_INFO.OSS_ID and OSS_INFO.HOSTNAME='"
query += args.oss + "' AND "
query += "TIMESTAMP_INFO.TS_ID=OSS_DATA.TS_ID AND "
query += "TIMESTAMP_INFO.TIMESTAMP >= '"
query += beginTimestamp.timestr
query += "' AND TIMESTAMP_INFO.TIMESTAMP <= '"
query += endTimestamp.timestr
query += "'"
try:
cursor = fsrc['conn'].cursor(MySQLdb.cursors.DictCursor)
if args.verbose == True:
print "\t%s" % query
cursor.execute (query)
except MySQLdb.Error, e:
cursor.close()
print "CPU.get_ops_data_from_db: Error %d: %s\n%s" % (e.args[0], e.args[1], query)
return(None)
def do_main(args):
"""
It looks like it is possible to get an incomplet coverage of the set of time
steps if you only get rate and brw_stats data for one OST. I should fix this
in the base modules.
"""
fsrc = LMTConfig.process_configuration(args)
sie = Timestamp.calc_sie(args.time)
beginTimestamp = Timestamp.Timestamp(sie - 5)
endTimestamp = Timestamp.Timestamp(sie)
beginTimestamp.no_later_than(fsrc['conn'])
endTimestamp.no_earlier_than(fsrc['conn'])
if beginTimestamp.ts_id >= endTimestamp.ts_id:
print "observation: failed to get time stamp range for %s" % args.time
exit(1)
Steps = TimeSteps.TimeSteps()
Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
bulk = Bulk.Bulk(fsrc['name'])
bulk.getOSSs(fsrc['conn'])
bulk.setSteps(Steps)
bulk.getData()
# FIXME
# I am not handling masked values in the CPU data
# Should be an easy fix
bulk.getCPU()
brwfs = BrwFS.BrwFS(fsrc['name'])
brwfs.getOSSs(fsrc['conn'])
# we'll query for all of them even when we only want one
# because it make the processing more uniform
brwfs.getBrwStats(fsrc['conn'], stat=args.stat)
brwfs.getData(beginTimestamp, endTimestamp, stat=args.stat)
return(bulk, brwfs)
def do_main(args):
"""
args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
It will fill in todays year, month and day if left out.
hh:mm:ss will default to 00:00:00 as portions are left out.
config - (file) The lmtrc config file telling how to get to the DB
end - (string) As above giving the end of the data to be gathered.
FS - (string) The dbname entry in the config file for the file system of interest.
index - (int) The index of the file system of interest in the config file
oss - (string) The hostname of the OSS to be analyzed.
verbose - (boolean) Turn on debugging output
version - (boolean) print the version string and exit
"""
# do_main will:
# - process_configuration
# - get the oss in question
# - get the OSTs on it
# - Process timestamps
# - get the data including CPU utilization
# - return the oss
fsrc = LMTConfig.process_configuration(args)
try:
cursor = fsrc['conn'].cursor(MySQLdb.cursors.DictCursor)
query = "SELECT * FROM OSS_INFO"
cursor.execute (query)
rows = cursor.fetchall()
except MySQLdb.Error, e:
cursor.close()
print "OSS: Error %d: %s" % (e.args[0], e.args[1])
return
def do_main(args):
"""
args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
It will fill in todays year, month and day if left out.
hh:mm:ss will default to 00:00:00 as portions are left out.
config - (file) The lmtrc config file telling how to get to the DB
end - (string) As above giving the end of the data to be gathered.
fs - (string) The dbname entry in the config file for the file system of interest.
index - (int) The index of the file system of interest in the config file
verbose - (boolean) Turn on debugging output
version - (boolean) print the version string and exit
"""
# do_main will:
# - process_configuration
# - get all the OSSs which gets their OSTs as well
# - Process timestamps
# - get the data including CPU utilization if asked
# - return the bulk_aggregate object
fsrc = LMTConfig.process_configuration(args)
bulk = Bulk.Bulk(fsrc['name'])
if args.verbose == True:
bulk.debug()
bulk.getOSSs(fsrc['conn'])
(beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
Steps = TimeSteps.TimeSteps()
Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
bulk.setSteps(Steps)
if (bulk.Steps == None) or (bulk.Steps.steps() == 0):
print "bulk_aggregate: Warning - No steps from FS %s" % bulk.name
return(None)
bulk.getData()
bulk.getCPU()
return(bulk)
def do_main(args):
"""
args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
It will fill in todays year, month and day if left out.
hh:mm:ss will default to 00:00:00 as portions are left out.
config - (file) The lmtrc config file telling how to get to the DB
end - (string) As above giving the end of the data to be gathered.
fs - (string) The dbname entry in the config file for the file system of interest.
index - (int) The index of the file system of interest in the config file
verbose - (boolean) Turn on debugging output
version - (boolean) print the version string and exit
"""
fsrc = LMTConfig.process_configuration(args)
mds = MDS.MDS(host=fsrc["host"], fs=fsrc["name"])
if args.verbose == True:
mds.debug()
mds.debug(module="Operation")
# mds.debug(module="Timestamp")
mds.opsFromDB(fsrc["conn"])
(begin_ts, end_ts) = Timestamp.process_timestamps(args, fsrc)
mds.getData(begin=begin_ts, end=end_ts)
if mds.haveData == False:
print "got no data from %s to %s" % (args.begin, args.end)
return None
mds.getCPU()
return mds
def do_main(args):
"""
args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
It will fill in todays year, month and day if left out.
hh:mm:ss will default to 00:00:00 as portions are left out.
both - Add together the read and write rates for calculation
config - (file) The lmtrc config file telling how to get to the DB
cpu - (boolean) Do the CPU utilization calculations (always set this true)
end - (string) As above giving the end of the data to be gathered.
fs - (string) The dbname entry in the config file for the file system of interest.
index - (int) The index of the file system of interest in the config file
oss - (string) The hostname of the OSS to be analyzed.
ost - (string) The name of the OST to be analyzed.
read - Caclulate for the read data rate
verbose - (boolean) Turn on debugging output
version - (boolean) print the version string and exit
write - Calculate for the write data rate
"""
fsrc = LMTConfig.process_configuration(args)
bulk = Bulk.Bulk(fsrc['dbname'])
if args.verbose == True:
bulk.debug()
bulk.getOSSs(fsrc['conn'])
(beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
Steps = TimeSteps.TimeSteps()
Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
bulk.setSteps(Steps)
if (bulk.Steps == None) or (bulk.Steps.steps() == 0):
print "data_moved.do_main(): Warning - No steps from FS %s" % bulk.name
sys.exit(1)
bulk.getData()
if args.cpu == True:
bulk.getCPU()
return(bulk)
def do_main(args):
"""
It looks like it is possible to get an incomplet coverage of the set of time
steps if you only get rate and brw_stats data for one OST. I should fix this
in the base modules.
"""
fsrc = LMTConfig.process_configuration(args)
(beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
bulk = Bulk.Bulk(fsrc['name'])
bulk.getOSSs(fsrc['conn'])
Steps = TimeSteps.TimeSteps()
Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
bulk.setSteps(Steps)
bulk.getData()
bulk.getCPU()
brwfs = BrwFS.BrwFS(fsrc['name'])
brwfs.getOSSs(fsrc['conn'])
brwfs.setSteps(Steps)
if args.stat == 'BRW_IOSIZE':
iosize_bins = np.array([4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576])
brwfs.getBrwStats(fsrc['conn'], args.stat, bins=iosize_bins)
else:
brwfs.getBrwStats(fsrc['conn'], args.stat)
brwfs.getData(args.stat)
return(bulk, brwfs)
def doMain(args):
"""
"""
fsrc = LMTConfig.process_configuration(args)
args.host = fsrc['host']
#beginSie = Timestamp.calc_sie(args.begin)
#endSie = Timestamp.calc_sie(args.end)
beginSie = int(time.mktime(time.strptime(args.begin, "%Y-%m-%d %H:%M:%S" )))
endSie = int(time.mktime(time.strptime(args.end, "%Y-%m-%d %H:%M:%S" )))
fs = FS.FS(fsrc['name'])
fs.getInfo(fsrc['conn'])
return(fs, beginSie, endSie)
def do_main(args):
"""
args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
It will fill in todays year, month and day if left out.
hh:mm:ss will default to 00:00:00 as portions are left out.
config - (file) The lmtrc config file telling how to get to the DB
cpu - (boolean) Get CPU utilization data and potentially plot it
end - (string) As above giving the end of the data to be gathered.
FS - (string) The dbname entry in the config file for the file system of interest.
index - (int) The index of the file system of interest in the config file
metadata - (boolean) Do metadata rather than bulk I/O data
plot - (string) The name of the file to which the graph should be saved.
'noplot' is allowed if you just want a report.
report - (boolean) Print out summary info about the analyzed operations
verbose - (boolean) Turn on debugging output
version - (boolean) print the version string and exit
x_correlate - (boolean) plot the ops versus the CPU utilization rather than
the ops versus time.
ybound - (float) Use the given value as the maximum of the y-acis
You can work with a data set interactively by entering ipython at the command line
and carrying out this sequence of preparatory steps:
from pyLMT import FS
from pyLMT.tests import test_FS
args = FS.process_args()
args.begin = '2012-01-12'
args.end = '2012-01-13'
args.cpu = True
args.FS = 'filesystem_scratch'
fs = test_FS.do_main(args)
It may take as much as half an hour to load up 24 hours worth of data,
but once you have it in hand you can explore the data set interactively
without having to go back and query the LMT DB over and over.
"""
# do_main will:
# - process_configuration
# - get the OSSs and metadata ops
# - Process timestamps
# - get the bulk data
# - get the metadata
# - return the FS object
fsrc = LMTConfig.process_configuration(args)
fs = FS.FS(fsrc['name'])
if args.verbose == True:
fs.debug()
fs.getInfo(fsrc['conn'])
(begin_ts, end_ts) = Timestamp.process_timestamps(args, fsrc)
fs.getData(begin_ts,
end_ts)
if (args.cpu == True) or (args.x_correlate == True):
fs.getCPU()
return(fs)
def do_main(args):
"""
"""
fsrc = LMTConfig.process_configuration(args)
try:
cursor = fsrc['conn'].cursor(MySQLdb.cursors.DictCursor)
query = "SELECT * FROM OST_INFO"
cursor.execute (query)
rows = cursor.fetchall()
except MySQLdb.Error, e:
cursor.close()
print "OST: Error %d: %s" % (e.args[0], e.args[1])
return(None)
def do_main(args):
"""
args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
It will fill in todays year, month and day if left out.
hh:mm:ss will default to 00:00:00 as portions are left out.
both - Add together the read and wtites in the plot
config - (file) The lmtrc config file telling how to get to the DB
cpu - plot CPU utilization
end - (string) As above giving the end of the data to be gathered.
fs - (string) The dbname entry in the config file for the file system of interest.
index - (int) The index of the file system of interest in the config file
lines - (boolean) Graph using lines (no matter how many data points)
mask - (key=value:keys in {mincpu, maxcpu, minval, maxval}) mask values outside the given range
plot - (string) The name of the file to which the graph should be saved.
'noplot' is allowed if you just want a report.
plotSdevs (boolean) Calculate and plot the standard deviation across OSTs at each timestep
report - (boolean) Print out summary info about the analyzed operations
read - Plot read data rate
spectrum - Plot the cross-correlation spectrum
frac - Filter out samples of the spectrum below this fraction of the max
verbose - (boolean) Turn on debugging output
version - (boolean) print the version string and exit
write - Plot the write rate
x_correlate - (boolean) plot the ops versus the CPU utilization rather than
the ops versus time.
ybound - (float) Use the given value as the maximum of the y-acis
"""
# do_main will:
# - process_configuration
# - get all the OSSs which gets their OSTs as well
# - Process timestamps
# - get the data including CPU utilization if asked
# - return the bulk object
fsrc = LMTConfig.process_configuration(args)
if fsrc is None:
return None
B = Bulk.Bulk(fsrc['name'])
if args.verbose == True:
B.debug()
B.getOSSs(fsrc['conn'])
(beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
Steps = TimeSteps.TimeSteps()
Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
B.setSteps(Steps)
if (B.Steps is None) or (B.Steps.steps() == 0):
print "bulk: Warning - No steps from FS %s" % B.name
sys.exit(1)
B.getData()
if (args.cpu == True) or (args.x_correlate == True):
B.getCPU()
return(B)
def do_main(args):
"""
args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
It will fill in todays year, month and day if left out.
hh:mm:ss will default to 00:00:00 as portions are left out.
config - (file) The lmtrc config file telling how to get to the DB
cpu - plot CPU utilization
end - (string) As above giving the end of the data to be gathered.
extra - (int) If None then ignore, otherwise pad the graph before and after
after the interval with this much extra, but don't inlcude in
average calculation.
fs - (string) The dbname entry in the config file for the file system of interest.
index - (int) The index of the file system of interest in the config file
mask - (key=value:keys in {mincpu, maxcpu, minval, maxval}) mask values outside the given range
plot - (string) The name of the file to which the graph should be saved.
'noplot' is allowed if you just want a report.
report - (boolean) Print out summary info about the analyzed operations
show_ops - (boolean) List the available operations
show_steps - (boolean) List the steps that are in the -M <mask> region
spectrum - (boolean) Produce a graph of the Ops rate versus CPU utilization spectrum
verbose - (boolean) Turn on debugging output
version - (boolean) print the version string and exit
x_correlate - (boolean) plot the ops versus the CPU utilization rather than
the ops versus time.
ybound - (float) Use the given value as the maximum of the y-acis
"""
fsrc = LMTConfig.process_configuration(args)
mds = MDS.MDS(host=fsrc['host'], fs=fsrc['name'])
if args.verbose == True:
mds.debug()
mds.debug(module="Operation")
#mds.debug(module="Timestamp")
mds.opsFromDB(fsrc['conn'])
if args.show_ops == True:
mds.showOps()
return
(beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
Steps = TimeSteps.TimeSteps()
Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
mds.setSteps(Steps)
mds.getData()
if mds.haveData == False:
print "got no data from %s to %s" % (args.begin, args.end)
return(None)
if (args.cpu == True) or (args.x_correlate == True):
mds.getCPU()
if not args.extra is None:
if (args.extra <= 0) or (args.extra >= 100):
print "mds: extra should be an integer between 0 and 100"
args.extra = None
return(mds)
def do_main(args):
"""
"""
fsrc = LMTConfig.process_configuration(args)
(beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
try:
cursor = fsrc['conn'].cursor(MySQLdb.cursors.DictCursor)
query = "SELECT * FROM OST_INFO"
cursor.execute (query)
rows = cursor.fetchall()
except MySQLdb.Error, e:
cursor.close()
print "ost: Error %d: %s" % (e.args[0], e.args[1])
sys.exit(1)
def do_main(args):
"""
args.apportion - A seconds in epoch value at which to show the amount of
I/O from each OST.
begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
It will fill in todays year, month and day if left out.
hh:mm:ss will default to 00:00:00 as portions are left out.
both - Add together the read and writes in the plot
config - (file) The lmtrc config file telling how to get to the DB
cpu - plot CPU utilization
end - (string) As above giving the end of the data to be gathered.
fs - (string) The dbname entry in the config file for the file system of interest.
index - (int) The index of the file system of interest in the config file
lines - (boolean) Graph using lines (no matter how many data points)
mask - (key=value:keys in {mincpu, maxcpu, minval, maxval}) mask values outside the given range
oss - (string) The hostname of the OSS to be analyzed.
plot - (string) The name of the file to which the graph should be saved.
'noplot' is allowed if you just want a report.
plotSdevs (boolean) Calculate and plot the standard deviation across OSTs at each timestep
report - (boolean) Print out summary info about the analyzed operations
read - Plot read data rate
show_osss - (boolean) Just list the available OSS host names to choose
from for the -o <oss> argument, then exit.
show_osts - (boolean) Just list the OSTs on the OSS -o <oss>
show_steps - (boolean) List the steps that are in the -M <mask> region
spectrum - (boolean) Produce a graph of the rate versus CPU utilization spectrum
verbose - (boolean) Turn on debugging output
version - (boolean) print the version string and exit
write - Plot the write rate
x_correlate - (boolean) plot the ops versus the CPU utilization rather than
the ops versus time.
ybound - (float) Use the given value as the maximum of the y-acis
"""
# do_main will:
# - process_configuration
# - get the oss in question with early terminiation if it's just a show_osss
# - get the OSTs on it with early termination if it's just a show_osts
# - Process timestamps
# - get the data including CPU utilization if asked
# - return the oss
fsrc = LMTConfig.process_configuration(args)
try:
cursor = fsrc['conn'].cursor(MySQLdb.cursors.DictCursor)
query = "SELECT * FROM OSS_INFO"
cursor.execute (query)
rows = cursor.fetchall()
except MySQLdb.Error, e:
cursor.close()
print "oss: Error %d: %s" % (e.args[0], e.args[1])
return
def get_fsrc(args):
fsrc = None
if args.host == 'franklin':
if args.filesys == 'scratch':
args.fs = 'filesystem_franklin_scratch_20090420'
fsrc = LMTConfig.process_configuration(args)
elif args.filesys == 'scratch2':
args.fs = 'filesystem_franklin_scratch2_20090420'
fsrc = LMTConfig.process_configuration(args)
ymax = 18000
elif args.host == 'hopper':
if args.filesys == 'scratch':
args.fs = 'filesystem_scratch'
fsrc = LMTConfig.process_configuration(args)
if fsrc is None:
return('fsrc is None')
else:
return(fsrc)
elif args.filesys == 'scratch2':
args.fs = 'filesystem_scratch2'
fsrc = LMTConfig.process_configuration(args)
ymax = 50000
return(fsrc)
def do_main(args):
"""
args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
It will fill in todays year, month and day if left out.
hh:mm:ss will default to 00:00:00 as portions are left out.
config - (file) The lmtrc config file telling how to get to the DB
end - (string) As above giving the end of the data to be gathered.
fs - (string) The dbname entry in the config file for the file system of interest.
index - (int) The index of the file system of interest in the config file
plot - (string) The name of the file to which the graph should be saved.
'noplot' is allowed if you just want a report.
report - (boolean) Print out summary info about the analyzed operations
read - Plot read data rate
stat - Show histograms only for this statistic
verbose - (boolean) Turn on debugging output
version - (boolean) print the version string and exit
write - Plot the write rate
"""
# do_main will:
# - process_configuration
# - get the oss in question
# - get the OSTs on it
# - Process timestamps
# - get the data
# - return the oss
fsrc = LMTConfig.process_configuration(args)
brwfs = BrwFS.BrwFS(fsrc['dbname'])
brwfs.getOSSs(fsrc['conn'])
if args.verbose == True:
brwfs.debug()
brwfs.debug(module="BrwFS")
#oss.debug(module="Timestamp")
(beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
Steps = TimeSteps.TimeSteps()
Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
brwfs.setSteps(Steps)
brwfs.getBrwStats(conn=fsrc['conn'], stat=args.stat)
brwfs.getData(stat=args.stat)
if (brwfs.Bins is None) or (len(brwfs.Bins) == 0):
print "test_BrwFS: Warning - No HistBins objects from OSS %s" % oss.name
return
if (brwfs.Steps is None) or (brwfs.Steps.steps() == 0):
print "test_BrwFS: Warning - No steps from OSS %s" % oss.name
return
return(brwfs)
def do_main(args):
'''
'''
fsrc = LMTConfig.process_configuration(args)
(beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
try:
cursor = fsrc['conn'].cursor(MySQLdb.cursors.DictCursor)
query = "select TS_ID,TIMESTAMP from TIMESTAMP_INFO where TIMESTAMP >= '"
query += beginTimestamp.timestr
query += "' and TIMESTAMP <= '"
query += endTimestamp.timestr
query += "'"
cursor.execute (query)
rows = cursor.fetchall()
except MySQLdb.Error, e:
cursor.close()
print "Counter: Error %d: %s" % (e.args[0], e.args[1])
sys.exit(1)
def doMain(args):
"""
It looks like it is possible to get an incomplete coverage of the set of time
steps if you only get rate and brw_stats data for one OST. I should fix this
in the base modules.
"""
fsrc = LMTConfig.process_configuration(args)
(beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
Steps = TimeSteps.TimeSteps()
Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
fs = FS.FS(fsrc['name'])
fs.getInfo(fsrc['conn'])
fs.setSteps(Steps)
brwfs = BrwFS.BrwFS(fsrc['name'])
brwfs.getOSSs(fsrc['conn'])
iosize_bins = np.array([4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576])
brwfs.setSteps(Steps)
brwfs.getBrwStats(fsrc['conn'], stat="BRW_IOSIZE", bins=iosize_bins)
return(fs, brwfs)
def do_main(args):
"""
args.both - Add together the read and write rates for calculation
config - (file) The lmtrc config file telling how to get to the DB
cpu - (boolean) Do the CPU utilization calculations (always set this true)
fs - (string) The dbname entry in the config file for the file system of
interest.
index - (int) The index of the file system of interest in the config file
oss - (string) The hostname of the OSS to be analyzed.
read - Caclulate for the read data rate
seconds - (int) The length of time over which to average the observations.
threshold - (int) Sound the alarm if the average exceeds this value.
default - 40: GiB for rates, or pct. CPU utilization
verbose - (boolean) Turn on debugging output
version - (boolean) print the version string and exit
write - Calculate for the write data rate
"""
fsrc = LMTConfig.process_configuration(args)
bulk = Bulk.Bulk(fsrc['dbname'])
if args.verbose == True:
bulk.debug()
bulk.getOSSs(fsrc['conn'])
now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# Now we want to generate the begin/end pair for this day
end_sie = Timestamp.calc_sie(now)
# This will be upset by savings time changes
begin_sie = end_sie - args.seconds
args.begin = Timestamp.format_timestamp(begin_sie)
args.end = Timestamp.format_timestamp(end_sie)
(beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
Steps = TimeSteps.TimeSteps()
Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
bulk.setSteps(Steps)
if (bulk.Steps == None) or (bulk.Steps.steps() == 0):
print "pyalarm.do_main(): Warning - No steps from FS %s" % bulk.name
return(None)
bulk.getData()
if args.cpu == True:
bulk.getCPU()
return(bulk)
def do_main(args):
"""
It looks like it is possible to get an incomplet coverage of the set of time
steps if you only get rate and brw_stats data for one OST. I should fix this
in the base modules.
"""
fsrc = LMTConfig.process_configuration(args)
(beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
Steps = TimeSteps.TimeSteps()
Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
thisost = None
thisbo = None
bulk = Bulk.Bulk(fsrc['name'])
bulk.getOSSs(fsrc['conn'])
bulk.setSteps(Steps)
bulk.getData()
bulk.getCPU()
brwfs = BrwFS.BrwFS(fsrc['name'])
brwfs.getOSSs(fsrc['conn'])
brwfs.getBrwStats(fsrc['conn'], args.stat)
brwfs.getData(beginTimestamp, endTimestamp, args.stat)
return(bulk, brwfs)
def do_main(args):
"""
args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
It will fill in todays year, month and day if left out.
hh:mm:ss will default to 00:00:00 as portions are left out.
both - Add together the read and wtites in the plot
config - (file) The lmtrc config file telling how to get to the DB
end - (string) As above giving the end of the data to be gathered.
fs - (string) The dbname entry in the config file for the file system of interest.
index - (int) The index of the file system of interest in the config file
plot - (string) The name of the file to which the graph should be saved.
'noplot' is allowed if you just want a report.
report - (boolean) Print out summary info about the analyzed operations
read - Plot read data rate
verbose - (boolean) Turn on debugging output
version - (boolean) print the version string and exit
write - Plot the write rate
ybound - (float) Use the given value as the maximum of the y-acis
"""
# do_main will:
# - process_configuration
# - get all the OSSs which gets their OSTs as well
# - Process timestamps
# - get the data
# - return the osts object
fsrc = LMTConfig.process_configuration(args)
bulk = Bulk.Bulk(fsrc['name'])
if args.verbose == True:
bulk.debug()
bulk.getOSSs(fsrc['conn'])
(beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
Steps = TimeSteps.TimeSteps()
Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
bulk.setSteps(Steps)
if (bulk.Steps is None) or (bulk.Steps.steps() == 0):
print "osts: Warning - No steps from FS %s" % bulk.name
sys.exit(1)
bulk.getData()
return(bulk)
def do_main(args):
"""
args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
It will fill in todays year, month and day if left out.
hh:mm:ss will default to 00:00:00 as portions are left out.
config - (file) The lmtrc config file telling how to get to the DB
cpu - plot CPU utilization
end - (string) As above giving the end of the data to be gathered.
fs - (string) The dbname entry in the config file for the file system of interest.
index - (int) The index of the file system of interest in the config file
mask - (key=value:keys in {mincpu, maxcpu, minval, maxval}) mask values outside the given range
plot - (string) The name of the file to which the graph should be saved.
'noplot' is allowed if you just want a report.
report - (boolean) Print out summary info about the analyzed operations
spectrum - Plot the cross-correlation spectrum
verbose - (boolean) Turn on debugging output
version - (boolean) print the version string and exit
x_correlate - (boolean) plot the ops versus the CPU utilization rather than
the ops versus time.
ybound - (float) Use the given value as the maximum of the y-acis
"""
fsrc = LMTConfig.process_configuration(args)
mds = MDS.MDS(host=fsrc['host'],fs=fsrc['name'])
if args.verbose == True:
mds.debug()
mds.debug(module="Operation")
#mds.getOSSs(fsrc['conn'])
(beginTimestamp, endTimestamp) = Timestamp.process_timestamps(args, fsrc)
Steps = TimeSteps.TimeSteps()
Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
mds.setSteps(Steps)
mds.getQuickData(conn=fsrc['conn'])
if (mds.Steps == None) or (mds.Steps.steps() == 0):
print "mds_quick: Warning - No steps from FS %s" % mds.name
return(None)
mds.getCPU()
return(mds)
def do_main(args):
"""
args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
It will fill in todays year, month and day if left out.
hh:mm:ss will default to 00:00:00 as portions are left out.
config - (file) The lmtrc config file telling how to get to the DB
end - (string) As above giving the end of the data to be gathered.
fs - (string) The dbname entry in the config file for the file system of interest.
index - (int) The index of the file system of interest in the config file
oss - (string) The hostname of the OSS to be analyzed.
plot - (string) The name of the file to which the graph should be saved.
'noplot' is allowed if you just want a report.
report - (boolean) Print out summary info about the analyzed operations
read - Plot read data rate
stat - Show histograms only for this statistic
verbose - (boolean) Turn on debugging output
version - (boolean) print the version string and exit
write - Plot the write rate
"""
# do_main will:
# - process_configuration
# - get the oss in question
# - get the OSTs on it
# - Process timestamps
# - get the data
# - return the oss
fsrc = LMTConfig.process_configuration(args)
try:
cursor = fsrc['conn'].cursor(MySQLdb.cursors.DictCursor)
query = "SELECT * FROM OSS_INFO"
cursor.execute (query)
rows = cursor.fetchall()
except MySQLdb.Error, e:
cursor.close()
print "test_BrwOSS: Error %d: %s" % (e.args[0], e.args[1])
return
print "MDS: Error - No ops have been specified"
else:
args.mds_op = [args.hilite_op]
if (args.x_correlate == True) and ((len(args.mds_op) != 1) or (args.mds_op[0] == "cpu")):
print "MDS: If you want a cross correlation (-x) you need to provide exactly one other operation (not -m cpu)"
sys.exit()
if args.x_correlate == True:
args.mds_op += ["cpu"]
return(args)
#*******************************************************************************
print "This code is out of date. Revise before using."
sys.exit(1)
args = process_args()
fsrc = LMTConfig.process_configuration(args)
mds = MDS.MDS(host=fsrc['host'], fs=fsrc['name'])
if args.verbose == True:
mds.debug()
mds.debug(module="Operation")
#mds.debug(module="Timestamp")
mds.opsFromDB(fsrc['conn'])
(beginTimestamp, endTimestamp) = Timestamp.day(args, fsrc)
Steps = TimeSteps.TimeSteps()
Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
mds.setSteps(Steps)
mds.getData()
if mds.haveData == False:
print "got no data from %s to %s" % (args.begin, args.end)
sys.exit(0)
if args.report == True:
def do_main(args):
"""
args.begin - (string) Gives the date in yyyy-mm-dd hh:mm:ss format.
It will fill in todays year, month and day if left out.
hh:mm:ss will default to 00:00:00 as portions are left out.
config - (file) The lmtrc config file telling how to get to the DB
end - (string) As above giving the end of the data to be gathered.
fs - (string) The dbname entry in the config file for the file system of interest.
index - (int) The index of the file system of interest in the config file
plot - (string) The name of the file to which the graph should be saved.
'noplot' is allowed if you just want a report.
progress - Print the name of each OST as you work through the list
report - (boolean) Print out summary info about the analyzed operations
read - Plot read data rate
stat - Show histograms only for this statistic
verbose - (boolean) Turn on debugging output
version - (boolean) print the version string and exit
write - Plot the write rate
"""
# do_main will:
# - process_configuration
# - get the oss in question
# - get the OSTs on it
# - Process timestamps
# - get the data
# - return the oss
bins = None
if args.stat == "BRW_IOSIZE":
bins = np.array([4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576])
fsrc = LMTConfig.process_configuration(args)
sie = Timestamp.calc_sie(args.begin)
beginTimestamp = Timestamp.Timestamp(sie)
endTimestamp = Timestamp.Timestamp(sie + 5)
beginTimestamp.no_later_than(fsrc['conn'])
endTimestamp.no_earlier_than(fsrc['conn'])
if beginTimestamp.ts_id >= endTimestamp.ts_id:
print "observation: failed to get time stamp range for %s" % args.time
exit(1)
brwBegin = BrwFS.BrwFS(fsrc['dbname'])
brwBegin.getOSSs(fsrc['conn'])
if args.verbose == True:
brwBegin.debug()
brwBegin.debug(module="BrwFS")
#oss.debug(module="Timestamp")
Steps = TimeSteps.TimeSteps()
Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
brwBegin.setSteps(Steps)
brwBegin.getBrwStats(conn=fsrc['conn'], stat=args.stat, bins=bins)
brwBegin.getDataSlice(stat=args.stat)
if (brwBegin.Bins is None) or (len(brwBegin.Bins) == 0):
print "test_BrwFS: Warning - No HistBins objects from OSS %s" % oss.name
return
if (brwBegin.Steps is None) or (brwBegin.Steps.steps() == 0):
print "test_BrwFS: Warning - No steps from OSS %s" % oss.name
return
sie = Timestamp.calc_sie(args.end)
beginTimestamp = Timestamp.Timestamp(sie)
endTimestamp = Timestamp.Timestamp(sie + 5)
beginTimestamp.no_later_than(fsrc['conn'])
endTimestamp.no_earlier_than(fsrc['conn'])
if beginTimestamp.ts_id >= endTimestamp.ts_id:
print "observation: failed to get time stamp range for %s" % args.time
return(None)
brwEnd = BrwFS.BrwFS(fsrc['dbname'])
brwEnd.getOSSs(fsrc['conn'])
if args.verbose == True:
brwEnd.debug()
brwEnd.debug(module="BrwFS")
#oss.debug(module="Timestamp")
Steps = TimeSteps.TimeSteps()
Steps.getTimeSteps(beginTimestamp, endTimestamp, fsrc['conn'])
brwEnd.setSteps(Steps)
brwEnd.getBrwStats(conn=fsrc['conn'], stat=args.stat, bins=bins)
brwEnd.getDataSlice(stat=args.stat)
if (brwEnd.Bins is None) or (len(brwEnd.Bins) == 0):
print "test_BrwFS: Warning - No HistBins objects from OSS %s" % oss.name
return
if (brwEnd.Steps is None) or (brwEnd.Steps.steps() == 0):
print "test_BrwFS: Warning - No steps from OSS %s" % oss.name
return
return(brwBegin, brwEnd)