def run(self):
super(Cosbench, self).run()
self.dropcaches()
self.cluster.dump_config(self.run_dir)
monitoring.start(self.run_dir)
# Run cosbench test
try:
self._run()
except KeyboardInterrupt:
logger.warning("accept keyboard interrupt, cancel this run")
conf = self.config
stdout, stderr = common.pdsh(
"%s@%s" % (self.user, conf["controller"]),
'sh %s/cli.sh cancel %s' %
(conf["cosbench_dir"], self.runid)).communicate()
logger.info("%s", stdout)
self.check_workload_status()
self.check_cosbench_res_dir()
monitoring.stop(self.run_dir)
self.cluster.dump_historic_ops(self.run_dir)
common.sync_files('%s/*' % self.run_dir, self.out_dir)
common.sync_files(
'%s/archive/%s*' % (self.config["cosbench_dir"], self.runid),
self.out_dir)
def initialize(self):
super(LibrbdFio, self).initialize()
logger.info('Running scrub monitoring.')
monitoring.start("%s/scrub_monitoring" % self.run_dir)
self.cluster.check_scrub()
monitoring.stop()
logger.info('Pausing for 60s for idle monitoring.')
monitoring.start("%s/idle_monitoring" % self.run_dir)
time.sleep(60)
monitoring.stop()
common.sync_files('%s/*' % self.run_dir, self.out_dir)
self.mkimages()
# Create the run directory
common.make_remote_dir(self.run_dir)
# populate the fio files
ps = []
logger.info('Attempting to populating fio files...')
for i in xrange(self.volumes_per_client):
pre_cmd = 'sudo %s --ioengine=rbd --clientname=admin --pool=%s --rbdname=cbt-librbdfio-`hostname -s`-%d --invalidate=0 --rw=write --numjobs=%s --bs=4M --size %dM %s > /dev/null' % (
self.cmd_path, self.poolname, i, self.numjobs, self.vol_size,
self.names)
p = common.pdsh(settings.getnodes('clients'), pre_cmd)
ps.append(p)
for p in ps:
p.wait()
return True
def initialize(self):
super(LibrbdFio, self).initialize()
# Clean and Create the run directory
common.clean_remote_dir(self.run_dir)
common.make_remote_dir(self.run_dir)
logger.info('Pausing for 60s for idle monitoring.')
monitoring.start("%s/idle_monitoring" % self.run_dir)
time.sleep(60)
monitoring.stop()
common.sync_files('%s/*' % self.run_dir, self.out_dir)
self.mkimages()
# populate the fio files
ps = []
logger.info('Attempting to populating fio files...')
if (self.use_existing_volumes == False):
for volnum in range(self.volumes_per_client):
rbd_name = 'cbt-librbdfio-`%s`-%d' % (common.get_fqdn_cmd(),
volnum)
pre_cmd = 'sudo %s --ioengine=rbd --clientname=admin --pool=%s --rbdname=%s --invalidate=0 --rw=write --numjobs=%s --bs=4M --size %dM %s --output-format=%s > /dev/null' % (
self.cmd_path, self.pool_name, rbd_name, self.numjobs,
self.vol_size, self.names, self.fio_out_format)
p = common.pdsh(settings.getnodes('clients'), pre_cmd)
ps.append(p)
for p in ps:
p.wait()
def _run(self, mode, run_dir, out_dir):
# We'll always drop caches for rados bench
self.dropcaches()
if self.concurrent_ops:
concurrent_ops_str = "--concurrent-ios %s" % self.concurrent_ops
op_size_str = "-b %s" % self.op_size
common.make_remote_dir(run_dir)
monitoring.start(run_dir)
# Run rados bench
print "Running radosbench read test."
ps = []
for i in xrange(self.concurrent_procs):
out_file = "%s/output.%s" % (run_dir, i)
objecter_log = "%s/objecter.%s.log" % (run_dir, i)
p = common.pdsh(
settings.cluster.get("clients"),
"/usr/bin/rados -p rados-bench-%s %s bench %s %s %s --no-cleanup 2> %s > %s"
% (i, op_size_str, self.time, mode, concurrent_ops_str, objecter_log, out_file),
)
ps.append(p)
for p in ps:
p.wait()
monitoring.stop(run_dir)
common.sync_files("%s/*" % run_dir, out_dir)
def initialize(self):
super(RbdFio, self).initialize()
logger.info('Running scrub monitoring.')
monitoring.start("%s/scrub_monitoring" % self.run_dir)
self.cluster.check_scrub()
monitoring.stop()
logger.info('Pausing for 60s for idle monitoring.')
monitoring.start("%s/idle_monitoring" % self.run_dir)
time.sleep(60)
monitoring.stop()
common.sync_files('%s/*' % self.run_dir, self.out_dir)
self.mkimages()
# Create the run directory
common.make_remote_dir(self.run_dir)
# populate the fio files
logger.info('Attempting to populating fio files...')
pre_cmd = 'sudo %s --ioengine=%s --rw=write --numjobs=%s --bs=4M --size %dM %s > /dev/null' % (self.cmd_path, self.ioengine, self.numjobs, self.vol_size*0.9, self.names)
common.pdsh(settings.getnodes('clients'), pre_cmd).communicate()
return True
def initialize(self):
common.cleanup_tests()
if not self.use_existing:
common.setup_cluster()
common.setup_ceph()
# Create the run directory
common.make_remote_dir(self.run_dir)
# Setup the pools
monitoring.start("%s/pool_monitoring" % self.run_dir)
for i in xrange(self.concurrent_procs):
for node in settings.getnodes('clients').split(','):
node = node.rpartition("@")[2]
common.pdsh(settings.getnodes('head'), 'sudo ceph osd pool create rados-bench-%s-%s %d %d' % (node, i, self.pgs_per_pool, self.pgs_per_pool)).communicate()
common.pdsh(settings.getnodes('head'), 'sudo ceph osd pool set rados-bench-%s-%s size 1' % (node, i)).communicate()
# check the health for each pool.
print 'Checking Healh after pool creation.'
common.check_health()
monitoring.stop()
print 'Running scrub monitoring.'
monitoring.start("%s/scrub_monitoring" % self.run_dir)
common.check_scrub()
monitoring.stop()
print 'Pausing for 60s for idle monitoring.'
monitoring.start("%s/idle_monitoring" % self.run_dir)
time.sleep(60)
monitoring.stop()
common.sync_files('%s/*' % self.run_dir, self.out_dir)
return True
def initialize(self):
super(LibrbdFio, self).initialize()
print "Running scrub monitoring."
monitoring.start("%s/scrub_monitoring" % self.run_dir)
self.cluster.check_scrub()
monitoring.stop()
print "Pausing for 60s for idle monitoring."
monitoring.start("%s/idle_monitoring" % self.run_dir)
time.sleep(60)
monitoring.stop()
common.sync_files("%s/*" % self.run_dir, self.out_dir)
self.mkimages()
# Create the run directory
common.make_remote_dir(self.run_dir)
# populate the fio files
ps = []
print "Attempting to populating fio files..."
for i in xrange(self.volumes_per_client):
pre_cmd = (
"sudo %s --ioengine=rbd --clientname=admin --pool=%s --rbdname=cbt-librbdfio-`hostname -s`-%d --invalidate=0 --rw=write --numjobs=%s --bs=4M --size %dM %s > /dev/null"
% (self.cmd_path, self.poolname, i, self.numjobs, self.vol_size, self.names)
)
p = common.pdsh(settings.getnodes("clients"), pre_cmd)
ps.append(p)
for p in ps:
p.wait()
return True
def run(self):
super(CephTestRados, self).run()
# Remake the pool
self.mkpool()
self.dropcaches()
self.cluster.dump_config(self.run_dir)
monitoring.start(self.run_dir)
time.sleep(5)
# Run the backfill testing thread if requested
if 'recovery_test' in self.cluster.config:
recovery_callback = self.recovery_callback
self.cluster.create_recovery_test(self.run_dir, recovery_callback)
logger.info('Running ceph_test_rados.')
ps = []
for i in xrange(1):
p = common.pdsh(settings.getnodes('clients'), self.mkcmd())
ps.append(p)
for p in ps:
p.wait()
# If we were doing recovery, wait until it's done.
if 'recovery_test' in self.cluster.config:
self.cluster.wait_recovery_done()
monitoring.stop(self.run_dir)
# Finally, get the historic ops
self.cluster.dump_historic_ops(self.run_dir)
common.sync_files('%s/*' % self.run_dir, self.out_dir)
def run(self):
super(RbdFio, self).run()
# We'll always drop caches for rados bench
self.dropcaches()
common.make_remote_dir(self.run_dir)
monitoring.start(self.run_dir)
# Run rados bench
print 'Running rbd fio %s test.' % self.mode
names = ""
for i in xrange(self.concurrent_procs):
names += "--name=/srv/rbdfio-`hostname -s`-%d/cbt-rbdfio " % i
out_file = '%s/output' % self.run_dir
fio_cmd = 'sudo fio --rw=%s -ioengine=%s --runtime=%s --numjobs=1 --direct=1 --bs=%dB --iodepth=%d --size %dM %s > %s' % (
self.mode, self.ioengine, self.time, self.op_size, self.iodepth,
self.vol_size * 9 / 10, names, out_file)
common.pdsh(settings.getnodes('clients'), fio_cmd).communicate()
# ps = []
# for i in xrange(self.concurrent_procs):
# out_file = '%s/output.%s' % (self.run_dir, i)
# p = common.pdsh(settings.cluster.get('clients'), 'sudo fio --rw=%s -ioengine=%s --runtime=%s --name=/srv/rbdfio-`hostname -s`-%d/cbt-rbdfio --numjobs=1 --direct=1 --bs=%dB --iodepth=%d --size %dM > %s' % (self.mode, self.ioengine, self.time, i, self.op_size, self.iodepth, self.vol_size * 9/10, out_file))
# ps.append(p)
# for p in ps:
# p.wait()
monitoring.stop(self.run_dir)
common.sync_files('%s/*' % self.run_dir, self.out_dir)
def run(self):
super(LibrbdFio, self).run()
# We'll always drop caches for rados bench
self.dropcaches()
# dump the cluster config
self.cluster.dump_config(self.run_dir)
monitoring.start(self.run_dir)
time.sleep(5)
# Run the backfill testing thread if requested
if "recovery_test" in self.cluster.config:
recovery_callback = self.recovery_callback
self.cluster.create_recovery_test(self.run_dir, recovery_callback)
print "Running rbd fio %s test." % self.mode
ps = []
for i in xrange(self.volumes_per_client):
fio_cmd = self.mkfiocmd(i)
p = common.pdsh(settings.getnodes("clients"), fio_cmd)
ps.append(p)
for p in ps:
p.wait()
# If we were doing recovery, wait until it's done.
if "recovery_test" in self.cluster.config:
self.cluster.wait_recovery_done()
monitoring.stop(self.run_dir)
# Finally, get the historic ops
self.cluster.dump_historic_ops(self.run_dir)
common.sync_files("%s/*" % self.run_dir, self.out_dir)
def initialize(self):
super(LibrbdFio, self).initialize()
print 'Running scrub monitoring.'
monitoring.start("%s/scrub_monitoring" % self.run_dir)
self.cluster.check_scrub()
monitoring.stop()
print 'Pausing for 60s for idle monitoring.'
monitoring.start("%s/idle_monitoring" % self.run_dir)
time.sleep(60)
monitoring.stop()
common.sync_files('%s/*' % self.run_dir, self.out_dir)
self.mkimages()
# Create the run directory
common.make_remote_dir(self.run_dir)
# populate the fio files
print 'Attempting to populating fio files...'
pre_cmd = 'sudo %s --ioengine=rbd --clientname=admin --pool=%s --rbdname=cbt-librbdfio-`hostname -s` --invalidate=0 --rw=write --numjobs=%s --bs=4M --size %dM %s > /dev/null' % (self.cmd_path, self.poolname, self.numjobs, self.vol_size, self.names)
common.pdsh(settings.getnodes('clients'), pre_cmd).communicate()
return True
def run(self):
# First create a credential file for each gateway
self.mkcredfiles()
# We'll always drop caches for rados bench
self.dropcaches()
# dump the cluster config
self.cluster.dump_config(self.run_dir)
# Run the backfill testing thread if requested
if 'recovery_test' in self.cluster.config:
recovery_callback = self.recovery_callback
self.cluster.create_recovery_test(self.run_dir, recovery_callback)
# Run getput
monitoring.start(self.run_dir)
logger.info('Running getput %s test.' % self.test)
ps = []
for i in xrange(0, len(self.auth_urls)):
cmd = self.mkgetputcmd("%s/gw%02d.cred" % (self.run_dir, i), i)
p = common.pdsh(settings.getnodes('clients'), cmd)
ps.append(p)
for p in ps:
p.wait()
monitoring.stop(self.run_dir)
# If we were doing recovery, wait until it's done.
if 'recovery_test' in self.cluster.config:
self.cluster.wait_recovery_done()
# Finally, get the historic ops
self.cluster.dump_historic_ops(self.run_dir)
common.sync_files('%s/*' % self.run_dir, self.out_dir)
def run(self):
super(CephTestRados, self).run()
# Remake the pool
self.mkpool()
self.dropcaches()
self.cluster.dump_config(self.run_dir)
monitoring.start(self.run_dir)
time.sleep(5)
# Run the backfill testing thread if requested
if 'recovery_test' in self.cluster.config:
recovery_callback = self.recovery_callback
self.cluster.create_recovery_test(self.run_dir, recovery_callback)
logger.info('Running ceph_test_rados.')
ps = []
for i in xrange(1):
p = common.pdsh(settings.getnodes('clients'), self.mkcmd())
ps.append(p)
for p in ps:
p.wait()
# If we were doing recovery, wait until it's done.
if 'recovery_test' in self.cluster.config:
self.cluster.wait_recovery_done()
monitoring.stop(self.run_dir)
# Finally, get the historic ops
self.cluster.dump_historic_ops(self.run_dir)
common.sync_files('%s/*' % self.run_dir, self.out_dir)
def _run(self, mode, run_dir, out_dir):
# We'll always drop caches for rados bench
self.dropcaches()
if self.concurrent_ops:
concurrent_ops_str = '--concurrent-ios %s' % self.concurrent_ops
op_size_str = '-b %s' % self.op_size
common.make_remote_dir(run_dir)
# dump the cluster config
common.dump_config(run_dir)
monitoring.start(run_dir)
# Run rados bench
print 'Running radosbench read test.'
ps = []
for i in xrange(self.concurrent_procs):
out_file = '%s/output.%s' % (run_dir, i)
objecter_log = '%s/objecter.%s.log' % (run_dir, i)
p = common.pdsh(settings.getnodes('clients'), '/usr/bin/rados -p rados-bench-`hostname -s`-%s %s bench %s %s %s --no-cleanup 2> %s > %s' % (i, op_size_str, self.time, mode, concurrent_ops_str, objecter_log, out_file))
ps.append(p)
for p in ps:
p.wait()
monitoring.stop(run_dir)
# Get the historic ops
common.dump_historic_ops(run_dir)
common.sync_files('%s/*' % run_dir, out_dir)
def run(self):
super(RbdFio, self).run()
# Set client readahead
self.set_client_param('read_ahead_kb', self.client_ra)
# We'll always drop caches for rados bench
self.dropcaches()
common.make_remote_dir(self.run_dir)
monitoring.start(self.run_dir)
# Run rados bench
print 'Running rbd fio %s test.' % self.mode
names = ""
for i in xrange(self.concurrent_procs):
names += "--name=%s/mnt/rbdfio-`hostname -s`-%d/cbt-rbdfio " % (self.tmp_dir, i)
out_file = '%s/output' % self.run_dir
fio_cmd = 'sudo fio --rw=%s -ioengine=%s --runtime=%s --numjobs=1 --direct=1 --bs=%dB --iodepth=%d --size %dM %s > %s' % (self.mode, self.ioengine, self.time, self.op_size, self.iodepth, self.vol_size * 9/10, names, out_file)
common.pdsh(settings.getnodes('clients'), fio_cmd).communicate()
# ps = []
# for i in xrange(self.concurrent_procs):
# out_file = '%s/output.%s' % (self.run_dir, i)
# p = common.pdsh(settings.cluster.get('clients'), 'sudo fio --rw=%s -ioengine=%s --runtime=%s --name=/srv/rbdfio-`hostname -s`-%d/cbt-rbdfio --numjobs=1 --direct=1 --bs=%dB --iodepth=%d --size %dM > %s' % (self.mode, self.ioengine, self.time, i, self.op_size, self.iodepth, self.vol_size * 9/10, out_file))
# ps.append(p)
# for p in ps:
# p.wait()
monitoring.stop(self.run_dir)
common.sync_files('%s/*' % self.run_dir, self.out_dir)
def initialize(self):
super(LibrbdFio, self).initialize()
# Clean and Create the run directory
common.clean_remote_dir(self.run_dir)
common.make_remote_dir(self.run_dir)
logger.info('Pausing for 60s for idle monitoring.')
monitoring.start("%s/idle_monitoring" % self.run_dir)
time.sleep(60)
monitoring.stop()
common.sync_files('%s/*' % self.run_dir, self.out_dir)
self.mkimages()
# populate the fio files
ps = []
logger.info('Attempting to populating fio files...')
if (self.use_existing_volumes == False):
for volnum in xrange(self.volumes_per_client):
rbd_name = 'cbt-librbdfio-`%s`-%d' % (common.get_fqdn_cmd(), volnum)
pre_cmd = 'sudo %s --ioengine=rbd --clientname=admin --pool=%s --rbdname=%s --invalidate=0 --rw=write --numjobs=%s --bs=4M --size %dM %s --output-format=%s > /dev/null' % (self.cmd_path, self.pool_name, rbd_name, self.numjobs, self.vol_size, self.names, self.fio_out_format)
p = common.pdsh(settings.getnodes('clients'), pre_cmd)
ps.append(p)
for p in ps:
p.wait()
return True
def run(self):
super(KvmRbdFio, self).run()
# Set client readahead
self.set_client_param('read_ahead_kb', self.client_ra)
clnts = settings.getnodes('clients')
# We'll always drop caches for rados bench
self.dropcaches()
monitoring.start(self.run_dir)
time.sleep(5)
# Run the backfill testing thread if requested
if 'recovery_test' in self.cluster.config:
recovery_callback = self.recovery_callback
self.cluster.create_recovery_test(self.run_dir, recovery_callback)
logger.info('Starting rbd fio %s test.', self.mode)
fio_process_list = []
for i in range(self.concurrent_procs):
b = self.block_devices[i % len(self.block_devices)]
bnm = os.path.basename(b)
mtpt = '/srv/rbdfio-`hostname -s`-%s' % bnm
fiopath = os.path.join(mtpt, 'fio%d.img' % i)
out_file = '%s/output.%d' % (self.run_dir, i)
fio_cmd = 'sudo %s' % self.fio_cmd
fio_cmd += ' --rw=%s' % self.mode
if (self.mode == 'readwrite' or self.mode == 'randrw'):
fio_cmd += ' --rwmixread=%s --rwmixwrite=%s' % (
self.rwmixread, self.rwmixwrite)
fio_cmd += ' --ioengine=%s' % self.ioengine
fio_cmd += ' --runtime=%s' % self.time
fio_cmd += ' --ramp_time=%s' % self.ramp
if self.startdelay:
fio_cmd += ' --startdelay=%s' % self.startdelay
if self.rate_iops:
fio_cmd += ' --rate_iops=%s' % self.rate_iops
fio_cmd += ' --numjobs=%s' % self.numjobs
fio_cmd += ' --direct=1'
fio_cmd += ' --bs=%dB' % self.op_size
fio_cmd += ' --iodepth=%d' % self.iodepth
fio_cmd += ' --size=%dM' % self.vol_size
if self.log_iops:
fio_cmd += ' --write_iops_log=%s' % out_file
if self.log_bw:
fio_cmd += ' --write_bw_log=%s' % out_file
if self.log_lat:
fio_cmd += ' --write_lat_log=%s' % out_file
if 'recovery_test' in self.cluster.config:
fio_cmd += ' --time_based'
fio_cmd += ' --name=%s > %s' % (fiopath, out_file)
fio_process_list.append(
common.pdsh(clnts, fio_cmd, continue_if_error=False))
for p in fio_process_list:
p.communicate()
monitoring.stop(self.run_dir)
logger.info('Finished rbd fio test')
common.sync_files('%s/*' % self.run_dir, self.out_dir)
def run(self):
super(KvmRbdFio, self).run()
# We'll always drop caches for rados bench
self.dropcaches()
monitoring.start(self.run_dir)
time.sleep(5)
names = ""
for i in xrange(self.concurrent_procs):
names += "--name=/srv/rbdfio-`hostname -s`-%d/cbt-kvmrbdfio " % i
out_file = '%s/output' % self.run_dir
pre_cmd = 'sudo fio --rw=read -ioengine=sync --numjobs=1 --bs=4M --runtime=1 --size %dM %s > /dev/null' % (
self.vol_size * 9 / 10, names)
fio_cmd = 'sudo fio --rw=%s -ioengine=%s --runtime=%s --numjobs=1 --direct=1 --bs=%dB --iodepth=%d --size %dM %s > %s' % (
self.mode, self.ioengine, self.time, self.op_size, self.iodepth,
self.vol_size * 9 / 10, names, out_file)
print 'Attempting to populating fio files...'
common.pdsh(settings.getnodes('clients'), pre_cmd).communicate()
print 'Running rbd fio %s test.' % self.mode
common.pdsh(settings.getnodes('clients'), fio_cmd).communicate()
# ps = []
# for i in xrange(self.concurrent_procs):
# out_file = '%s/output.%s' % (self.run_dir, i)
# p = common.pdsh(settings.cluster.get('clients'), 'sudo fio --rw=%s -ioengine=%s --runtime=%s --name=/srv/rbdfio-`hostname -s`-%d/cbt-rbdfio --numjobs=1 --direct=1 --bs=%dB --iodepth=%d --size %dM > %s' % (self.mode, self.ioengine, self.time, i, self.op_size, self.iodepth, self.vol_size * 9/10, out_file))
# ps.append(p)
# for p in ps:
# p.wait()
monitoring.stop(self.run_dir)
common.sync_files('%s/*' % self.run_dir, self.out_dir)
def initialize(self):
super(Cosbench, self).initialize()
logger.debug('Running cosbench and radosgw check.')
self.prerun_check()
logger.debug('Running scrub monitoring.')
monitoring.start("%s/scrub_monitoring" % self.run_dir)
self.cluster.check_scrub()
monitoring.stop()
logger.debug('Pausing for 60s for idle monitoring.')
monitoring.start("%s/idle_monitoring" % self.run_dir)
time.sleep(60)
monitoring.stop()
common.sync_files('%s' % self.run_dir, self.out_dir)
# Create the run directory
common.make_remote_dir(self.run_dir)
conf = self.config
if not self.config["template"]:
self.config["template"] = "default"
self.config["workload"] = self.choose_template("default", conf)
self.prepare_xml(self.config["workload"])
return True
def initialize(self):
super(Cosbench, self).initialize()
logger.debug('Running cosbench and radosgw check.')
self.prerun_check()
logger.debug('Running scrub monitoring.')
monitoring.start("%s/scrub_monitoring" % self.run_dir)
self.cluster.check_scrub()
monitoring.stop()
logger.debug('Pausing for 60s for idle monitoring.')
monitoring.start("%s/idle_monitoring" % self.run_dir)
time.sleep(60)
monitoring.stop()
common.sync_files('%s' % self.run_dir, self.out_dir)
# Create the run directory
common.make_remote_dir(self.run_dir)
conf = self.config
if not self.config["template"]:
self.config["template"] = "default"
self.config["workload"] = self.choose_template("default", conf)
self.prepare_xml(self.config["workload"])
return True
def run(self):
# First create a credential file for each gateway
self.mkcredfiles()
# We'll always drop caches for rados bench
self.dropcaches()
# dump the cluster config
self.cluster.dump_config(self.run_dir)
# Run the backfill testing thread if requested
if 'recovery_test' in self.cluster.config:
recovery_callback = self.recovery_callback
self.cluster.create_recovery_test(self.run_dir, recovery_callback)
# Run getput
monitoring.start(self.run_dir)
logger.info('Running getput %s test.' % self.test)
ps = []
for i in range(0, len(self.auth_urls)):
cmd = self.mkgetputcmd("%s/gw%02d.cred" % (self.run_dir, i), i)
p = common.pdsh(settings.getnodes('clients'), cmd)
ps.append(p)
for p in ps:
p.wait()
monitoring.stop(self.run_dir)
# If we were doing recovery, wait until it's done.
if 'recovery_test' in self.cluster.config:
self.cluster.wait_recovery_done()
# Finally, get the historic ops
self.cluster.dump_historic_ops(self.run_dir)
common.sync_files('%s/*' % self.run_dir, self.out_dir)
def run(self):
super(Fio, self).run()
# We'll always drop caches for rados bench
self.dropcaches()
# dump the cluster config
self.cluster.dump_config(self.run_dir)
monitoring.start(self.run_dir)
time.sleep(5)
# Run the backfill testing thread if requested
if 'recovery_test' in self.cluster.config:
recovery_callback = self.recovery_callback
self.cluster.create_recovery_test(self.run_dir, recovery_callback)
logger.info('Running fio %s test.', self.mode)
ps = []
for i in xrange(self.endpoints_per_client):
p = common.pdsh(settings.getnodes('clients'), self.run_command(i))
ps.append(p)
for p in ps:
p.wait()
# If we were doing recovery, wait until it's done.
if 'recovery_test' in self.cluster.config:
self.cluster.wait_recovery_done()
monitoring.stop(self.run_dir)
# Finally, get the historic ops
self.cluster.dump_historic_ops(self.run_dir)
common.sync_files('%s/*' % self.run_dir, self.out_dir)
self.analyze(self.out_dir)
def run(self):
super(KvmRbdFio, self).run()
# We'll always drop caches for rados bench
self.dropcaches()
monitoring.start(self.run_dir)
time.sleep(5)
names = ""
for i in xrange(self.concurrent_procs):
names += "--name=/srv/rbdfio-`hostname -s`-%d/cbt-kvmrbdfio " % i
out_file = '%s/output' % self.run_dir
pre_cmd = 'sudo fio --rw=read -ioengine=sync --numjobs=1 --bs=4M --runtime=1 --size %dM %s > /dev/null' % (self.vol_size * 9/10, names)
fio_cmd = 'sudo fio --rw=%s -ioengine=%s --runtime=%s --numjobs=1 --direct=1 --bs=%dB --iodepth=%d --size %dM %s > %s' % (self.mode, self.ioengine, self.time, self.op_size, self.iodepth, self.vol_size * 9/10, names, out_file)
print 'Attempting to populating fio files...'
common.pdsh(settings.cluster.get('clients'), pre_cmd).communicate()
print 'Running rbd fio %s test.' % self.mode
common.pdsh(settings.cluster.get('clients'), fio_cmd).communicate()
# ps = []
# for i in xrange(self.concurrent_procs):
# out_file = '%s/output.%s' % (self.run_dir, i)
# p = common.pdsh(settings.cluster.get('clients'), 'sudo fio --rw=%s -ioengine=%s --runtime=%s --name=/srv/rbdfio-`hostname -s`-%d/cbt-rbdfio --numjobs=1 --direct=1 --bs=%dB --iodepth=%d --size %dM > %s' % (self.mode, self.ioengine, self.time, i, self.op_size, self.iodepth, self.vol_size * 9/10, out_file))
# ps.append(p)
# for p in ps:
# p.wait()
monitoring.stop(self.run_dir)
common.sync_files('%s/*' % self.run_dir, self.out_dir)
def run(self):
super(RbdFio, self).run()
# Set client readahead
self.set_client_param('read_ahead_kb', self.client_ra)
# We'll always drop caches for rados bench
self.dropcaches()
monitoring.start(self.run_dir)
# Run the backfill testing thread if requested
if 'recovery_test' in self.cluster.config:
recovery_callback = self.recovery_callback
self.cluster.create_recovery_test(self.run_dir, recovery_callback)
time.sleep(5)
out_file = '%s/output' % self.run_dir
fio_cmd = 'sudo %s' % (self.cmd_path_full)
fio_cmd += ' --rw=%s' % self.mode
if (self.mode == 'readwrite' or self.mode == 'randrw'):
fio_cmd += ' --rwmixread=%s --rwmixwrite=%s' % (self.rwmixread,
self.rwmixwrite)
fio_cmd += ' --ioengine=%s' % self.ioengine
if self.time is not None:
fio_cmd += ' --runtime=%s' % self.time
if self.ramp is not None:
fio_cmd += ' --ramp_time=%s' % self.ramp
fio_cmd += ' --numjobs=%s' % self.numjobs
fio_cmd += ' --direct=%s' % self.direct
fio_cmd += ' --bs=%dB' % self.op_size
fio_cmd += ' --iodepth=%d' % self.iodepth
if self.vol_size:
fio_cmd += ' --size=%dM' % (int(self.vol_size) * 0.9)
if self.log_iops:
fio_cmd += ' --write_iops_log=%s' % out_file
if self.log_bw:
fio_cmd += ' --write_bw_log=%s' % out_file
if self.log_lat:
fio_cmd += ' --write_lat_log=%s' % out_file
if 'recovery_test' in self.cluster.config:
fio_cmd += ' --time_based'
if self.random_distribution is not None:
fio_cmd += ' --random_distribution=%s' % self.random_distribution
fio_cmd += ' %s > %s' % (self.names, out_file)
if self.log_avg_msec is not None:
fio_cmd += ' --log_avg_msec=%s' % self.log_avg_msec
logger.info('Running rbd fio %s test.', self.mode)
common.pdsh(settings.getnodes('clients'), fio_cmd).communicate()
# If we were doing recovery, wait until it's done.
if 'recovery_test' in self.cluster.config:
self.cluster.wait_recovery_done()
monitoring.stop(self.run_dir)
# Finally, get the historic ops
self.cluster.dump_historic_ops(self.run_dir)
common.sync_files('%s/*' % self.run_dir, self.out_dir)
def run(self):
super(LibrbdFio, self).run()
# We'll always drop caches for rados bench
self.dropcaches()
# dump the cluster config
self.cluster.dump_config(self.run_dir)
monitoring.start(self.run_dir)
time.sleep(5)
out_file = '%s/output' % self.run_dir
fio_cmd = 'sudo %s --ioengine=rbd --clientname=admin --pool=%s --rbdname=cbt-librbdfio-`hostname -s` --invalidate=0' % (
self.cmd_path_full, self.poolname)
fio_cmd += ' --rw=%s' % self.mode
if (self.mode == 'readwrite' or self.mode == 'randrw'):
fio_cmd += ' --rwmixread=%s --rwmixwrite=%s' % (self.rwmixread,
self.rwmixwrite)
# fio_cmd += ' --ioengine=%s' % self.ioengine
if self.time is not None:
fio_cmd += ' --runtime=%s' % self.time
if self.ramp is not None:
fio_cmd += ' --ramp_time=%s' % self.ramp
fio_cmd += ' --numjobs=%s' % self.numjobs
fio_cmd += ' --direct=1'
fio_cmd += ' --bs=%dB' % self.op_size
fio_cmd += ' --iodepth=%d' % self.iodepth
fio_cmd += ' --end_fsync=%s' % self.end_fsync
# if self.vol_size:
# fio_cmd += ' -- size=%dM' % self.vol_size
fio_cmd += ' --write_iops_log=%s' % out_file
fio_cmd += ' --write_bw_log=%s' % out_file
fio_cmd += ' --write_lat_log=%s' % out_file
if 'recovery_test' in self.cluster.config:
fio_cmd += ' --time_based'
if self.random_distribution is not None:
fio_cmd += ' --random_distribution=%s' % self.random_distribution
if self.log_avg_msec is not None:
fio_cmd += ' --log_avg_msec=%s' % self.log_avg_msec
fio_cmd += ' %s > %s' % (self.names, out_file)
# Run the backfill testing thread if requested
if 'recovery_test' in self.cluster.config:
recovery_callback = self.recovery_callback
self.cluster.create_recovery_test(self.run_dir, recovery_callback)
print 'Running rbd fio %s test.' % self.mode
common.pdsh(settings.getnodes('clients'), fio_cmd).communicate()
# If we were doing recovery, wait until it's done.
if 'recovery_test' in self.cluster.config:
self.cluster.wait_recovery_done()
monitoring.stop(self.run_dir)
# Finally, get the historic ops
self.cluster.dump_historic_ops(self.run_dir)
common.sync_files('%s/*' % self.run_dir, self.out_dir)
def initialize(self):
super(Radosbench, self).initialize()
logger.info('Pausing for 60s for idle monitoring.')
with monitoring.monitor("%s/idle_monitoring" % self.run_dir):
time.sleep(60)
common.sync_files('%s/*' % self.run_dir, self.out_dir)
def run(self):
super(LibrbdFio, self).run()
# We'll always drop caches for rados bench
self.dropcaches()
# dump the cluster config
self.cluster.dump_config(self.run_dir)
monitoring.start(self.run_dir)
time.sleep(5)
out_file = '%s/output' % self.run_dir
fio_cmd = 'sudo %s --ioengine=rbd --clientname=admin --pool=%s --rbdname=cbt-librbdfio-`hostname -s` --invalidate=0' % (self.cmd_path_full, self.poolname)
fio_cmd += ' --rw=%s' % self.mode
if (self.mode == 'readwrite' or self.mode == 'randrw'):
fio_cmd += ' --rwmixread=%s --rwmixwrite=%s' % (self.rwmixread, self.rwmixwrite)
# fio_cmd += ' --ioengine=%s' % self.ioengine
if self.time is not None:
fio_cmd += ' --runtime=%s' % self.time
if self.ramp is not None:
fio_cmd += ' --ramp_time=%s' % self.ramp
fio_cmd += ' --numjobs=%s' % self.numjobs
fio_cmd += ' --direct=1'
fio_cmd += ' --bs=%dB' % self.op_size
fio_cmd += ' --iodepth=%d' % self.iodepth
fio_cmd += ' --end_fsync=%s' % self.end_fsync
# if self.vol_size:
# fio_cmd += ' -- size=%dM' % self.vol_size
fio_cmd += ' --write_iops_log=%s' % out_file
fio_cmd += ' --write_bw_log=%s' % out_file
fio_cmd += ' --write_lat_log=%s' % out_file
if 'recovery_test' in self.cluster.config:
fio_cmd += ' --time_based'
if self.random_distribution is not None:
fio_cmd += ' --random_distribution=%s' % self.random_distribution
if self.log_avg_msec is not None:
fio_cmd += ' --log_avg_msec=%s' % self.log_avg_msec
fio_cmd += ' %s > %s' % (self.names, out_file)
# Run the backfill testing thread if requested
if 'recovery_test' in self.cluster.config:
recovery_callback = self.recovery_callback
self.cluster.create_recovery_test(self.run_dir, recovery_callback)
print 'Running rbd fio %s test.' % self.mode
common.pdsh(settings.getnodes('clients'), fio_cmd).communicate()
# If we were doing recovery, wait until it's done.
if 'recovery_test' in self.cluster.config:
self.cluster.wait_recovery_done()
monitoring.stop(self.run_dir)
# Finally, get the historic ops
self.cluster.dump_historic_ops(self.run_dir)
common.sync_files('%s/*' % self.run_dir, self.out_dir)
def _run(self, mode, run_dir, out_dir):
# We'll always drop caches for rados bench
#self.dropcaches()
if self.concurrent_ops:
concurrent_ops_str = '--concurrent-ios %s' % self.concurrent_ops
#determine rados version
#rados_version_str = subprocess.check_output(["rados", "-v"])
rados_version_str, err = common.pdsh(
settings.getnodes('head'), '/usr/bin/rados -v').communicate()
m = re.findall("version (\d+)", rados_version_str)
rados_version = int(m[0])
if mode in ['write'] or rados_version < 9:
op_size_str = '-b %s' % self.op_size
else:
op_size_str = ''
common.make_remote_dir(run_dir)
# dump the cluster config
self.cluster.dump_config(run_dir)
# Run the backfill testing thread if requested
if 'recovery_test' in self.cluster.config:
recovery_callback = self.recovery_callback
self.cluster.create_recovery_test(run_dir, recovery_callback)
# Run rados bench
monitoring.start(run_dir)
logger.info('Running radosbench %s test.' % mode)
ps = []
for i in xrange(self.concurrent_procs):
out_file = '%s/output.%s' % (run_dir, i)
objecter_log = '%s/objecter.%s.log' % (run_dir, i)
# default behavior is to use a single storage pool
pool_name = self.pool
run_name = '--run-name %s`hostname -s`-%s' % (self.object_set_id,
i)
if self.pool_per_proc: # support previous behavior of 1 storage pool per rados process
pool_name = 'rados-bench-`hostname -s`-%s' % i
run_name = ''
rados_bench_cmd = '%s -c %s -p %s bench %s %s %s %s %s --no-cleanup 2> %s > %s' % \
(self.cmd_path_full, self.tmp_conf, pool_name, op_size_str, self.time, mode, concurrent_ops_str, run_name, objecter_log, out_file)
p = common.pdsh(settings.getnodes('clients'), rados_bench_cmd)
ps.append(p)
for p in ps:
p.wait()
monitoring.stop(run_dir)
# If we were doing recovery, wait until it's done.
if 'recovery_test' in self.cluster.config:
self.cluster.wait_recovery_done()
# Finally, get the historic ops
self.cluster.dump_historic_ops(run_dir)
common.sync_files('%s/*' % run_dir, out_dir)
def initialize(self):
super(Cosbench, self).initialize()
logger.debug('Running cosbench and radosgw check.')
self.prerun_check()
logger.debug('Pausing for 60s for idle monitoring.')
monitoring.start("%s/idle_monitoring" % self.run_dir)
time.sleep(60)
monitoring.stop()
common.sync_files('%s' % self.run_dir, self.out_dir)
# Create the run directory
common.make_remote_dir(self.run_dir)
conf = self.config
if not self.config["template"]:
self.config["template"] = "default"
self.config["workload"] = self.choose_template("default", conf)
# add a "prepare" stage if mode is read or mix
if not self.container_prepare_check():
workstage_init = {
"name": "init",
"work": {
"type":
"init",
"workers":
conf["workers"],
"config":
"containers=r(1,%s);cprefix=%s-%s-%s" %
(conf["containers_max"], conf["obj_size"], conf["mode"],
conf["objects_max"])
}
}
workstage_prepare = {
"name": "prepare",
"work": {
"type":
"prepare",
"workers":
conf["workers"],
"config":
"containers=r(1,%s);objects=r(1,%s);cprefix=%s-%s-%s;sizes=c(%s)%s"
% (conf["containers_max"], conf["objects_max"],
conf["obj_size"], conf["mode"], conf["objects_max"],
conf["obj_size_num"], conf["obj_size_unit"])
}
}
self.config["workload"]["workflow"]["workstage"].insert(
0, workstage_prepare)
self.config["workload"]["workflow"]["workstage"].insert(
0, workstage_init)
self.prepare_xml(self.config["workload"])
return True
def run(self):
super(StdFioBench, self).run()
# Set client readahead
self.set_client_param('read_ahead_kb', self.client_ra)
# We'll always drop caches for rados bench
self.dropcaches()
monitoring.start(self.run_dir)
time.sleep(5)
out_file = '%s/output' % self.run_dir
fio_cmd = 'sudo %s' % self.fio_cmd
fio_cmd += ' --rw=%s' % self.mode
if (self.mode == 'readwrite' or self.mode == 'randrw'):
fio_cmd += ' --rwmixread=%s --rwmixwrite=%s' % (self.rwmixread,
self.rwmixwrite)
fio_cmd += ' --ioengine=%s' % self.ioengine
fio_cmd += ' --runtime=%s' % self.time
fio_cmd += ' --ramp_time=%s' % self.ramp
fio_cmd += ' --numjobs=%s' % self.numjobs
fio_cmd += ' --direct=1'
fio_cmd += ' --randrepeat=0'
fio_cmd += ' --group_reporting'
fio_cmd += ' --bs=%dB' % self.op_size
fio_cmd += ' --iodepth=%d' % self.iodepth
fio_cmd += ' --size=%dM' % self.vol_size
fio_cmd += ' --output-format=%s' % self.output_format
if (self.output_format == 'normal'):
fio_cmd += ' --write_iops_log=%s' % out_file
fio_cmd += ' --write_bw_log=%s' % out_file
fio_cmd += ' --write_lat_log=%s' % out_file
if 'recovery_test' in self.cluster.config:
fio_cmd += ' --time_based'
fio_cmd += ' %s > %s 2> %s/error_log' % (self.names, out_file,
self.run_dir)
# Run the backfill testing thread if requested
if 'recovery_test' in self.cluster.config:
recovery_callback = self.recovery_callback
self.cluster.create_recovery_test(self.run_dir, recovery_callback)
logger.info('Running fio %s test.', self.mode)
common.pdsh(settings.getnodes('clients'), fio_cmd).communicate()
# FIO output Parsing logic
if (self.output_format == 'terse'):
hostname = '`hostname -s`'
parse_cmd = 'sudo sed "s/$/;%s;%s;%s;%s;%s;%s;%s/" ' % (
hostname, self.mode, self.op_size, self.iodepth, self.numjobs,
self.client_ra, self.concurrent_procs)
parse_cmd += ' %s > %s/terse_output' % (out_file, self.run_dir)
common.pdsh(settings.getnodes('clients'), parse_cmd).communicate()
monitoring.stop(self.run_dir)
common.sync_files('%s/*' % self.run_dir, self.out_dir)
def run(self):
super(RawFio, self).run()
# Set client readahead
clnts = settings.getnodes('clients')
# We'll always drop caches for rados bench
self.dropcaches()
monitoring.start(self.run_dir)
time.sleep(5)
logger.info('Starting raw fio %s test.', self.mode)
fio_process_list = []
for i in range(self.concurrent_procs):
b = self.block_devices[i % len(self.block_devices)]
fiopath = b
out_file = '%s/output.%d' % (self.run_dir, i)
fio_cmd = 'sudo %s' % self.fio_cmd
fio_cmd += ' --rw=%s' % self.mode
if (self.mode == 'readwrite' or self.mode == 'randrw'):
fio_cmd += ' --rwmixread=%s --rwmixwrite=%s' % (
self.rwmixread, self.rwmixwrite)
fio_cmd += ' --ioengine=%s' % self.ioengine
fio_cmd += ' --runtime=%s' % self.time
fio_cmd += ' --ramp_time=%s' % self.ramp
if self.startdelay:
fio_cmd += ' --startdelay=%s' % self.startdelay
if self.rate_iops:
fio_cmd += ' --rate_iops=%s' % self.rate_iops
fio_cmd += ' --numjobs=%s' % self.numjobs
fio_cmd += ' --direct=%s' % self.direct
fio_cmd += ' --bs=%dB' % self.op_size
fio_cmd += ' --iodepth=%d' % self.iodepth
fio_cmd += ' --size=%dM' % self.vol_size
if self.log_iops:
fio_cmd += ' --write_iops_log=%s' % out_file
if self.log_bw:
fio_cmd += ' --write_bw_log=%s' % out_file
if self.log_lat:
fio_cmd += ' --write_lat_log=%s' % out_file
fio_cmd += ' --output-format=%s' % self.fio_out_format
if 'recovery_test' in self.cluster.config:
fio_cmd += ' --time_based'
fio_cmd += ' --name=%s > %s' % (fiopath, out_file)
logger.debug("FIO CMD: %s" % fio_cmd)
fio_process_list.append(
common.pdsh(clnts, fio_cmd, continue_if_error=False))
for p in fio_process_list:
p.communicate()
monitoring.stop(self.run_dir)
logger.info('Finished raw fio test')
common.sync_files('%s/*' % self.run_dir, self.out_dir)
def run(self):
super(KvmRbdFio, self).run()
# Set client readahead
self.set_client_param('read_ahead_kb', self.client_ra)
clnts = settings.getnodes('clients')
# We'll always drop caches for rados bench
self.dropcaches()
monitoring.start(self.run_dir)
time.sleep(5)
# Run the backfill testing thread if requested
if 'recovery_test' in self.cluster.config:
recovery_callback = self.recovery_callback
self.cluster.create_recovery_test(self.run_dir, recovery_callback)
logger.info('Starting rbd fio %s test.', self.mode)
fio_process_list = []
for i in range(self.concurrent_procs):
b = self.block_devices[i % len(self.block_devices)]
bnm = os.path.basename(b)
mtpt = '/srv/rbdfio-`hostname -s`-%s' % bnm
fiopath = os.path.join(mtpt, 'fio%d.img' % i)
out_file = '%s/output.%d' % (self.run_dir, i)
fio_cmd = 'sudo %s' % self.fio_cmd
fio_cmd += ' --rw=%s' % self.mode
if (self.mode == 'readwrite' or self.mode == 'randrw'):
fio_cmd += ' --rwmixread=%s --rwmixwrite=%s' % (self.rwmixread, self.rwmixwrite)
fio_cmd += ' --ioengine=%s' % self.ioengine
fio_cmd += ' --runtime=%s' % self.time
fio_cmd += ' --ramp_time=%s' % self.ramp
if self.startdelay:
fio_cmd += ' --startdelay=%s' % self.startdelay
if self.rate_iops:
fio_cmd += ' --rate_iops=%s' % self.rate_iops
fio_cmd += ' --numjobs=%s' % self.numjobs
fio_cmd += ' --direct=1'
fio_cmd += ' --bs=%dB' % self.op_size
fio_cmd += ' --iodepth=%d' % self.iodepth
fio_cmd += ' --size=%dM' % self.vol_size
fio_cmd += ' --write_iops_log=%s' % out_file
fio_cmd += ' --write_bw_log=%s' % out_file
fio_cmd += ' --write_lat_log=%s' % out_file
if 'recovery_test' in self.cluster.config:
fio_cmd += ' --time_based'
fio_cmd += ' --name=%s > %s' % (fiopath, out_file)
fio_process_list.append(common.pdsh(clnts, fio_cmd, continue_if_error=False))
for p in fio_process_list:
p.communicate()
monitoring.stop(self.run_dir)
logger.info('Finished rbd fio test')
common.sync_files('%s/*' % self.run_dir, self.out_dir)
def run(self):
super(RbdFio, self).run()
# Set client readahead
self.set_client_param('read_ahead_kb', self.client_ra)
# We'll always drop caches for rados bench
self.dropcaches()
monitoring.start(self.run_dir)
# Run the backfill testing thread if requested
if 'recovery_test' in self.cluster.config:
recovery_callback = self.recovery_callback
self.cluster.create_recovery_test(self.run_dir, recovery_callback)
time.sleep(5)
out_file = '%s/output' % self.run_dir
fio_cmd = 'sudo %s' % (self.cmd_path_full)
fio_cmd += ' --rw=%s' % self.mode
if (self.mode == 'readwrite' or self.mode == 'randrw'):
fio_cmd += ' --rwmixread=%s --rwmixwrite=%s' % (self.rwmixread, self.rwmixwrite)
fio_cmd += ' --ioengine=%s' % self.ioengine
if self.time is not None:
fio_cmd += ' --runtime=%s' % self.time
if self.ramp is not None:
fio_cmd += ' --ramp_time=%s' % self.ramp
fio_cmd += ' --numjobs=%s' % self.numjobs
fio_cmd += ' --direct=1'
fio_cmd += ' --bs=%dB' % self.op_size
fio_cmd += ' --iodepth=%d' % self.iodepth
if self.vol_size:
fio_cmd += ' --size=%dM' % (int(self.vol_size) * 0.9)
fio_cmd += ' --write_iops_log=%s' % out_file
fio_cmd += ' --write_bw_log=%s' % out_file
fio_cmd += ' --write_lat_log=%s' % out_file
if 'recovery_test' in self.cluster.config:
fio_cmd += ' --time_based'
if self.random_distribution is not None:
fio_cmd += ' --random_distribution=%s' % self.random_distribution
fio_cmd += ' %s > %s' % (self.names, out_file)
if self.log_avg_msec is not None:
fio_cmd += ' --log_avg_msec=%s' % self.log_avg_msec
logger.info('Running rbd fio %s test.', self.mode)
common.pdsh(settings.getnodes('clients'), fio_cmd).communicate()
# If we were doing recovery, wait until it's done.
if 'recovery_test' in self.cluster.config:
self.cluster.wait_recovery_done()
monitoring.stop(self.run_dir)
# Finally, get the historic ops
self.cluster.dump_historic_ops(self.run_dir)
common.sync_files('%s/*' % self.run_dir, self.out_dir)
def _run(self, mode, run_dir, out_dir):
# We'll always drop caches for rados bench
self.dropcaches()
if self.concurrent_ops:
concurrent_ops_str = '--concurrent-ios %s' % self.concurrent_ops
#determine rados version
rados_version_str, err = common.pdsh(settings.getnodes('head'), '/usr/bin/rados -v').communicate()
m = re.findall("version (\d+)", rados_version_str)
rados_version = int(m[0])
if mode in ['write'] or rados_version < 9:
op_size_str = '-b %s' % self.op_size
else:
op_size_str = ''
common.make_remote_dir(run_dir)
# dump the cluster config
self.cluster.dump_config(run_dir)
# Run the backfill testing thread if requested
if 'recovery_test' in self.cluster.config:
recovery_callback = self.recovery_callback
self.cluster.create_recovery_test(run_dir, recovery_callback)
# Run rados bench
monitoring.start(run_dir)
logger.info('Running radosbench %s test.' % mode)
ps = []
for i in xrange(self.concurrent_procs):
out_file = '%s/output.%s' % (run_dir, i)
objecter_log = '%s/objecter.%s.log' % (run_dir, i)
# default behavior is to use a single storage pool
pool_name = self.pool
run_name = '--run-name %s`hostname -s`-%s'%(self.object_set_id, i)
if self.pool_per_proc: # support previous behavior of 1 storage pool per rados process
pool_name = 'rados-bench-`hostname -s`-%s'%i
run_name = ''
rados_bench_cmd = '%s -c %s -p %s bench %s %s %s %s %s --no-cleanup 2> %s > %s' % \
(self.cmd_path_full, self.tmp_conf, pool_name, op_size_str, self.time, mode, concurrent_ops_str, run_name, objecter_log, out_file)
p = common.pdsh(settings.getnodes('clients'), rados_bench_cmd)
ps.append(p)
for p in ps:
p.wait()
monitoring.stop(run_dir)
# If we were doing recovery, wait until it's done.
if 'recovery_test' in self.cluster.config:
self.cluster.wait_recovery_done()
# Finally, get the historic ops
self.cluster.dump_historic_ops(run_dir)
common.sync_files('%s/*' % run_dir, out_dir)
def initialize(self):
super(Radosbench, self).initialize()
logger.info('Pausing for 60s for idle monitoring.')
monitoring.start("%s/idle_monitoring" % self.run_dir)
time.sleep(60)
monitoring.stop()
common.sync_files('%s/*' % self.run_dir, self.out_dir)
return True
def initialize(self):
super(Radosbench, self).initialize()
logger.info('Pausing for 60s for idle monitoring.')
monitoring.start("%s/idle_monitoring" % self.run_dir)
time.sleep(60)
monitoring.stop()
common.sync_files('%s/*' % self.run_dir, self.out_dir)
return True
def run(self):
super(RawFio, self).run()
# Set client readahead
clnts = settings.getnodes('clients')
# We'll always drop caches for rados bench
self.dropcaches()
monitoring.start(self.run_dir)
time.sleep(5)
logger.info('Starting raw fio %s test.', self.mode)
fio_process_list = []
for i in range(self.concurrent_procs):
b = self.block_devices[i % len(self.block_devices)]
fiopath = b
out_file = '%s/output.%d' % (self.run_dir, i)
fio_cmd = 'sudo %s' % self.fio_cmd
fio_cmd += ' --rw=%s' % self.mode
if (self.mode == 'readwrite' or self.mode == 'randrw'):
fio_cmd += ' --rwmixread=%s --rwmixwrite=%s' % (self.rwmixread, self.rwmixwrite)
fio_cmd += ' --ioengine=%s' % self.ioengine
fio_cmd += ' --runtime=%s' % self.time
fio_cmd += ' --ramp_time=%s' % self.ramp
if self.startdelay:
fio_cmd += ' --startdelay=%s' % self.startdelay
if self.rate_iops:
fio_cmd += ' --rate_iops=%s' % self.rate_iops
fio_cmd += ' --numjobs=%s' % self.numjobs
fio_cmd += ' --direct=%s' % self.direct
fio_cmd += ' --bs=%dB' % self.op_size
fio_cmd += ' --iodepth=%d' % self.iodepth
fio_cmd += ' --size=%dM' % self.vol_size
fio_cmd += ' --write_iops_log=%s' % out_file
fio_cmd += ' --write_bw_log=%s' % out_file
fio_cmd += ' --write_lat_log=%s' % out_file
fio_cmd += ' --output-format=%s' % self.fio_out_format
if 'recovery_test' in self.cluster.config:
fio_cmd += ' --time_based'
fio_cmd += ' --name=%s > %s' % (fiopath, out_file)
logger.debug("FIO CMD: %s" % fio_cmd)
fio_process_list.append(common.pdsh(clnts, fio_cmd, continue_if_error=False))
for p in fio_process_list:
p.communicate()
monitoring.stop(self.run_dir)
logger.info('Finished raw fio test')
common.sync_files('%s/*' % self.run_dir, self.out_dir)
def initialize(self):
super(Radosbench, self).initialize()
print 'Running scrub monitoring.'
monitoring.start("%s/scrub_monitoring" % self.run_dir)
self.cluster.check_scrub()
monitoring.stop()
print 'Pausing for 60s for idle monitoring.'
monitoring.start("%s/idle_monitoring" % self.run_dir)
time.sleep(60)
monitoring.stop()
common.sync_files('%s/*' % self.run_dir, self.out_dir)
return True
def initialize(self):
super(Radosbench, self).initialize()
print 'Running scrub monitoring.'
monitoring.start("%s/scrub_monitoring" % self.run_dir)
self.cluster.check_scrub()
monitoring.stop()
print 'Pausing for 60s for idle monitoring.'
monitoring.start("%s/idle_monitoring" % self.run_dir)
time.sleep(60)
monitoring.stop()
common.sync_files('%s/*' % self.run_dir, self.out_dir)
return True
def initialize(self):
super(Getput, self).initialize()
# create the user and key
self.cluster.add_swift_user(self.user, self.subuser, self.key)
# Clean and Create the run directory
common.clean_remote_dir(self.run_dir)
common.make_remote_dir(self.run_dir)
logger.info('Pausing for 60s for idle monitoring.')
monitoring.start("%s/idle_monitoring" % self.run_dir)
time.sleep(60)
monitoring.stop()
common.sync_files('%s/*' % self.run_dir, self.out_dir)
def _run(self, mode, run_dir, out_dir):
# We'll always drop caches for rados bench
self.dropcaches()
if self.concurrent_ops:
concurrent_ops_str = '--concurrent-ios %s' % self.concurrent_ops
op_size_str = '-b %s' % self.op_size
common.make_remote_dir(run_dir)
# dump the cluster config
self.cluster.dump_config(run_dir)
# Run the backfill testing thread if requested
if 'recovery_test' in self.cluster.config:
recovery_callback = self.recovery_callback
self.cluster.create_recovery_test(run_dir, recovery_callback)
# Run rados bench
monitoring.start(run_dir)
logger.info('Running radosbench read test.')
ps = []
for i in xrange(self.concurrent_procs):
out_file = '%s/output.%s' % (run_dir, i)
objecter_log = '%s/objecter.%s.log' % (run_dir, i)
# default behavior is to use a single storage pool
pool_name = 'rados-bench-cbt'
run_name = '--run-name `hostname -s`-%s' % i
if self.pool_per_proc: # support previous behavior of 1 storage pool per rados process
pool_name = 'rados-bench-`hostname -s`-%s' % i
run_name = ''
rados_bench_cmd = '%s -c %s -p %s bench %s %s %s %s %s --no-cleanup 2> %s > %s' % \
(self.cmd_path_full, self.tmp_conf, pool_name, op_size_str, self.time, mode, concurrent_ops_str, run_name, objecter_log, out_file)
p = common.pdsh(settings.getnodes('clients'), rados_bench_cmd)
ps.append(p)
for p in ps:
p.wait()
monitoring.stop(run_dir)
# If we were doing recovery, wait until it's done.
if 'recovery_test' in self.cluster.config:
self.cluster.wait_recovery_done()
# Finally, get the historic ops
self.cluster.dump_historic_ops(run_dir)
common.sync_files('%s/*' % run_dir, out_dir)
def initialize(self):
super(Cosbench, self).initialize()
logger.debug('Running cosbench and radosgw check.')
self.prerun_check()
logger.debug('Running scrub monitoring.')
monitoring.start("%s/scrub_monitoring" % self.run_dir)
self.cluster.check_scrub()
monitoring.stop()
logger.debug('Pausing for 60s for idle monitoring.')
monitoring.start("%s/idle_monitoring" % self.run_dir)
time.sleep(60)
monitoring.stop()
common.sync_files('%s' % self.run_dir, self.out_dir)
# Create the run directory
common.make_remote_dir(self.run_dir)
conf = self.config
if not self.config["template"]:
self.config["template"] = "default"
self.config["workload"] = self.choose_template("default", conf)
# add a "prepare" stage if mode is read or mix
if not self.container_prepare_check():
workstage_init = {
"name": "init",
"work": {"type":"init", "workers":conf["workers"], "config":"containers=r(1,%s);cprefix=%s-%s-%s" % (conf["containers_max"], conf["obj_size"], conf["mode"], conf["objects_max"])}
}
workstage_prepare = {
"name":"prepare",
"work": {
"type":"prepare",
"workers":conf["workers"],
"config":"containers=r(1,%s);objects=r(1,%s);cprefix=%s-%s-%s;sizes=c(%s)%s" %
(conf["containers_max"], conf["objects_max"], conf["obj_size"], conf["mode"], conf["objects_max"], conf["obj_size_num"], conf["obj_size_unit"])
}
}
self.config["workload"]["workflow"]["workstage"].insert(0, workstage_prepare)
self.config["workload"]["workflow"]["workstage"].insert(0, workstage_init)
self.prepare_xml(self.config["workload"])
return True
def run(self):
super(LibrbdFio, self).run()
# We'll always drop caches for rados bench
self.dropcaches()
# dump the cluster config
self.cluster.dump_config(self.run_dir)
monitoring.start(self.run_dir)
time.sleep(5)
# If the pg autoscaler kicks in before starting the test,
# wait for it to complete. Otherwise, results may be skewed.
ret = self.cluster.check_pg_autoscaler(
self.wait_pgautoscaler_timeout,
"%s/pgautoscaler.log" % self.run_dir)
if ret == 1:
logger.warn(
"PG autoscaler taking longer to complete. Continuing anyway...results may be skewed."
)
# Run the backfill testing thread if requested
if 'recovery_test' in self.cluster.config:
recovery_callback = self.recovery_callback
self.cluster.create_recovery_test(self.run_dir, recovery_callback)
logger.info('Running rbd fio %s test.', self.mode)
ps = []
for i in range(self.volumes_per_client):
fio_cmd = self.mkfiocmd(i)
p = common.pdsh(settings.getnodes('clients'), fio_cmd)
ps.append(p)
for p in ps:
p.wait()
# If we were doing recovery, wait until it's done.
if 'recovery_test' in self.cluster.config:
self.cluster.wait_recovery_done()
monitoring.stop(self.run_dir)
# Finally, get the historic ops
self.cluster.dump_historic_ops(self.run_dir)
common.sync_files('%s/*' % self.run_dir, self.out_dir)
self.analyze(self.out_dir)
def run(self):
super(KvmRbdFio, self).run()
# Set client readahead
self.set_client_param('read_ahead_kb', self.client_ra)
# We'll always drop caches for rados bench
self.dropcaches()
monitoring.start(self.run_dir)
time.sleep(5)
# names = ''
# for i in xrange(self.concurrent_procs):
# names += "--name=/srv/rbdfio-`hostname -s`-%d/cbt-kvmrbdfio " % i
# names += '--name=/srv/rbdfio-`hostname -s`-0/cbt-kvmrbdfio-%d ' % i
out_file = '%s/output' % self.run_dir
# pre_cmd = 'sudo fio --rw=write -ioengine=sync --numjobs=%s --bs=4M --size %dM %s > /dev/null' % (self.numjobs, self.vol_size, self.names)
fio_cmd = 'sudo %s' % self.fio_cmd
fio_cmd += ' --rw=%s' % self.mode
if (self.mode == 'readwrite' or self.mode == 'randrw'):
fio_cmd += ' --rwmixread=%s --rwmixwrite=%s' % (self.rwmixread, self.rwmixwrite)
fio_cmd += ' --ioengine=%s' % self.ioengine
fio_cmd += ' --runtime=%s' % self.time
fio_cmd += ' --ramp_time=%s' % self.ramp
fio_cmd += ' --numjobs=%s' % self.numjobs
fio_cmd += ' --direct=1'
fio_cmd += ' --bs=%dB' % self.op_size
fio_cmd += ' --iodepth=%d' % self.iodepth
fio_cmd += ' --size=%dM' % self.vol_size
fio_cmd += ' --write_iops_log=%s' % out_file
fio_cmd += ' --write_bw_log=%s' % out_file
fio_cmd += ' --write_lat_log=%s' % out_file
if 'recovery_test' in self.cluster.config:
fio_cmd += ' --time_based'
fio_cmd += ' %s > %s' % (self.names, out_file)
# Run the backfill testing thread if requested
if 'recovery_test' in self.cluster.config:
recovery_callback = self.recovery_callback
self.cluster.create_recovery_test(self.run_dir, recovery_callback)
logger.info('Running rbd fio %s test.', self.mode)
common.pdsh(settings.getnodes('clients'), fio_cmd).communicate()
monitoring.stop(self.run_dir)
common.sync_files('%s/*' % self.run_dir, self.out_dir)
def initialize(self):
super(Getput, self).initialize()
# create the user and key
self.cluster.add_swift_user(self.user, self.subuser, self.key)
# Clean and Create the run directory
common.clean_remote_dir(self.run_dir)
common.make_remote_dir(self.run_dir)
logger.info('Pausing for 60s for idle monitoring.')
monitoring.start("%s/idle_monitoring" % self.run_dir)
time.sleep(60)
monitoring.stop()
common.sync_files('%s/*' % self.run_dir, self.out_dir)
return True
def _run(self, mode, run_dir, out_dir):
# We'll always drop caches for rados bench
self.dropcaches()
if self.concurrent_ops:
concurrent_ops_str = '--concurrent-ios %s' % self.concurrent_ops
op_size_str = '-b %s' % self.op_size
common.make_remote_dir(run_dir)
# dump the cluster config
self.cluster.dump_config(run_dir)
# Run the backfill testing thread if requested
if 'recovery_test' in self.cluster.config:
recovery_callback = self.recovery_callback
self.cluster.create_recovery_test(run_dir, recovery_callback)
# Run rados bench
monitoring.start(run_dir)
print 'Running radosbench read test.'
ps = []
for i in xrange(self.concurrent_procs):
out_file = '%s/output.%s' % (run_dir, i)
objecter_log = '%s/objecter.%s.log' % (run_dir, i)
p = common.pdsh(
settings.getnodes('clients'),
'%s -c %s -p rados-bench-`hostname -s`-%s %s bench %s %s %s --no-cleanup 2> %s > %s'
%
(self.cmd_path_full, self.tmp_conf, i, op_size_str, self.time,
mode, concurrent_ops_str, objecter_log, out_file))
ps.append(p)
for p in ps:
p.wait()
monitoring.stop(run_dir)
# If we were doing recovery, wait until it's done.
if 'recovery_test' in self.cluster.config:
self.cluster.wait_recovery_done()
# Finally, get the historic ops
self.cluster.dump_historic_ops(run_dir)
common.sync_files('%s/*' % run_dir, out_dir)
def run(self):
super(Cosbench, self).run()
self.dropcaches()
self.cluster.dump_config(self.run_dir)
monitoring.start(self.run_dir)
# Run cosbench test
try:
self._run()
except KeyboardInterrupt:
logger.warning("accept keyboard interrupt, cancel this run")
conf = self.config
stdout, stderr = common.pdsh("%s@%s" % (self.user, conf["controller"]),'sh %s/cli.sh cancel %s' % (conf["cosbench_dir"], self.runid)).communicate()
logger.info("%s", stdout)
self.check_workload_status()
self.check_cosbench_res_dir()
monitoring.stop(self.run_dir)
self.cluster.dump_historic_ops(self.run_dir)
common.sync_files('%s/*' % self.run_dir, self.out_dir)
def _run(self, mode, run_dir, out_dir):
# We'll always drop caches for rados bench
self.dropcaches()
if self.concurrent_ops:
concurrent_ops_str = '--concurrent-ios %s' % self.concurrent_ops
op_size_str = '-b %s' % self.op_size
common.make_remote_dir(run_dir)
# dump the cluster config
self.cluster.dump_config(run_dir)
# Run the backfill testing thread if requested
if 'recovery_test' in self.cluster.config:
recovery_callback = self.recovery_callback
self.cluster.create_recovery_test(run_dir, recovery_callback)
# Run rados bench
monitoring.start(run_dir)
print 'Running radosbench read test.'
ps = []
for i in xrange(self.concurrent_procs):
out_file = '%s/output.%s' % (run_dir, i)
objecter_log = '%s/objecter.%s.log' % (run_dir, i)
p = common.pdsh(settings.getnodes('clients'), '%s -c %s -p rados-bench-`hostname -s`-%s %s bench %s %s %s --no-cleanup 2> %s > %s' % (self.cmd_path_full, self.tmp_conf, i, op_size_str, self.time, mode, concurrent_ops_str, objecter_log, out_file))
ps.append(p)
for p in ps:
p.wait()
monitoring.stop(run_dir)
# If we were doing recovery, wait until it's done.
if 'recovery_test' in self.cluster.config:
self.cluster.wait_recovery_done()
# Finally, get the historic ops
self.cluster.dump_historic_ops(run_dir)
common.sync_files('%s/*' % run_dir, out_dir)
def _run(self, mode, run_dir, out_dir):
# We'll always drop caches for rados bench
self.dropcaches()
if self.concurrent_ops:
concurrent_ops_str = '--concurrent-ios %s' % self.concurrent_ops
#determine rados version
rados_version_str = self.get_rados_version()
m = re.findall("version (\d+)", rados_version_str)
if not m:
m = re.findall("version v(\d+)", rados_version_str)
rados_version = int(m[0])
if mode in ['write'] or rados_version < 9:
op_size_str = '-b %s' % self.op_size
else:
op_size_str = ''
# Max Objects
max_objects_str = ''
if self.max_objects and rados_version < 9:
raise ValueError('max_objects not supported by rados_version < 9')
if self.max_objects and rados_version > 9:
max_objects_str = '--max-objects %s' % self.max_objects
# Write to OMAP
write_omap_str = ''
if self.write_omap and rados_version < 9:
raise ValueError('write_omap not supported by rados_version < 9')
if self.write_omap and rados_version > 9:
write_omap_str = '--write-omap'
common.make_remote_dir(run_dir)
# dump the cluster config
self.cluster.dump_config(run_dir)
# Run the backfill testing thread if requested
if 'recovery_test' in self.cluster.config:
recovery_callback = self.recovery_callback
self.cluster.create_recovery_test(run_dir, recovery_callback)
# Run rados bench
monitoring.start(run_dir)
logger.info('Running radosbench %s test.' % mode)
ps = []
for i in xrange(self.concurrent_procs):
out_file = '%s/output.%s' % (run_dir, i)
objecter_log = '%s/objecter.%s.log' % (run_dir, i)
# default behavior is to use a single storage pool
pool_name = self.pool
run_name = '--run-name %s`%s`-%s'%(self.object_set_id, common.get_fqdn_cmd(), i)
if self.pool_per_proc: # support previous behavior of 1 storage pool per rados process
pool_name = 'rados-bench-``-%s'% (common.get_fqdn_cmd(), i)
run_name = ''
rados_bench_cmd = '%s -c %s -p %s bench %s %s %s %s %s %s %s --no-cleanup 2> %s > %s' % \
(self.cmd_path_full, self.tmp_conf, pool_name, op_size_str, self.time, mode, concurrent_ops_str, max_objects_str, write_omap_str, run_name, objecter_log, out_file)
p = common.pdsh(settings.getnodes('clients'), rados_bench_cmd)
ps.append(p)
for p in ps:
p.wait()
monitoring.stop(run_dir)
# If we were doing recovery, wait until it's done.
if 'recovery_test' in self.cluster.config:
self.cluster.wait_recovery_done()
# Finally, get the historic ops
self.cluster.dump_historic_ops(run_dir)
common.sync_files('%s/*' % run_dir, out_dir)
self.analyze(out_dir)
def initialize(self):
super(MysqlSysBench, self).initialize()
if not self.no_rbd:
print 'Running scrub monitoring.'
monitoring.start("%s/scrub_monitoring" % self.run_dir)
self.cluster.check_scrub()
monitoring.stop()
print 'Pausing for 60s for idle monitoring.'
monitoring.start("%s/idle_monitoring" % self.run_dir)
time.sleep(60)
monitoring.stop()
# Create the run directory
common.make_remote_dir(self.run_dir)
# Create the out directory
common.make_remote_dir(self.out_dir)
common.sync_files('%s/*' % self.run_dir, self.out_dir)
self.mkimages()
if not self.no_create_db:
# Initialize the datadir
print 'Running mysql_install_db.'
mysql_install_cmd = 'sudo /usr/bin/mysql_install_db --no-defaults --datadir=%s --user=mysql --force > %s/mysql_install.out 2> %s/mysqlinstall.err ' % (self.mysql_datadir,self.out_dir,self.out_dir)
common.pdsh(settings.getnodes('clients'), mysql_install_cmd).communicate()
time.sleep(5)
# Starting MySQL on all nodes
print 'Starting MySQL'
mysql_cmd = 'sudo chmod 777 %s; ' % self.out_dir
mysql_cmd += 'sudo /usr/sbin/mysqld --no-defaults --user=mysql --datadir=%s ' % self.mysql_datadir
mysql_cmd += '--pid-file=/tmp/mysqlsysbench.pid '
mysql_cmd += '--innodb-buffer-pool-size=%s ' % self.mycnf_innodb_buffer_pool_size
mysql_cmd += '--innodb-log-file-size=%s ' % self.mycnf_innodb_log_file_size
mysql_cmd += '--innodb-log-buffer-size=%s ' % self.mycnf_innodb_log_buffer_size
mysql_cmd += '--innodb-read-io-threads=%s ' % self.mycnf_innodb_read_io_threads
mysql_cmd += '--innodb-write-io-threads=%s ' % self.mycnf_innodb_write_io_threads
mysql_cmd += '--innodb-purge-threads=%s ' % self.mycnf_innodb_purge_threads
mysql_cmd += '--innodb-doublewrite=%s ' % self.mycnf_innodb_doublewrite
mysql_cmd += '--innodb-file-format=%s ' % self.mycnf_innodb_file_format
mysql_cmd += '--innodb-flush-method=%s ' % self.mycnf_innodb_flush_method
mysql_cmd += '--innodb-flush-log-at-trx-commit=%s ' % self.mycnf_innodb_flush_log_at_trx_commit
#mysql_cmd += '--innodb-flush-neighbors=%s ' % self.mycnf_innodb_flush_neighbors # only for percona server
mysql_cmd += '--log-error=%s/mysqld.log ' % self.out_dir
mysql_cmd += '--socket=%s ' % self.mysql_socket
mysql_cmd += '--skip-networking '
mysql_cmd += '--query-cache-size=0 '
mysql_cmd += '--innodb-file-per-table '
mysql_cmd += '--skip-performance-schema '
mysql_cmd += ' > %s/mysql_start.out 2> %s/mysql_start.err ' % (self.out_dir,self.out_dir)
mysql_cmd += '&'
common.pdsh(settings.getnodes('clients'), mysql_cmd).communicate()
#give it time to start up
print 'Waiting for 60s for mysql to start...'
time.sleep(60)
# Create the sysbench tables
print 'Creating the Sysbench database...'
mysql_cmd = '/usr/bin/mysql -e "create database sbtest;" '
mysql_cmd += '-u root '
mysql_cmd += '--socket=%s ' % self.mysql_socket
common.pdsh(settings.getnodes('clients'), mysql_cmd).communicate()
if self.existing_database_is_preloaded = 0:
# Creation of the benchmark tables
print 'Creating the Sysbench tables...'
pre_cmd = '%s ' % self.cmd_path_full
pre_cmd += '--test=%s ' % self.prepare_path
if not self.no_create_db:
pre_cmd += '--mysql-user=root '
else:
pre_cmd += '--mysql-user=%s --mysql-password=%s ' % (self.mysql_user, self.mysql_pass)
pre_cmd += '--mysql-socket=%s ' % self.mysql_socket
pre_cmd += '--mysql-db=%s ' % self.mysql_database
pre_cmd += '--mysql-table-engine=%s ' % self.mysql_engine
pre_cmd += '--oltp-tables-count=%s ' % self.oltp_table_count
pre_cmd += '--oltp-table-size=%s ' % self.oltp_table_size
pre_cmd += '--num-threads=%s run ' % self.threads
pre_cmd += ' > %s/sysbench_prepare.out 2> %s/sysbench_prepare.err ' % (self.out_dir,self.out_dir)
common.pdsh(settings.getnodes('clients'), pre_cmd).communicate()
def run(self):
super(MysqlSysBench, self).run()
# Set client readahead
self.set_client_param('read_ahead_kb', self.client_ra)
# We'll always drop caches for rados bench
self.dropcaches()
monitoring.start(self.run_dir)
# Run the backfill testing thread if requested
if 'recovery_test' in self.cluster.config:
recovery_callback = self.recovery_callback
self.cluster.create_recovery_test(self.run_dir, recovery_callback)
# Let's warmup MySQL
time.sleep(5)
sysbench_cmd = '%s --max-requests=0 ' % (self.cmd_path_full)
sysbench_cmd += '--max-time=%s ' % self.warmup_time
sysbench_cmd += '--num-threads=%s ' % self.threads
sysbench_cmd += '--test=%s ' % self.test_path
if not self.no_create_db:
sysbench_cmd += '-u root '
else:
sysbench_cmd += '--mysql-user=%s --mysql-password=%s ' % (self.mysql_user, self.mysql_pass)
sysbench_cmd += '--mysql-db=%s ' % self.mysql_database
sysbench_cmd += '--mysql-socket=%s ' % self.mysql_socket
sysbench_cmd += '--oltp-tables-count=%s ' % self.oltp_table_count
sysbench_cmd += '--oltp-table-size=%s ' % self.oltp_table_size
sysbench_cmd += '--oltp-read-only=%s ' % self.oltp_read_only
sysbench_cmd += '--oltp-point-select=%s ' % self.oltp_point_selects
sysbench_cmd += '--oltp-range-size=%s ' % self.oltp_range_size
sysbench_cmd += '--oltp-simple-ranges=%s ' % self.oltp_simple_ranges
sysbench_cmd += '--oltp-sum-ranges=%s ' % self.oltp_sum_ranges
sysbench_cmd += '--oltp-order-ranges=%s ' % self.oltp_order_ranges
sysbench_cmd += '--oltp-distinct-ranges=%s ' % self.oltp_distinct_ranges
sysbench_cmd += '--oltp-index-updates=%s ' % self.oltp_index_updates
sysbench_cmd += '--oltp-non-index-updates=%s ' % self.oltp_non_index_updates
sysbench_cmd += 'run '
sysbench_cmd += ' > %s/sysbench_warmup.out 2> %s/sysbench_warmup.err ' % (self.out_dir,self.out_dir)
print 'Running sysbench mysql warmup.'
common.pdsh(settings.getnodes('clients'), sysbench_cmd).communicate()
# Now the real benchmark
sysbench_cmd = '%s --max-requests=0 ' % (self.cmd_path_full)
sysbench_cmd += '--max-time=%s ' % self.max_time
sysbench_cmd += '--num-threads=%s ' % self.threads
sysbench_cmd += '--test=%s ' % self.test_path
if not self.no_create_db:
sysbench_cmd += '-u root '
else:
sysbench_cmd += '--mysql-user=%s --mysql-password=%s ' % (self.mysql_user, self.mysql_pass)
sysbench_cmd += '--mysql-db=%s ' % self.mysql_database
sysbench_cmd += '--mysql-socket=%s ' % self.mysql_socket
sysbench_cmd += '--oltp-tables-count=%s ' % self.oltp_table_count
sysbench_cmd += '--oltp-table-size=%s ' % self.oltp_table_size
sysbench_cmd += '--oltp-read-only=%s ' % self.oltp_read_only
sysbench_cmd += '--oltp-point-select=%s ' % self.oltp_point_selects
sysbench_cmd += '--oltp-range-size=%s ' % self.oltp_range_size
sysbench_cmd += '--oltp-simple-ranges=%s ' % self.oltp_simple_ranges
sysbench_cmd += '--oltp-sum-ranges=%s ' % self.oltp_sum_ranges
sysbench_cmd += '--oltp-order-ranges=%s ' % self.oltp_order_ranges
sysbench_cmd += '--oltp-distinct-ranges=%s ' % self.oltp_distinct_ranges
sysbench_cmd += '--oltp-index-updates=%s ' % self.oltp_index_updates
sysbench_cmd += '--oltp-non-index-updates=%s ' % self.oltp_non_index_updates
sysbench_cmd += 'run '
sysbench_cmd += ' > %s/sysbench.out 2> %s/sysbench.err ' % (self.out_dir,self.out_dir)
print 'Running sysbench mysql.'
common.pdsh(settings.getnodes('clients'), sysbench_cmd).communicate()
monitoring.stop(self.run_dir)
# Finally, get the historic ops
self.cluster.dump_historic_ops(self.run_dir)
common.sync_files('%s/*' % self.run_dir, self.out_dir)
def _run(self, mode, run_dir, out_dir):
# We'll always drop caches for rados bench
self.dropcaches()
if self.concurrent_ops:
concurrent_ops_str = '--concurrent-ios %s' % self.concurrent_ops
#determine rados version
rados_version_str = self.get_rados_version()
m = re.findall("version (\d+)", rados_version_str)
if not m:
m = re.findall("version v(\d+)", rados_version_str)
rados_version = int(m[0])
if mode in ['write'] or rados_version < 9:
op_size_str = '-b %s' % self.op_size
else:
op_size_str = ''
# Max Objects
max_objects_str = ''
if self.max_objects and rados_version < 9:
raise ValueError('max_objects not supported by rados_version < 9')
if self.max_objects and rados_version > 9:
max_objects_str = '--max-objects %s' % self.max_objects
# Write to OMAP
write_omap_str = ''
if self.write_omap and rados_version < 9:
raise ValueError('write_omap not supported by rados_version < 9')
if self.write_omap and rados_version > 9:
write_omap_str = '--write-omap'
common.make_remote_dir(run_dir)
# dump the cluster config
self.cluster.dump_config(run_dir)
# Run the backfill testing thread if requested
if 'recovery_test' in self.cluster.config:
recovery_callback = self.recovery_callback
self.cluster.create_recovery_test(run_dir, recovery_callback)
# Run rados bench
monitoring.start(run_dir)
logger.info('Running radosbench %s test.' % mode)
ps = []
for i in xrange(self.concurrent_procs):
out_file = '%s/output.%s' % (run_dir, i)
objecter_log = '%s/objecter.%s.log' % (run_dir, i)
# default behavior is to use a single storage pool
pool_name = self.pool
run_name = '--run-name %s`%s`-%s' % (self.object_set_id,
common.get_fqdn_cmd(), i)
if self.pool_per_proc: # support previous behavior of 1 storage pool per rados process
pool_name = 'rados-bench-``-%s' % (common.get_fqdn_cmd(), i)
run_name = ''
rados_bench_cmd = '%s -c %s -p %s bench %s %s %s %s %s %s %s --no-cleanup 2> %s > %s' % \
(self.cmd_path_full, self.tmp_conf, pool_name, op_size_str, self.time, mode, concurrent_ops_str, max_objects_str, write_omap_str, run_name, objecter_log, out_file)
p = common.pdsh(settings.getnodes('clients'), rados_bench_cmd)
ps.append(p)
for p in ps:
p.wait()
monitoring.stop(run_dir)
# If we were doing recovery, wait until it's done.
if 'recovery_test' in self.cluster.config:
self.cluster.wait_recovery_done()
# Finally, get the historic ops
self.cluster.dump_historic_ops(run_dir)
common.sync_files('%s/*' % run_dir, out_dir)
self.analyze(out_dir)
def _run(self, mode, run_dir, out_dir, max_objects, runtime):
# We'll always drop caches for rados bench
self.dropcaches()
if self.concurrent_ops:
concurrent_ops_str = '--concurrent-ios %s' % self.concurrent_ops
rados_version = self.get_rados_version()
# Max Objects
max_objects_str = ''
if max_objects:
if rados_version < 10:
raise ValueError('max_objects not supported by rados_version < 10')
max_objects_str = '--max-objects %s' % max_objects
# Operation type
op_type = mode
if mode == 'prefill':
op_type = 'write'
if op_type == 'write':
op_size_str = '-b %s' % self.op_size
else:
op_size_str = ''
# Write to OMAP
write_omap_str = ''
if self.write_omap:
if rados_version < 10:
raise ValueError('write_omap not supported by rados_version < 10')
write_omap_str = '--write-omap'
run_dir = os.path.join(self.run_dir, run_dir)
common.make_remote_dir(run_dir)
# dump the cluster config
self.cluster.dump_config(run_dir)
# Run the backfill testing thread if requested (but not for prefill)
if mode != 'prefill' and 'recovery_test' in self.cluster.config:
recovery_callback = self.recovery_callback
self.cluster.create_recovery_test(run_dir, recovery_callback)
# Run rados bench
with monitoring.monitor(run_dir) as monitor:
logger.info('Running radosbench %s test.' % mode)
ps = []
for i in range(self.concurrent_procs):
out_file = '%s/output.%s' % (run_dir, i)
objecter_log = '%s/objecter.%s.log' % (run_dir, i)
if self.pool_per_proc:
# support previous behavior of 1 storage pool per rados process
pool_name_cmd = 'rados-bench-`{fqdn_cmd}`-{i}'
pool_name = pool_name_cmd.format(fqdn_cmd=common.get_fqdn_cmd(), i=i)
run_name = ''
else:
# default behavior is to use a single storage pool
pool_name = self.pool
run_name_fmt = '--run-name {object_set_id} `{fqdn_cmd}`-{i}'
run_name = run_name_fmt.format(
object_set_id=self.object_set_id,
fqdn_cmd=common.get_fqdn_cmd(),
i=i)
rados_bench_cmd_fmt = \
'{cmd} -c {conf} -p {pool} bench {op_size_arg} {duration} ' \
'{op_type} {concurrent_ops_arg} {max_objects_arg} ' \
'{write_omap_arg} {run_name} --no-cleanup ' \
'2> {stderr} > {stdout}'
rados_bench_cmd = rados_bench_cmd_fmt.format(
cmd=self.cmd_path_full,
conf=self.tmp_conf,
pool=pool_name,
op_size_arg=op_size_str,
duration=runtime,
op_type=op_type,
concurrent_ops_arg=concurrent_ops_str,
max_objects_arg=max_objects_str,
write_omap_arg=write_omap_str,
run_name=run_name,
stderr=objecter_log,
stdout=out_file)
p = common.pdsh(settings.getnodes('clients'), rados_bench_cmd)
ps.append(p)
for p in ps:
p.wait()
# If we were doing recovery, wait until it's done (but not for prefill).
if mode != 'prefill' and 'recovery_test' in self.cluster.config:
self.cluster.wait_recovery_done()
# Finally, get the historic ops
self.cluster.dump_historic_ops(run_dir)
out_dir = os.path.join(self.out_dir, out_dir)
common.sync_files('%s/*' % run_dir, out_dir)
self.analyze(out_dir)