def initialize(self): super(LibrbdFio, self).initialize() # Clean and Create the run directory common.clean_remote_dir(self.run_dir) common.make_remote_dir(self.run_dir) logger.info('Pausing for 60s for idle monitoring.') monitoring.start("%s/idle_monitoring" % self.run_dir) time.sleep(60) monitoring.stop() common.sync_files('%s/*' % self.run_dir, self.out_dir) self.mkimages() # populate the fio files ps = [] logger.info('Attempting to populating fio files...') if (self.use_existing_volumes == False): for volnum in xrange(self.volumes_per_client): rbd_name = 'cbt-librbdfio-`%s`-%d' % (common.get_fqdn_cmd(), volnum) pre_cmd = 'sudo %s --ioengine=rbd --clientname=admin --pool=%s --rbdname=%s --invalidate=0 --rw=write --numjobs=%s --bs=4M --size %dM %s --output-format=%s > /dev/null' % (self.cmd_path, self.pool_name, rbd_name, self.numjobs, self.vol_size, self.names, self.fio_out_format) p = common.pdsh(settings.getnodes('clients'), pre_cmd) ps.append(p) for p in ps: p.wait() return True
def mkgetputcmd(self, cred_file, gwnum): # grab the executable to use getput_cmd = '%s ' % self.cmd_path # Set the options if self.container_prefix is not None: container_prefix_flag = '-c%s' % self.container_prefix if self.ctype == 'byproc' or self.ctype == 'bynodegw': container_prefix_flag = '%s-gw%s' % (container_prefix_flag, gwnum) getput_cmd += '%s ' % container_prefix_flag # For now we'll only test distinct objects per client/gw if self.object_prefix is not None: getput_cmd += '-o%s-`%s`-gw%s ' % (self.object_prefix, common.get_fqdn_cmd(), gwnum) else: getput_cmd += '-o`%s`-gw%s ' % (common.get_fqdn_cmd(), gwnum) getput_cmd += '-s%s ' % self.op_size getput_cmd += '-t%s ' % self.test getput_cmd += '--procs %s ' % self.procs if self.ops_per_proc is not None: getput_cmd += '-n%s ' % self.ops_per_proc if self.runtime is not None: getput_cmd += '--runtime %s ' % self.runtime if self.ctype is not None: getput_cmd += '--ctype %s ' % self.ctype if self.debug is not None: getput_cmd += '--debug %s ' % self.debug if self.logops is not None: getput_cmd += '--logops %s ' % self.logops if self.grace is not None: getput_cmd += '--grace %s ' % self.grace getput_cmd += '--cred %s ' % cred_file # End the getput_cmd getput_cmd += '> %s/output.gw%s' % (self.run_dir, gwnum) return getput_cmd
def __init__(self, archive_dir, cluster, config): super(LibrbdFio, self).__init__(archive_dir, cluster, config) # FIXME there are too many permutations, need to put results in SQLITE3 self.cmd_path = config.get('cmd_path', '/usr/bin/fio') self.pool_profile = config.get('pool_profile', 'default') self.data_pool_profile = config.get('data_pool_profile', None) self.time = config.get('time', None) self.time_based = bool(config.get('time_based', False)) self.ramp = config.get('ramp', None) self.iodepth = config.get('iodepth', 16) self.numjobs = config.get('numjobs', 1) self.end_fsync = config.get('end_fsync', 0) self.mode = config.get('mode', 'write') self.rwmixread = config.get('rwmixread', 50) self.rwmixwrite = 100 - self.rwmixread self.log_avg_msec = config.get('log_avg_msec', None) # self.ioengine = config.get('ioengine', 'libaio') self.op_size = config.get('op_size', 4194304) self.pgs = config.get('pgs', 2048) self.vol_size = config.get('vol_size', 65536) self.vol_object_size = config.get('vol_object_size', 22) self.volumes_per_client = config.get('volumes_per_client', 1) self.procs_per_volume = config.get('procs_per_volume', 1) self.random_distribution = config.get('random_distribution', None) self.rate_iops = config.get('rate_iops', None) self.fio_out_format = "json,normal" self.data_pool = None # use_existing_volumes needs to be true to set the pool and rbd names self.use_existing_volumes = config.get('use_existing_volumes', False) self.pool_name = config.get("poolname", "cbt-librbdfio") self.rbdname = config.get('rbdname', '') self.total_procs = self.procs_per_volume * self.volumes_per_client * len( settings.getnodes('clients').split(',')) self.run_dir = '%s/osd_ra-%08d/op_size-%08d/concurrent_procs-%03d/iodepth-%03d/%s' % ( self.run_dir, int(self.osd_ra), int(self.op_size), int(self.total_procs), int(self.iodepth), self.mode) self.out_dir = self.archive_dir self.norandommap = config.get("norandommap", False) self.wait_pgautoscaler_timeout = config.get( "wait_pgautoscaler_timeout", -1) # Make the file names string (repeated across volumes) self.names = '' for proc_num in range(self.procs_per_volume): rbd_name = 'cbt-librbdfio-`%s`-file-%d' % (common.get_fqdn_cmd(), proc_num) self.names += '--name=%s ' % rbd_name
def mkfiocmd(self, volnum): rbdname = 'cbt-librbdfio-`%s`-%d' % (common.get_fqdn_cmd(), volnum) out_file = '%s/output.%d' % (self.run_dir, volnum) fio_cmd = 'sudo %s --ioengine=rbd --clientname=admin --pool=%s --rbdname=%s --invalidate=0' % ( self.cmd_path_full, self.pool_name, rbdname) fio_cmd += ' --rw=%s' % self.mode fio_cmd += ' --output-format=%s' % self.fio_out_format if (self.mode == 'readwrite' or self.mode == 'randrw'): fio_cmd += ' --rwmixread=%s --rwmixwrite=%s' % (self.rwmixread, self.rwmixwrite) # fio_cmd += ' --ioengine=%s' % self.ioengine if self.time is not None: fio_cmd += ' --runtime=%s' % self.time if self.time_based is True: fio_cmd += ' --time_based' if self.ramp is not None: fio_cmd += ' --ramp_time=%s' % self.ramp fio_cmd += ' --numjobs=%s' % self.numjobs fio_cmd += ' --direct=1' fio_cmd += ' --bs=%dB' % self.op_size fio_cmd += ' --iodepth=%d' % self.iodepth fio_cmd += ' --end_fsync=%s' % self.end_fsync # if self.vol_size: # fio_cmd += ' -- size=%dM' % self.vol_size if self.norandommap: fio_cmd += ' --norandommap' if self.log_iops: fio_cmd += ' --write_iops_log=%s' % out_file if self.log_bw: fio_cmd += ' --write_bw_log=%s' % out_file if self.log_lat: fio_cmd += ' --write_lat_log=%s' % out_file if 'recovery_test' in self.cluster.config: fio_cmd += ' --time_based' if self.random_distribution is not None: fio_cmd += ' --random_distribution=%s' % self.random_distribution if self.log_avg_msec is not None: fio_cmd += ' --log_avg_msec=%s' % self.log_avg_msec if self.rate_iops is not None: fio_cmd += ' --rate_iops=%s' % self.rate_iops # End the fio_cmd fio_cmd += ' %s > %s' % (self.names, out_file) return fio_cmd
def fio_command_extra(self, ep_num): cmd = '' # typical directory endpoints if self.endpoint_type == 'directory': for proc_num in range(self.procs_per_endpoint): cmd += ' --name=%s/`%s`-%s-%s' % (self.endpoints[ep_num], common.get_fqdn_cmd(), ep_num, proc_num) # handle rbd endpoints with the librbbd engine. elif self.endpoint_type == 'rbd': pool_name, rbd_name = self.endpoints[ep_num].split("/") cmd += ' --clientname=admin' cmd += ' --pool=%s' % pool_name cmd += ' --rbdname=%s' % rbd_name cmd += ' --invalidate=0' for proc_num in range(self.procs_per_endpoint): rbd_name = '%s-%d' % (self.endpoints[ep_num], proc_num) cmd += ' --name=%s' % rbd_name return cmd
def __init__(self, cluster, config): super(LibrbdFio, self).__init__(cluster, config) # FIXME there are too many permutations, need to put results in SQLITE3 self.cmd_path = config.get('cmd_path', '/usr/bin/fio') self.pool_profile = config.get('pool_profile', 'default') self.data_pool_profile = config.get('data_pool_profile', None) self.time = str(config.get('time', None)) self.time_based = bool(config.get('time_based', False)) self.ramp = str(config.get('ramp', None)) self.iodepth = config.get('iodepth', 16) self.numjobs = config.get('numjobs', 1) self.end_fsync = str(config.get('end_fsync', 0)) self.mode = config.get('mode', 'write') self.rwmixread = config.get('rwmixread', 50) self.rwmixwrite = 100 - self.rwmixread self.log_avg_msec = config.get('log_avg_msec', None) # self.ioengine = config.get('ioengine', 'libaio') self.op_size = config.get('op_size', 4194304) self.pgs = config.get('pgs', 2048) self.vol_size = config.get('vol_size', 65536) self.vol_order = config.get('vol_order', 22) self.volumes_per_client = config.get('volumes_per_client', 1) self.procs_per_volume = config.get('procs_per_volume', 1) self.random_distribution = config.get('random_distribution', None) self.rate_iops = config.get('rate_iops', None) self.pool_name = "cbt-librbdfio" self.fio_out_format = "json,normal" self.data_pool = None self.use_existing_volumes = config.get('use_existing_volumes', False) self.total_procs = self.procs_per_volume * self.volumes_per_client * len(settings.getnodes('clients').split(',')) self.run_dir = '%s/osd_ra-%08d/op_size-%08d/concurrent_procs-%03d/iodepth-%03d/%s' % (self.run_dir, int(self.osd_ra), int(self.op_size), int(self.total_procs), int(self.iodepth), self.mode) self.out_dir = self.archive_dir self.norandommap = config.get("norandommap", False) # Make the file names string (repeated across volumes) self.names = '' for proc_num in xrange(self.procs_per_volume): rbd_name = 'cbt-librbdfio-`%s`-file-%d' % (common.get_fqdn_cmd(), proc_num) self.names += '--name=%s ' % rbd_name
def mkfiocmd(self, volnum): rbdname = 'cbt-librbdfio-`%s`-%d' % (common.get_fqdn_cmd(), volnum) out_file = '%s/output.%d' % (self.run_dir, volnum) fio_cmd = 'sudo %s --ioengine=rbd --clientname=admin --pool=%s --rbdname=%s --invalidate=0' % (self.cmd_path_full, self.pool_name, rbdname) fio_cmd += ' --rw=%s' % self.mode fio_cmd += ' --output-format=%s' % self.fio_out_format if (self.mode == 'readwrite' or self.mode == 'randrw'): fio_cmd += ' --rwmixread=%s --rwmixwrite=%s' % (self.rwmixread, self.rwmixwrite) # fio_cmd += ' --ioengine=%s' % self.ioengine if self.time is not None: fio_cmd += ' --runtime=%s' % self.time if self.time_based is True: fio_cmd += ' --time_based' if self.ramp is not None: fio_cmd += ' --ramp_time=%s' % self.ramp fio_cmd += ' --numjobs=%s' % self.numjobs fio_cmd += ' --direct=1' fio_cmd += ' --bs=%dB' % self.op_size fio_cmd += ' --iodepth=%d' % self.iodepth fio_cmd += ' --end_fsync=%s' % self.end_fsync # if self.vol_size: # fio_cmd += ' -- size=%dM' % self.vol_size if self.norandommap: fio_cmd += ' --norandommap' fio_cmd += ' --write_iops_log=%s' % out_file fio_cmd += ' --write_bw_log=%s' % out_file fio_cmd += ' --write_lat_log=%s' % out_file if 'recovery_test' in self.cluster.config: fio_cmd += ' --time_based' if self.random_distribution is not None: fio_cmd += ' --random_distribution=%s' % self.random_distribution if self.log_avg_msec is not None: fio_cmd += ' --log_avg_msec=%s' % self.log_avg_msec if self.rate_iops is not None: fio_cmd += ' --rate_iops=%s' % self.rate_iops # End the fio_cmd fio_cmd += ' %s > %s' % (self.names, out_file) return fio_cmd
def initialize(self): super(KvmRbdFio, self).initialize() common.pdsh(settings.getnodes('clients', 'osds', 'mons', 'rgws'), 'sudo rm -rf %s' % self.run_dir, continue_if_error=False).communicate() common.make_remote_dir(self.run_dir) clnts = settings.getnodes('clients') logger.info('creating mountpoints...') for b in self.block_devices: bnm = os.path.basename(b) mtpt = '/srv/rbdfio-`%s`-%s' % (common.get_fqdn_cmd(), bnm) common.pdsh(clnts, 'sudo mkfs.ext4 %s' % b, continue_if_error=False).communicate() common.pdsh(clnts, 'sudo mkdir -p %s' % mtpt, continue_if_error=False).communicate() common.pdsh(clnts, 'sudo mount -t ext4 -o noatime %s %s' % (b, mtpt), continue_if_error=False).communicate() logger.info('Attempting to initialize fio files...') initializer_list = [] for i in range(self.concurrent_procs): b = self.block_devices[i % len(self.block_devices)] bnm = os.path.basename(b) mtpt = '/srv/rbdfio-`hostname -s`-%s' % bnm fiopath = os.path.join(mtpt, 'fio%d.img' % i) pre_cmd = 'sudo %s --rw=write -ioengine=sync --bs=4M ' % self.fio_cmd pre_cmd = '%s --size %dM --name=%s > /dev/null' % ( pre_cmd, self.vol_size, fiopath) initializer_list.append( common.pdsh(clnts, pre_cmd, continue_if_error=False)) for p in initializer_list: p.communicate() # Create the run directory common.pdsh(clnts, 'rm -rf %s' % self.run_dir, continue_if_error=False).communicate() common.make_remote_dir(self.run_dir)
def initialize(self): super(LibrbdFio, self).initialize() # Clean and Create the run directory common.clean_remote_dir(self.run_dir) common.make_remote_dir(self.run_dir) logger.info('Running scrub monitoring.') monitoring.start("%s/scrub_monitoring" % self.run_dir) self.cluster.check_scrub() monitoring.stop() logger.info('Pausing for 60s for idle monitoring.') monitoring.start("%s/idle_monitoring" % self.run_dir) time.sleep(60) monitoring.stop() common.sync_files('%s/*' % self.run_dir, self.out_dir) self.mkimages() # populate the fio files ps = [] logger.info('Attempting to populating fio files...') if (self.use_existing_volumes == False): for volnum in xrange(self.volumes_per_client): rbd_name = 'cbt-librbdfio-`%s`-%d' % (common.get_fqdn_cmd(), volnum) pre_cmd = 'sudo %s --ioengine=rbd --clientname=admin --pool=%s --rbdname=%s --invalidate=0 --rw=write --numjobs=%s --bs=4M --size %dM %s > /dev/null' % ( self.cmd_path, self.pool_name, rbd_name, self.numjobs, self.vol_size, self.names) p = common.pdsh(settings.getnodes('clients'), pre_cmd) ps.append(p) for p in ps: p.wait() return True
def initialize(self): super(KvmRbdFio, self).initialize() common.pdsh(settings.getnodes('clients', 'osds', 'mons', 'rgws'), 'sudo rm -rf %s' % self.run_dir, continue_if_error=False).communicate() common.make_remote_dir(self.run_dir) clnts = settings.getnodes('clients') logger.info('creating mountpoints...') for b in self.block_devices: bnm = os.path.basename(b) mtpt = '/srv/rbdfio-`%s`-%s' % (common.get_fqdn_cmd(), bnm) common.pdsh(clnts, 'sudo mkfs.ext4 %s' % b, continue_if_error=False).communicate() common.pdsh(clnts, 'sudo mkdir -p %s' % mtpt, continue_if_error=False).communicate() common.pdsh(clnts, 'sudo mount -t ext4 -o noatime %s %s' % (b,mtpt), continue_if_error=False).communicate() logger.info('Attempting to initialize fio files...') initializer_list = [] for i in range(self.concurrent_procs): b = self.block_devices[i % len(self.block_devices)] bnm = os.path.basename(b) mtpt = '/srv/rbdfio-`hostname -s`-%s' % bnm fiopath = os.path.join(mtpt, 'fio%d.img' % i) pre_cmd = 'sudo %s --rw=write -ioengine=sync --bs=4M ' % self.fio_cmd pre_cmd = '%s --size %dM --name=%s > /dev/null' % ( pre_cmd, self.vol_size, fiopath) initializer_list.append(common.pdsh(clnts, pre_cmd, continue_if_error=False)) for p in initializer_list: p.communicate() # Create the run directory common.pdsh(clnts, 'rm -rf %s' % self.run_dir, continue_if_error=False).communicate() common.make_remote_dir(self.run_dir)
def get_local_rbd_name(self, ep_num): return '`%s`-%d' % (common.get_fqdn_cmd(), ep_num)
def _run(self, mode, run_dir, out_dir, max_objects, runtime): # We'll always drop caches for rados bench self.dropcaches() if self.concurrent_ops: concurrent_ops_str = '--concurrent-ios %s' % self.concurrent_ops rados_version = self.get_rados_version() # Max Objects max_objects_str = '' if max_objects: if rados_version < 10: raise ValueError('max_objects not supported by rados_version < 10') max_objects_str = '--max-objects %s' % max_objects # Operation type op_type = mode if mode == 'prefill': op_type = 'write' if op_type == 'write': op_size_str = '-b %s' % self.op_size else: op_size_str = '' # Write to OMAP write_omap_str = '' if self.write_omap: if rados_version < 10: raise ValueError('write_omap not supported by rados_version < 10') write_omap_str = '--write-omap' run_dir = os.path.join(self.run_dir, run_dir) common.make_remote_dir(run_dir) # dump the cluster config self.cluster.dump_config(run_dir) # Run the backfill testing thread if requested (but not for prefill) if mode != 'prefill' and 'recovery_test' in self.cluster.config: recovery_callback = self.recovery_callback self.cluster.create_recovery_test(run_dir, recovery_callback) # Run rados bench with monitoring.monitor(run_dir) as monitor: logger.info('Running radosbench %s test.' % mode) ps = [] for i in range(self.concurrent_procs): out_file = '%s/output.%s' % (run_dir, i) objecter_log = '%s/objecter.%s.log' % (run_dir, i) if self.pool_per_proc: # support previous behavior of 1 storage pool per rados process pool_name_cmd = 'rados-bench-`{fqdn_cmd}`-{i}' pool_name = pool_name_cmd.format(fqdn_cmd=common.get_fqdn_cmd(), i=i) run_name = '' else: # default behavior is to use a single storage pool pool_name = self.pool run_name_fmt = '--run-name {object_set_id} `{fqdn_cmd}`-{i}' run_name = run_name_fmt.format( object_set_id=self.object_set_id, fqdn_cmd=common.get_fqdn_cmd(), i=i) rados_bench_cmd_fmt = \ '{cmd} -c {conf} -p {pool} bench {op_size_arg} {duration} ' \ '{op_type} {concurrent_ops_arg} {max_objects_arg} ' \ '{write_omap_arg} {run_name} --no-cleanup ' \ '2> {stderr} > {stdout}' rados_bench_cmd = rados_bench_cmd_fmt.format( cmd=self.cmd_path_full, conf=self.tmp_conf, pool=pool_name, op_size_arg=op_size_str, duration=runtime, op_type=op_type, concurrent_ops_arg=concurrent_ops_str, max_objects_arg=max_objects_str, write_omap_arg=write_omap_str, run_name=run_name, stderr=objecter_log, stdout=out_file) p = common.pdsh(settings.getnodes('clients'), rados_bench_cmd) ps.append(p) for p in ps: p.wait() # If we were doing recovery, wait until it's done (but not for prefill). if mode != 'prefill' and 'recovery_test' in self.cluster.config: self.cluster.wait_recovery_done() # Finally, get the historic ops self.cluster.dump_historic_ops(run_dir) out_dir = os.path.join(self.out_dir, out_dir) common.sync_files('%s/*' % run_dir, out_dir) self.analyze(out_dir)
def _run(self, mode, run_dir, out_dir): # We'll always drop caches for rados bench self.dropcaches() if self.concurrent_ops: concurrent_ops_str = '--concurrent-ios %s' % self.concurrent_ops #determine rados version rados_version_str = self.get_rados_version() m = re.findall("version (\d+)", rados_version_str) if not m: m = re.findall("version v(\d+)", rados_version_str) rados_version = int(m[0]) if mode in ['write'] or rados_version < 9: op_size_str = '-b %s' % self.op_size else: op_size_str = '' # Max Objects max_objects_str = '' if self.max_objects and rados_version < 9: raise ValueError('max_objects not supported by rados_version < 9') if self.max_objects and rados_version > 9: max_objects_str = '--max-objects %s' % self.max_objects # Write to OMAP write_omap_str = '' if self.write_omap and rados_version < 9: raise ValueError('write_omap not supported by rados_version < 9') if self.write_omap and rados_version > 9: write_omap_str = '--write-omap' common.make_remote_dir(run_dir) # dump the cluster config self.cluster.dump_config(run_dir) # Run the backfill testing thread if requested if 'recovery_test' in self.cluster.config: recovery_callback = self.recovery_callback self.cluster.create_recovery_test(run_dir, recovery_callback) # Run rados bench monitoring.start(run_dir) logger.info('Running radosbench %s test.' % mode) ps = [] for i in xrange(self.concurrent_procs): out_file = '%s/output.%s' % (run_dir, i) objecter_log = '%s/objecter.%s.log' % (run_dir, i) # default behavior is to use a single storage pool pool_name = self.pool run_name = '--run-name %s`%s`-%s'%(self.object_set_id, common.get_fqdn_cmd(), i) if self.pool_per_proc: # support previous behavior of 1 storage pool per rados process pool_name = 'rados-bench-``-%s'% (common.get_fqdn_cmd(), i) run_name = '' rados_bench_cmd = '%s -c %s -p %s bench %s %s %s %s %s %s %s --no-cleanup 2> %s > %s' % \ (self.cmd_path_full, self.tmp_conf, pool_name, op_size_str, self.time, mode, concurrent_ops_str, max_objects_str, write_omap_str, run_name, objecter_log, out_file) p = common.pdsh(settings.getnodes('clients'), rados_bench_cmd) ps.append(p) for p in ps: p.wait() monitoring.stop(run_dir) # If we were doing recovery, wait until it's done. if 'recovery_test' in self.cluster.config: self.cluster.wait_recovery_done() # Finally, get the historic ops self.cluster.dump_historic_ops(run_dir) common.sync_files('%s/*' % run_dir, out_dir) self.analyze(out_dir)
def _run(self, mode, run_dir, out_dir): # We'll always drop caches for rados bench self.dropcaches() if self.concurrent_ops: concurrent_ops_str = '--concurrent-ios %s' % self.concurrent_ops #determine rados version rados_version_str = self.get_rados_version() m = re.findall("version (\d+)", rados_version_str) if not m: m = re.findall("version v(\d+)", rados_version_str) rados_version = int(m[0]) if mode in ['write'] or rados_version < 9: op_size_str = '-b %s' % self.op_size else: op_size_str = '' # Max Objects max_objects_str = '' if self.max_objects and rados_version < 9: raise ValueError('max_objects not supported by rados_version < 9') if self.max_objects and rados_version > 9: max_objects_str = '--max-objects %s' % self.max_objects # Write to OMAP write_omap_str = '' if self.write_omap and rados_version < 9: raise ValueError('write_omap not supported by rados_version < 9') if self.write_omap and rados_version > 9: write_omap_str = '--write-omap' common.make_remote_dir(run_dir) # dump the cluster config self.cluster.dump_config(run_dir) # Run the backfill testing thread if requested if 'recovery_test' in self.cluster.config: recovery_callback = self.recovery_callback self.cluster.create_recovery_test(run_dir, recovery_callback) # Run rados bench monitoring.start(run_dir) logger.info('Running radosbench %s test.' % mode) ps = [] for i in xrange(self.concurrent_procs): out_file = '%s/output.%s' % (run_dir, i) objecter_log = '%s/objecter.%s.log' % (run_dir, i) # default behavior is to use a single storage pool pool_name = self.pool run_name = '--run-name %s`%s`-%s' % (self.object_set_id, common.get_fqdn_cmd(), i) if self.pool_per_proc: # support previous behavior of 1 storage pool per rados process pool_name = 'rados-bench-``-%s' % (common.get_fqdn_cmd(), i) run_name = '' rados_bench_cmd = '%s -c %s -p %s bench %s %s %s %s %s %s %s --no-cleanup 2> %s > %s' % \ (self.cmd_path_full, self.tmp_conf, pool_name, op_size_str, self.time, mode, concurrent_ops_str, max_objects_str, write_omap_str, run_name, objecter_log, out_file) p = common.pdsh(settings.getnodes('clients'), rados_bench_cmd) ps.append(p) for p in ps: p.wait() monitoring.stop(run_dir) # If we were doing recovery, wait until it's done. if 'recovery_test' in self.cluster.config: self.cluster.wait_recovery_done() # Finally, get the historic ops self.cluster.dump_historic_ops(run_dir) common.sync_files('%s/*' % run_dir, out_dir) self.analyze(out_dir)