def radosgw_user_create(self):
"""Create raodsgw user.
Create the radosgw ceph user
:returns: This method is called for its side effects.
:rtype: None
"""
if not self._stored.swift_bench_snap_installed:
_msg = "Upload swift-bench snap resource to proceed"
self.unit.status = ops.model.BlockedStatus(_msg)
return
_bench = bench_tools.BenchTools(self)
if not self.peers.swift_user_created:
logging.info("Create radosgw user and key")
try:
_bench.radosgw_user_create(self.CLIENT_NAME, "swift",
self.get_swift_key())
self.peers.set_swift_user_created(self.SWIFT_USER)
logging.info("Successfully created the radosgw user and key")
except subprocess.CalledProcessError as e:
if ("user: {} exists".format(self.CLIENT_NAME)
not in e.stderr.decode("UTF-8")):
_msg = ("Rados GW user and key creation failed: {}".format(
e.stderr.decode("UTF-8")))
logging.error(_msg)
else:
logging.warning("User: {} already exists.".format(
self.CLIENT_NAME))
def mount_rbd(self, event):
"""Mount rbd block device.
Create RBD image. Map RBD Image. Prepare and mount RBD block device.
:param event: Event
:type event: Operator framework event object
:returns: This method is called for its side effects.
:rtype: None
"""
_bench = bench_tools.BenchTools(self)
# Make and mount fs
logging.info("Setup filestem for rbd")
try:
_bench.make_rbd_fs(self.get_pool_name(event))
_bench.make_rbd_mount()
_bench.mount_rbd_mount(self.get_pool_name(event))
except subprocess.CalledProcessError as e:
_msg = ("Making or mounting fs failed: {}".format(
e.stderr.decode("UTF-8")))
logging.error(_msg)
event.fail(_msg)
event.set_results({"stderr": _msg, "code": "1"})
raise
def on_rbd_bench_action(self, event):
"""Event handler on RBD bench action.
Run the rbd-bench test.
:param event: Event
:type event: Operator framework event object
:returns: This method is called for its side effect of setting event
results.
:rtype: None
"""
# Prepare the rbd image
self.rbd_create_image(event)
self.rbd_map_image(event)
self.mount_rbd(event)
_bench = bench_tools.BenchTools(self)
# Run bench
logging.info("Running rbd bench")
try:
_result = _bench.rbd_bench(self.get_pool_name(event),
event.params["operation"])
event.set_results({self.action_output_key: _result})
except subprocess.CalledProcessError as e:
_msg = ("rbd bench failed: {}".format(e.stderr.decode("UTF-8")))
logging.error(_msg)
event.fail(_msg)
event.set_results({"stderr": _msg, "code": "1"})
raise
def on_rados_bench_action(self, event):
"""Event handler on RADOS bench action.
Run the rados-bench test.
:param event: Event
:type event: Operator framework event object
:returns: This method is called for its side effect of setting event
results.
:rtype: None
"""
_bench = bench_tools.BenchTools(self)
logging.info("Running rados bench {}".format(
event.params["operation"]))
try:
_result = _bench.rados_bench(self.get_pool_name(event),
event.params["seconds"],
event.params["operation"],
switches=event.params.get("switches"))
event.set_results({self.action_output_key: _result})
except subprocess.CalledProcessError as e:
_msg = ("rados bench failed: {}".format(e.stderr.decode("UTF-8")))
logging.error(_msg)
event.fail(_msg)
event.set_results({"stderr": _msg, "code": "1"})
def rbd_map_image(self, event):
"""Create map and mount rbd block device.
Create RBD image. Map RBD Image. Prepare and mount RBD block device.
:param event: Event
:type event: Operator framework event object
:returns: This method is called for its side effects.
:rtype: None
"""
_bench = bench_tools.BenchTools(self)
# Map the image
logging.info("rbd map image")
try:
_result = _bench.rbd_map_image(self.get_pool_name(event))
# XXX We actually don't care about this output unless we fail on
# subsequent steps
event.set_results({self.action_output_key: _result})
except subprocess.CalledProcessError as e:
_msg = ("rbd map image failed: {}".format(
e.stderr.decode("UTF-8")))
logging.error(_msg)
event.fail(_msg)
event.set_results({"stderr": _msg, "code": "1"})
raise
def create_map_mount_rbd(self, event):
_bench = bench_tools.BenchTools(self)
# Create the image
logging.info("Create the rbd image")
try:
_result = _bench.rbd_create_image(
self.get_pool_name(event),
event.params["image-size"])
# XXX We actually don't care about this output unless we fail on
# subsequent steps
event.set_results({self.action_output_key: _result})
except subprocess.CalledProcessError as e:
if "already exists" in e.stderr.decode("UTF-8"):
logging.warning(e.stderr.decode("UTF-8"))
else:
_msg = ("rbd create image failed: {}"
.format(e.stderr.decode("UTF-8")))
logging.error(_msg)
event.fail(_msg)
event.set_results({
"stderr": _msg,
"code": "1"})
raise
# Map the image
logging.info("Map the rbd image")
try:
_result = _bench.rbd_map_image(
self.get_pool_name(event))
# XXX We actually don't care about this output unless we fail on
# subsequent steps
event.set_results({self.action_output_key: _result})
except subprocess.CalledProcessError as e:
_msg = ("rbd map image failed: {}"
.format(e.stderr.decode("UTF-8")))
logging.error(_msg)
event.fail(_msg)
event.set_results({
"stderr": _msg,
"code": "1"})
raise
# Make and mount fs
logging.info("Setup filestem for rbd")
try:
_bench.make_rbd_fs(self.get_pool_name(event))
_bench.make_rbd_mount()
_bench.mount_rbd_mount(self.get_pool_name(event))
except subprocess.CalledProcessError as e:
_msg = ("Making or mounting fs failed: {}"
.format(e.stderr.decode("UTF-8")))
logging.error(_msg)
event.fail(_msg)
event.set_results({
"stderr": _msg,
"code": "1"})
raise
def on_swift_bench_action(self, event):
"""
swift-bench -c 64 -s 4096 -n 1000 -g 100 ./swift.conf
"""
_bench = bench_tools.BenchTools(self)
# Add action_parms to adapters
self.set_action_params(event)
# Add swift-bench.conf for rendering
self.configs_for_rendering.append(str(self.SWIFT_BENCH_CONF))
# Render swift-bench.conf with action_params
self.render_config(event)
logging.info("Create radosgw user and key")
if not self.peers.swift_user_created:
try:
_result = _bench.radosgw_user_create(
self.CLIENT_NAME,
"swift",
self.get_swift_key())
# XXX We actually don't care about this output unless we fail
# on subsequent steps
event.set_results({self.action_output_key: _result})
except subprocess.CalledProcessError as e:
_msg = ("Rados GW user and key creation failed: {}"
.format(e.stderr.decode("UTF-8")))
logging.error(_msg)
event.fail(_msg)
event.set_results({
"stderr": _msg,
"code": "1"})
raise
self.peers.set_swift_user_created(self.SWIFT_USER)
# Run bench
logging.info("Running swift bench")
try:
_result = _bench.swift_bench()
event.set_results({self.action_output_key: _result})
except subprocess.CalledProcessError as e:
# For some reason swift-bench sends outpout to stderr
# So stderr is also on stdout
_msg = ("swift bench failed: {}"
.format(e.stdout.decode("UTF-8")))
logging.error(_msg)
event.fail(_msg)
event.set_results({
"stderr": _msg,
"code": "1"})
raise
def on_rados_bench_action(self, event):
_bench = bench_tools.BenchTools(self)
logging.info(
"Running rados bench {}".format(event.params["operation"]))
try:
_result = _bench.rados_bench(
self.get_pool_name(event),
event.params["seconds"],
event.params["operation"],
switches=event.params.get("switches"))
event.set_results({self.action_output_key: _result})
except subprocess.CalledProcessError as e:
_msg = ("rados bench failed: {}"
.format(e.stderr.decode("UTF-8")))
logging.error(_msg)
event.fail(_msg)
event.set_results({
"stderr": _msg,
"code": "1"})
def on_rbd_bench_action(self, event):
# Prepare the rbd image
self.create_map_mount_rbd(event)
_bench = bench_tools.BenchTools(self)
# Run bench
logging.info("Running rbd bench")
try:
_result = _bench.rbd_bench(
self.get_pool_name(event),
event.params["operation"])
event.set_results({self.action_output_key: _result})
except subprocess.CalledProcessError as e:
_msg = ("rbd bench failed: {}"
.format(e.stderr.decode("UTF-8")))
logging.error(_msg)
event.fail(_msg)
event.set_results({
"stderr": _msg,
"code": "1"})
raise
def on_fio_action(self, event):
# If not disk specified use RBD mount
if not event.params.get("disk-devices"):
# Prepare the rbd image
self.create_map_mount_rbd(event)
# Add context for the render of rbd.fio
event.params["client"] = self.CLIENT_NAME
event.params["rbd_image"] = self.RBD_IMAGE
event.params["pool_name"] = self.get_pool_name(event)
_fio_conf = str(self.RBD_FIO_CONF)
else:
event.params["disk_devices"] = event.params["disk-devices"].split()
_fio_conf = str(self.DISK_FIO_CONF)
# Add action_parms to adapters
self.set_action_params(event)
# Render fio config file
self.configs_for_rendering.append(_fio_conf)
self.render_config(event)
_bench = bench_tools.BenchTools(self)
logging.info(
"Running fio {}".format(event.params["operation"]))
try:
_result = _bench.fio(_fio_conf)
event.set_results({self.action_output_key: _result})
except subprocess.CalledProcessError as e:
_msg = ("fio failed: {}"
.format(e.stderr.decode("UTF-8")))
logging.error(_msg)
event.fail(_msg)
event.set_results({
"stderr": _msg,
"code": "1"})
def on_fio_action(self, event):
"""Event handler on FIO action.
Run the FIO test.
:param event: Event
:type event: Operator framework event object
:returns: This method is called for its side effect of setting event
results.
:rtype: None
"""
test_devices = self.model.storages.get('test-devices')
# If storage binding provided then override disk-devices
# unless the application is related to ceph.
if not self.ceph_client.pools_available and test_devices:
event.params["disk-devices"] = (" ".join(
[str(d.location) for d in test_devices]))
# If not disk specified use RBD mount
if not event.params.get("disk-devices"):
# Prepare the rbd image
self.rbd_create_image(event)
if not ch_host.is_container():
self.rbd_map_image(event)
# Add context for the render of rbd.fio
event.params["client"] = self.CLIENT_NAME
event.params["rbd_image"] = self.RBD_IMAGE
event.params["pool_name"] = self.get_pool_name(event)
event.params["ioengine"] = 'rbd'
_fio_conf = str(self.RBD_FIO_CONF)
else:
event.params["disk_devices"] = event.params["disk-devices"].split()
event.params["ioengine"] = 'libaio'
_fio_conf = str(self.DISK_FIO_CONF)
# Add action_parms to adapters
self.set_action_params(event)
# Individual test execution runtime
# This allows us to report metrics periodically to prometheus
test_runtime = 30
# Total test duration time (from action params)
runtime = max(test_runtime, int(event.params.get('runtime')))
# Render fio config file
self.configs_for_rendering.append(_fio_conf)
self.render_config(event)
_bench = bench_tools.BenchTools(self)
# Prometheus target for scraping of collected FIO metrics
start_http_server(8088)
logging.info("Running fio {}".format(event.params["operation"]))
try:
test_end = (datetime.datetime.now() +
datetime.timedelta(seconds=runtime))
while (datetime.datetime.now() < test_end):
_result = json.loads(_bench.fio(_fio_conf))
for job in _result["jobs"]:
for metric in ('read', 'write'):
bandwidth = job[metric]["bw"]
iops = job[metric]["iops"]
# lat_ns is broadly slat + clat so
# represents what the calling application
# would actually see in terms of latency
latency = job[metric]["lat_ns"]["mean"]
if all((bandwidth, iops, latency)):
self.add_benchmark_metric(
'fio_{}_bandwidth'.format(metric),
'FIO {} bandwidth (B/s)'.format(metric),
bandwidth)
self.add_benchmark_metric(
'fio_{}_iops'.format(metric),
'FIO {} IOPS'.format(metric), iops)
self.add_benchmark_metric(
'fio_{}_latency'.format(metric),
'FIO {} latency (ns)'.format(metric), latency)
# But add some more detailed latency reporting anyway
_keys = ('min', 'max', 'mean', 'stddev')
for _key in _keys:
self.add_benchmark_metric(
'fio_{}_{}_{}'.format(metric, 'clat', _key),
'FIO {} {} {} (ns)'.format(
metric, 'clat',
_key), job[metric]["clat_ns"][_key])
percentiles = job[metric]["clat_ns"]["percentile"]
for percentile, latency in percentiles.items():
self.add_benchmark_metric(
'fio_{}_{}_{}'.format(
metric, 'clat',
percentile.replace('.', '_')),
'FIO {} {} {} (ns)'.format(
metric, 'clat', percentile), latency)
event.set_results({self.action_output_key: _result})
except subprocess.CalledProcessError as e:
_msg = ("fio failed: {}".format(e.stderr.decode("UTF-8")))
logging.error(_msg)
event.fail(_msg)
event.set_results({"stderr": _msg, "code": "1"})
def on_swift_bench_action(self, event):
"""Event handler on Swift bench action.
Run the swift-bench test.
:param event: Event
:type event: Operator framework event object
:returns: This method is called for its side effect of setting event
results.
:rtype: None
"""
if not self._stored.swift_bench_snap_installed:
_msg = "Upload swift-bench snap resource to proceed"
self.unit.status = ops.model.BlockedStatus(_msg)
event.fail(_msg)
event.set_results({"stderr": _msg, "code": "1"})
return
if not self.get_swift_key():
_msg = ("Unable to set swift key. Please run the action on the "
"leader.")
event.fail(_msg)
event.set_results({"stderr": _msg, "code": "1"})
return
if not self.peers.swift_user_created:
self.radosgw_user_create()
_bench = bench_tools.BenchTools(self)
# Add action_parms to adapters
self.set_action_params(event)
# Add swift-bench.conf for rendering
self.configs_for_rendering.append(str(self.SWIFT_BENCH_CONF))
# Render swift-bench.conf with action_params
self.render_config(event)
# Prometheus target for scraping of collected FIO metrics
start_http_server(8088)
# Run bench
logging.info("Running swift bench")
try:
_result = _bench.swift_bench(delete=event.params["delete-objects"])
job = self.parse_swift_bench_output(_result)
json_result = json.dumps(job)
for metric in job.keys():
bandwidth = job[metric]["bw"]
successes = job[metric]["successes"]
failures = job[metric]["failures"]
if all((bandwidth, successes, failures)):
self.add_benchmark_metric(
'swift_bench_{}_bandwidth'.format(metric),
'Swift Bench {} bandwidth (B/s)'.format(metric),
bandwidth)
self.add_benchmark_metric(
'swift_bench_{}_successes'.format(metric),
'Swift Bench {} Successes'.format(metric), successes)
self.add_benchmark_metric(
'swift_bench_{}_failures'.format(metric),
'Swift Bench {} failures'.format(metric), failures)
event.set_results({self.action_output_key: json_result})
except subprocess.CalledProcessError as e:
# For some reason swift-bench sends outpout to stderr
# So stderr is also on stdout
_output = e.stdout.decode("UTF-8")
_result = self.parse_swift_bench_output(_output)
# There may be too many connection reset tracebacks in the raw
# output overloading stack limits.
if _result.get("puts"):
# If we got partial data use that
_result = json.dumps(_result)
else:
# Otherwise, return raw output (tracebacks)
_result = _output
_msg = ("swift bench failed: {}".format(_result))
logging.error(_msg)
event.fail(_msg)
event.set_results({"stderr": _msg, "code": "1"})
raise