def __init__(self,
cfg,
sources=False,
build=False,
distribution=False,
external=False,
docker=False):
self.cfg = cfg
# we preload the track here but in rare cases (external pipeline and user did not specify the distribution version) we might need
# to reload the track again. We are assuming that a track always specifies the same challenges for each version (i.e. branch).
t = self._load_track()
challenge = self._find_challenge(t)
self.race = metrics.create_race(self.cfg, t, challenge)
self.metrics_store = metrics.metrics_store(
self.cfg,
track=self.race.track_name,
challenge=self.race.challenge_name,
read_only=False)
self.race_store = metrics.race_store(self.cfg)
self.sources = sources
self.build = build
self.distribution = distribution
self.external = external
self.docker = docker
self.actor_system = None
self.mechanic = None
self.ignore_unknown_return = self.cfg.opts("system",
"ignore.unknown.return")
def setup(self, sources=False):
# to load the track we need to know the correct cluster distribution version. Usually, this value should be set
# but there are rare cases (external pipeline and user did not specify the distribution version) where we need
# to derive it ourselves. For source builds we always assume "master"
if not sources and not self.cfg.exists("mechanic", "distribution.version"):
distribution_version = mechanic.cluster_distribution_version(self.cfg)
self.logger.info("Automatically derived distribution version [%s]", distribution_version)
self.cfg.add(config.Scope.benchmark, "mechanic", "distribution.version", distribution_version)
min_es_version = versions.Version.from_string(version.minimum_es_version())
specified_version = versions.Version.from_string(distribution_version)
if specified_version < min_es_version:
raise exceptions.SystemSetupError(f"Cluster version must be at least [{min_es_version}] but was [{distribution_version}]")
self.current_track = track.load_track(self.cfg)
self.track_revision = self.cfg.opts("track", "repository.revision", mandatory=False)
challenge_name = self.cfg.opts("track", "challenge.name")
self.current_challenge = self.current_track.find_challenge_or_default(challenge_name)
if self.current_challenge is None:
raise exceptions.SystemSetupError(
"Track [{}] does not provide challenge [{}]. List the available tracks with {} list tracks.".format(
self.current_track.name, challenge_name, PROGRAM_NAME))
if self.current_challenge.user_info:
console.info(self.current_challenge.user_info)
self.race = metrics.create_race(self.cfg, self.current_track, self.current_challenge, self.track_revision)
self.metrics_store = metrics.metrics_store(
self.cfg,
track=self.race.track_name,
challenge=self.race.challenge_name,
read_only=False
)
self.race_store = metrics.race_store(self.cfg)
def test_calculate_simple_index_stats(self):
cfg = config.Config()
cfg.add(config.Scope.application, "system", "env.name", "unittest")
cfg.add(config.Scope.application, "system", "time.start", datetime.datetime.now())
cfg.add(config.Scope.application, "system", "trial.id", "6ebc6e53-ee20-4b0c-99b4-09697987e9f4")
cfg.add(config.Scope.application, "reporting", "datastore.type", "in-memory")
cfg.add(config.Scope.application, "mechanic", "car.names", ["unittest_car"])
cfg.add(config.Scope.application, "mechanic", "car.params", {})
cfg.add(config.Scope.application, "mechanic", "plugin.params", {})
cfg.add(config.Scope.application, "race", "laps", 1)
cfg.add(config.Scope.application, "race", "user.tag", "")
cfg.add(config.Scope.application, "race", "pipeline", "from-sources-skip-build")
cfg.add(config.Scope.application, "track", "params", {})
index = track.Task(name="index #1", operation=track.Operation(name="index", operation_type=track.OperationType.Bulk, params=None))
challenge = track.Challenge(name="unittest", schedule=[index], default=True)
t = track.Track("unittest", "unittest-track", challenges=[challenge])
store = metrics.metrics_store(cfg, read_only=False, track=t, challenge=challenge)
store.lap = 1
store.put_value_cluster_level("throughput", 500, unit="docs/s", task="index #1", operation_type=track.OperationType.Bulk)
store.put_value_cluster_level("throughput", 1000, unit="docs/s", task="index #1", operation_type=track.OperationType.Bulk)
store.put_value_cluster_level("throughput", 2000, unit="docs/s", task="index #1", operation_type=track.OperationType.Bulk)
store.put_value_cluster_level("latency", 2800, unit="ms", task="index #1", operation_type=track.OperationType.Bulk,
sample_type=metrics.SampleType.Warmup)
store.put_value_cluster_level("latency", 200, unit="ms", task="index #1", operation_type=track.OperationType.Bulk)
store.put_value_cluster_level("latency", 220, unit="ms", task="index #1", operation_type=track.OperationType.Bulk)
store.put_value_cluster_level("latency", 225, unit="ms", task="index #1", operation_type=track.OperationType.Bulk)
store.put_value_cluster_level("service_time", 250, unit="ms", task="index #1", operation_type=track.OperationType.Bulk,
sample_type=metrics.SampleType.Warmup, meta_data={"success": False})
store.put_value_cluster_level("service_time", 190, unit="ms", task="index #1", operation_type=track.OperationType.Bulk,
meta_data={"success": True})
store.put_value_cluster_level("service_time", 200, unit="ms", task="index #1", operation_type=track.OperationType.Bulk,
meta_data={"success": False})
store.put_value_cluster_level("service_time", 215, unit="ms", task="index #1", operation_type=track.OperationType.Bulk,
meta_data={"success": True})
store.put_count_node_level("rally-node-0", "final_index_size_bytes", 2048, unit="bytes")
store.put_count_node_level("rally-node-1", "final_index_size_bytes", 4096, unit="bytes")
stats = reporter.calculate_results(store, metrics.create_race(cfg, t, challenge))
del store
opm = stats.metrics("index #1")
self.assertEqual(collections.OrderedDict([("min", 500), ("median", 1000), ("max", 2000), ("unit", "docs/s")]), opm["throughput"])
self.assertEqual(collections.OrderedDict([("50_0", 220), ("100_0", 225)]), opm["latency"])
self.assertEqual(collections.OrderedDict([("50_0", 200), ("100_0", 215)]), opm["service_time"])
self.assertAlmostEqual(0.3333333333333333, opm["error_rate"])
self.assertEqual(6144, stats.index_size)
def setup(self, msg, sender):
self.start_sender = sender
self.cfg = msg.cfg
# to load the track we need to know the correct cluster distribution version. Usually, this value should be set but there are rare
# cases (external pipeline and user did not specify the distribution version) where we need to derive it ourselves. For source
# builds we always assume "master"
if not msg.sources and not self.cfg.exists("mechanic",
"distribution.version"):
distribution_version = mechanic.cluster_distribution_version(
self.cfg)
if not distribution_version:
raise exceptions.SystemSetupError(
"A distribution version is required. Please specify it with --distribution-version."
)
self.logger.info("Automatically derived distribution version [%s]",
distribution_version)
self.cfg.add(config.Scope.benchmark, "mechanic",
"distribution.version", distribution_version)
t = track.load_track(self.cfg)
self.track_revision = self.cfg.opts("track",
"repository.revision",
mandatory=False)
challenge_name = self.cfg.opts("track", "challenge.name")
challenge = t.find_challenge_or_default(challenge_name)
if challenge is None:
raise exceptions.SystemSetupError(
"Track [%s] does not provide challenge [%s]. List the available tracks with %s list tracks."
% (t.name, challenge_name, PROGRAM_NAME))
if challenge.user_info:
console.info(challenge.user_info)
self.race = metrics.create_race(self.cfg, t, challenge,
self.track_revision)
self.metrics_store = metrics.metrics_store(
self.cfg,
track=self.race.track_name,
challenge=self.race.challenge_name,
read_only=False)
self.race_store = metrics.race_store(self.cfg)
self.logger.info("Asking mechanic to start the engine.")
cluster_settings = challenge.cluster_settings
self.mechanic = self.createActor(
mechanic.MechanicActor,
targetActorRequirements={"coordinator": True})
self.send(
self.mechanic,
mechanic.StartEngine(self.cfg, self.metrics_store.open_context,
cluster_settings, msg.sources, msg.build,
msg.distribution, msg.external, msg.docker))
def stop(cfg):
root_path = paths.install_root(cfg)
node_config = provisioner.load_node_configuration(root_path)
if node_config.build_type == "tar":
node_launcher = launcher.ProcessLauncher(cfg)
elif node_config.build_type == "docker":
node_launcher = launcher.DockerLauncher(cfg)
else:
raise exceptions.SystemSetupError("Unknown build type [{}]".format(
node_config.build_type))
nodes, race_id = _load_node_file(root_path)
cls = metrics.metrics_store_class(cfg)
metrics_store = cls(cfg)
race_store = metrics.race_store(cfg)
try:
current_race = race_store.find_by_race_id(race_id)
except exceptions.NotFound:
logging.getLogger(__name__).info(
"Could not find race [%s] most likely because an in-memory metrics store is "
"used across multiple machines. Use an Elasticsearch metrics store to persist "
"results.", race_id)
# we are assuming here that we use an Elasticsearch metrics store... . If we use a file race store (across
# multiple machines) we will not be able to retrieve a race. In that case we open our in-memory metrics store
# with settings derived from startup parameters (because we can't store system metrics persistently anyway).
current_race = metrics.create_race(cfg, track=None, challenge=None)
metrics_store.open(race_id=current_race.race_id,
race_timestamp=current_race.race_timestamp,
track_name=current_race.track_name,
challenge_name=current_race.challenge_name)
node_launcher.stop(nodes, metrics_store)
_delete_node_file(root_path)
metrics_store.flush(refresh=True)
for node in nodes:
results = metrics.calculate_system_results(metrics_store,
node.node_name)
current_race.add_results(results)
metrics.results_store(cfg).store_results(current_race)
metrics_store.close()
# TODO: Do we need to expose this as a separate command as well?
provisioner.cleanup(preserve=cfg.opts("mechanic", "preserve.install"),
install_dir=node_config.binary_path,
data_paths=node_config.data_paths)
def test_calculate_simple_index_stats(self):
cfg = config.Config()
cfg.add(config.Scope.application, "system", "env.name", "unittest")
cfg.add(config.Scope.application, "system", "time.start", datetime.datetime.now())
cfg.add(config.Scope.application, "reporting", "datastore.type", "in-memory")
cfg.add(config.Scope.application, "mechanic", "car.names", ["unittest_car"])
cfg.add(config.Scope.application, "race", "laps", 1)
cfg.add(config.Scope.application, "race", "user.tag", "")
cfg.add(config.Scope.application, "race", "pipeline", "from-sources-skip-build")
index = track.Task(operation=track.Operation(name="index", operation_type=track.OperationType.Index, params=None))
challenge = track.Challenge(name="unittest", description="", index_settings=None, schedule=[index], default=True)
t = track.Track("unittest", "unittest-track", challenges=[challenge])
store = metrics.metrics_store(cfg, read_only=False, track=t, challenge=challenge)
store.lap = 1
store.put_value_cluster_level("throughput", 500, unit="docs/s", operation="index", operation_type=track.OperationType.Index)
store.put_value_cluster_level("throughput", 1000, unit="docs/s", operation="index", operation_type=track.OperationType.Index)
store.put_value_cluster_level("throughput", 2000, unit="docs/s", operation="index", operation_type=track.OperationType.Index)
store.put_value_cluster_level("latency", 2800, unit="ms", operation="index", operation_type=track.OperationType.Index,
sample_type=metrics.SampleType.Warmup)
store.put_value_cluster_level("latency", 200, unit="ms", operation="index", operation_type=track.OperationType.Index)
store.put_value_cluster_level("latency", 220, unit="ms", operation="index", operation_type=track.OperationType.Index)
store.put_value_cluster_level("latency", 225, unit="ms", operation="index", operation_type=track.OperationType.Index)
store.put_value_cluster_level("service_time", 250, unit="ms", operation="index", operation_type=track.OperationType.Index,
sample_type=metrics.SampleType.Warmup, meta_data={"success": False})
store.put_value_cluster_level("service_time", 190, unit="ms", operation="index", operation_type=track.OperationType.Index,
meta_data={"success": True})
store.put_value_cluster_level("service_time", 200, unit="ms", operation="index", operation_type=track.OperationType.Index,
meta_data={"success": False})
store.put_value_cluster_level("service_time", 215, unit="ms", operation="index", operation_type=track.OperationType.Index,
meta_data={"success": True})
store.put_count_node_level("rally-node-0", "final_index_size_bytes", 2048, unit="bytes")
store.put_count_node_level("rally-node-1", "final_index_size_bytes", 4096, unit="bytes")
stats = reporter.calculate_results(store, metrics.create_race(cfg, t, challenge))
del store
opm = stats.metrics("index")
self.assertEqual(collections.OrderedDict([("min", 500), ("median", 1000), ("max", 2000), ("unit", "docs/s")]), opm["throughput"])
self.assertEqual(collections.OrderedDict([("50", 220), ("100", 225)]), opm["latency"])
self.assertEqual(collections.OrderedDict([("50", 200), ("100", 215)]), opm["service_time"])
self.assertAlmostEqual(0.3333333333333333, opm["error_rate"])
self.assertEqual(6144, stats.index_size)
def setup(self, msg):
self.mechanic = self.createActor(mechanic.MechanicActor,
#globalName="/rally/mechanic/coordinator",
targetActorRequirements={"coordinator": True})
self.cfg = msg.cfg
# to load the track we need to know the correct cluster distribution version. Usually, this value should be set but there are rare
# cases (external pipeline and user did not specify the distribution version) where we need to derive it ourselves. For source
# builds we always assume "master"
if not msg.sources and not self.cfg.exists("mechanic", "distribution.version"):
distribution_version = mechanic.cluster_distribution_version(self.cfg)
if not distribution_version:
raise exceptions.SystemSetupError("A distribution version is required. Please specify it with --distribution-version.")
logger.info("Automatically derived distribution version [%s]" % distribution_version)
self.cfg.add(config.Scope.benchmark, "mechanic", "distribution.version", distribution_version)
t = track.load_track(self.cfg)
challenge_name = self.cfg.opts("track", "challenge.name")
challenge = t.find_challenge_or_default(challenge_name)
if challenge is None:
raise exceptions.SystemSetupError("Track [%s] does not provide challenge [%s]. List the available tracks with %s list tracks."
% (t.name, challenge_name, PROGRAM_NAME))
if challenge.user_info:
console.info(challenge.user_info, logger=logger)
self.race = metrics.create_race(self.cfg, t, challenge)
self.metrics_store = metrics.metrics_store(
self.cfg,
track=self.race.track_name,
challenge=self.race.challenge_name,
read_only=False
)
self.lap_counter = LapCounter(self.race, self.metrics_store, self.cfg)
self.race_store = metrics.race_store(self.cfg)
logger.info("Asking mechanic to start the engine.")
cluster_settings = self.race.challenge.cluster_settings
self.send(self.mechanic, mechanic.StartEngine(self.cfg, self.metrics_store.open_context, cluster_settings, msg.sources, msg.build,
msg.distribution, msg.external, msg.docker))
def test_calculate_simple_index_stats(self):
cfg = config.Config()
cfg.add(config.Scope.application, "system", "env.name", "unittest")
cfg.add(config.Scope.application, "system", "time.start",
datetime.datetime.now())
cfg.add(config.Scope.application, "reporting", "datastore.type",
"in-memory")
cfg.add(config.Scope.application, "mechanic", "car.name",
"unittest_car")
cfg.add(config.Scope.application, "race", "laps", 1)
cfg.add(config.Scope.application, "race", "user.tag", "")
cfg.add(config.Scope.application, "race", "pipeline",
"from-sources-skip-build")
index = track.Task(
operation=track.Operation(name="index",
operation_type=track.OperationType.Index,
params=None))
challenge = track.Challenge(name="unittest",
description="",
index_settings=None,
schedule=[index],
default=True)
t = track.Track("unittest", "unittest-track", challenges=[challenge])
store = metrics.metrics_store(cfg,
read_only=False,
track=t,
challenge=challenge)
store.lap = 1
store.put_value_cluster_level("throughput",
500,
unit="docs/s",
operation="index",
operation_type=track.OperationType.Index)
store.put_value_cluster_level("throughput",
1000,
unit="docs/s",
operation="index",
operation_type=track.OperationType.Index)
store.put_value_cluster_level("throughput",
2000,
unit="docs/s",
operation="index",
operation_type=track.OperationType.Index)
store.put_value_cluster_level("latency",
2800,
unit="ms",
operation="index",
operation_type=track.OperationType.Index,
sample_type=metrics.SampleType.Warmup)
store.put_value_cluster_level("latency",
200,
unit="ms",
operation="index",
operation_type=track.OperationType.Index)
store.put_value_cluster_level("latency",
220,
unit="ms",
operation="index",
operation_type=track.OperationType.Index)
store.put_value_cluster_level("latency",
225,
unit="ms",
operation="index",
operation_type=track.OperationType.Index)
store.put_value_cluster_level("service_time",
250,
unit="ms",
operation="index",
operation_type=track.OperationType.Index,
sample_type=metrics.SampleType.Warmup,
meta_data={"success": False})
store.put_value_cluster_level("service_time",
190,
unit="ms",
operation="index",
operation_type=track.OperationType.Index,
meta_data={"success": True})
store.put_value_cluster_level("service_time",
200,
unit="ms",
operation="index",
operation_type=track.OperationType.Index,
meta_data={"success": False})
store.put_value_cluster_level("service_time",
215,
unit="ms",
operation="index",
operation_type=track.OperationType.Index,
meta_data={"success": True})
stats = reporter.calculate_results(
store, metrics.create_race(cfg, t, challenge))
del store
opm = stats.metrics("index")
self.assertEqual(
collections.OrderedDict([("min", 500), ("median", 1000),
("max", 2000), ("unit", "docs/s")]),
opm["throughput"])
self.assertEqual(collections.OrderedDict([("50", 220), ("100", 225)]),
opm["latency"])
self.assertEqual(collections.OrderedDict([("50", 200), ("100", 215)]),
opm["service_time"])
self.assertAlmostEqual(0.3333333333333333, opm["error_rate"])
