def open(self, invocation, track_name, challenge_name, car_name, create=False):
"""
Opens a metrics store for a specific invocation, track, challenge and car.
:param invocation: The invocation (timestamp).
:param track_name: Track name.
:param challenge_name: Challenge name.
:param car_name: Car name.
:param create: True if an index should be created (if necessary). This is typically True, when attempting to write metrics and
False when it is just opened for reading (as we can assume all necessary indices exist at this point).
"""
self._invocation = time.to_iso8601(invocation)
self._track = track_name
self._challenge = challenge_name
self._car = car_name
logger.info(
"Opening metrics store for invocation=[%s], track=[%s], challenge=[%s], car=[%s]"
% (self._invocation, track_name, challenge_name, car_name)
)
user_tag = self._config.opts("system", "user.tag", mandatory=False)
if user_tag and user_tag.strip() != "":
try:
user_tag_key, user_tag_value = user_tag.split(":")
# prefix user tag with "tag_" in order to avoid clashes with our internal meta data
self.add_meta_info(MetaInfoScope.cluster, None, "tag_%s" % user_tag_key, user_tag_value)
except ValueError:
msg = "User tag key and value have to separated by a ':'. Invalid value [%s]" % user_tag
logger.exception(msg)
raise exceptions.SystemSetupError(msg)
self._stop_watch.start()
def open(self,
invocation,
track_name,
challenge_name,
car_name,
create=False):
"""
Opens a metrics store for a specific invocation, track, challenge and car.
:param invocation: The invocation (timestamp).
:param track_name: Track name.
:param challenge_name: Challenge name.
:param car_name: Car name.
:param create: True if an index should be created (if necessary). This is typically True, when attempting to write metrics and
False when it is just opened for reading (as we can assume all necessary indices exist at this point).
"""
self._invocation = time.to_iso8601(invocation)
self._track = track_name
self._challenge = challenge_name
self._car = car_name
logger.info(
"Opening metrics store for invocation=[%s], track=[%s], challenge=[%s], car=[%s]"
% (self._invocation, track_name, challenge_name, car_name))
user_tag = self._config.opts("system", "user.tag", mandatory=False)
if user_tag and user_tag.strip() != "":
try:
user_tag_key, user_tag_value = user_tag.split(":")
# prefix user tag with "tag_" in order to avoid clashes with our internal meta data
self.add_meta_info(MetaInfoScope.cluster, None,
"tag_%s" % user_tag_key, user_tag_value)
except ValueError:
msg = "User tag key and value have to separated by a ':'. Invalid value [%s]" % user_tag
logger.exception(msg)
raise exceptions.SystemSetupError(msg)
self._stop_watch.start()
def open(self, invocation, track_name, track_setup_name, create=False):
"""
Opens a metrics store for a specific invocation, track and track setup.
:param invocation: The invocation (timestamp).
:param track_name: Track name.
:param track_setup_name: Track setup name.
:param create: True if an index should be created (if necessary). This is typically True, when attempting to write metrics and
False when it is just opened for reading (as we can assume all necessary indices exist at this point).
"""
self._invocation = time.to_iso8601(invocation)
self._track = track_name
self._track_setup = track_setup_name
self._index = index_name(invocation)
self._docs = []
# reduce a bit of noise in the metrics cluster log
if create and not self._client.exists(index=self._index):
self._client.put_template("rally", self._get_template())
self._client.create_index(index=self._index)
# ensure we can search immediately after opening
self._client.refresh(index=self._index)
user_tag = self._config.opts("system", "user.tag", mandatory=False)
if user_tag and user_tag.strip() != "":
try:
user_tag_key, user_tag_value = user_tag.split(":")
# prefix user tag with "tag_" in order to avoid clashes with our internal meta data
self.add_meta_info(MetaInfoScope.cluster, None, "tag_%s" % user_tag_key, user_tag_value)
except ValueError:
msg = "User tag key and value have to separated by a ':'. Invalid value [%s]" % user_tag
logger.exception(msg)
raise exceptions.SystemSetupError(msg)
self._stop_watch.start()
def store_race(self, t):
# always update the mapping to the latest version
self.client.put_template("rally", self.index_template_provider.template())
trial_timestamp = self.config.opts("meta", "time.start")
laps = self.config.opts("benchmarks", "laps")
selected_challenge = {}
for challenge in t.challenges:
if challenge.name == self.config.opts("benchmarks", "challenge"):
selected_challenge["name"] = challenge.name
selected_challenge["operations"] = []
for tasks in challenge.schedule:
for task in tasks:
selected_challenge["operations"].append(task.operation.name)
doc = {
"environment": self.environment_name,
"trial-timestamp": time.to_iso8601(trial_timestamp),
"pipeline": self.config.opts("system", "pipeline"),
"revision": self.config.opts("source", "revision"),
"distribution-version": self.config.opts("source", "distribution.version"),
"laps": laps,
"track": t.name,
"selected-challenge": selected_challenge,
"car": self.config.opts("benchmarks", "car"),
"target-hosts": [
"%s:%s" % (i["host"], i["port"]) for i in self.config.opts("launcher", "external.target.hosts")
],
"user-tag": self.config.opts("system", "user.tag"),
}
self.client.index(index_name(trial_timestamp), EsRaceStore.RACE_DOC_TYPE, doc)
def list_races(cfg):
print("Recent races:\n")
races = []
for race in race_store(cfg).list():
races.append([time.to_iso8601(race.trial_timestamp), race.track, race.challenge, race.car, race.user_tag])
print(tabulate.tabulate(races, headers=["Race Timestamp", "Track", "Challenge", "Car", "User Tag"]))
def store_race(self, t):
# always update the mapping to the latest version
self.client.put_template("rally", self.index_template_provider.template())
trial_timestamp = self.config.opts("meta", "time.start")
laps = self.config.opts("benchmarks", "laps")
selected_challenge = {}
for challenge in t.challenges:
if challenge.name == self.config.opts("benchmarks", "challenge"):
selected_challenge["name"] = challenge.name
selected_challenge["operations"] = []
for tasks in challenge.schedule:
for task in tasks:
selected_challenge["operations"].append(task.operation.name)
doc = {
"environment": self.environment_name,
"trial-timestamp": time.to_iso8601(trial_timestamp),
"pipeline": self.config.opts("system", "pipeline"),
"revision": self.config.opts("source", "revision"),
"distribution-version": self.config.opts("source", "distribution.version"),
"laps": laps,
"track": t.name,
"selected-challenge": selected_challenge,
"car": self.config.opts("benchmarks", "car"),
"target-hosts": ["%s:%s" % (i["host"], i["port"]) for i in self.config.opts("launcher", "external.target.hosts")],
"user-tag": self.config.opts("system", "user.tag")
}
self.client.index(index_name(trial_timestamp), EsRaceStore.RACE_DOC_TYPE, doc)
def store_race(self, track, hosts, revision, distribution_version):
laps = self.config.opts("race", "laps")
challenge = track.find_challenge_or_default(self.config.opts("track", "challenge.name"))
selected_challenge = {
"name": challenge.name,
"operations": []
}
for tasks in challenge.schedule:
for task in tasks:
selected_challenge["operations"].append(task.operation.name)
doc = {
"environment": self.environment_name,
"trial-timestamp": time.to_iso8601(self.trial_timestamp),
"pipeline": self.config.opts("race", "pipeline"),
"revision": revision,
"distribution-version": distribution_version,
"laps": laps,
"track": track.name,
"selected-challenge": selected_challenge,
"car": self.config.opts("mechanic", "car.name"),
"target-hosts": ["%s:%s" % (i["host"], i["port"]) for i in hosts],
"user-tag": self.config.opts("race", "user.tag")
}
self.current_race = Race(doc)
self._store(doc)
def test_store_race(self):
from esrally import time
schedule = [
track.Task("index #1",
track.Operation("index", track.OperationType.Bulk))
]
t = track.Track(
name="unittest",
indices=[track.Index(name="tests", types=["test-type"])],
challenges=[
track.Challenge(name="index", default=True, schedule=schedule)
])
race = metrics.Race(rally_version="0.4.4",
environment_name="unittest",
trial_id=FileRaceStoreTests.TRIAL_ID,
trial_timestamp=FileRaceStoreTests.TRIAL_TIMESTAMP,
pipeline="from-sources",
user_tags={"os": "Linux"},
track=t,
track_params={"clients": 12},
challenge=t.default_challenge,
car="4gheap",
total_laps=12,
cluster=FileRaceStoreTests.DictHolder({
"distribution-version":
"5.0.0",
"nodes": [{
"node_name": "node0",
"ip": "127.0.0.1"
}]
}),
lap_results=[],
results=FileRaceStoreTests.DictHolder({
"young_gc_time":
100,
"old_gc_time":
5,
"op_metrics": [{
"task": "index #1",
"operation": "index",
"throughput": {
"min": 1000,
"median": 1250,
"max": 1500,
"unit": "docs/s"
}
}]
}))
self.race_store.store_race(race)
retrieved_race = self.race_store.find_by_timestamp(
timestamp=time.to_iso8601(FileRaceStoreTests.TRIAL_TIMESTAMP))
self.assertEqual(race.trial_timestamp, retrieved_race.trial_timestamp)
self.assertEqual(1, len(self.race_store.list()))
def list_races(cfg):
races = []
for race in race_store(cfg).list():
races.append([time.to_iso8601(race.trial_timestamp), race.track, race.challenge, race.car, race.user_tag])
if len(races) > 0:
console.println("\nRecent races:\n")
console.println(tabulate.tabulate(races, headers=["Race Timestamp", "Track", "Challenge", "Car", "User Tag"]))
else:
console.println("")
console.println("No recent races found.")
def list_races(cfg):
races = []
for race in race_store(cfg).list():
races.append([time.to_iso8601(race.trial_timestamp), race.track, race.challenge, race.car, race.user_tag])
if len(races) > 0:
console.println("\nRecent races:\n")
console.println(tabulate.tabulate(races, headers=["Race Timestamp", "Track", "Challenge", "Car", "User Tag"]))
else:
console.println("")
console.println("No recent races found.")
def store_race(self):
trial_timestamp = self.config.opts("meta", "time.start")
doc = {
"environment": self.environment_name,
"trial-timestamp": time.to_iso8601(trial_timestamp),
"pipeline": self.config.opts("system", "pipeline"),
"revision": self.config.opts("source", "revision"),
"distribution-version": self.config.opts("source", "distribution.version"),
"track": self.config.opts("system", "track"),
"track-setups": self.config.opts("benchmarks", "tracksetups.selected"),
"target-hosts": self.config.opts("launcher", "external.target.hosts"),
"user-tag": self.config.opts("system", "user.tag")
}
self.client.index(index_name(trial_timestamp), RaceStore.RACE_DOC_TYPE, doc)
def test_store_race(self):
from esrally import time
schedule = [
track.Task(track.Operation("index", track.OperationType.Index))
]
t = track.Track(name="unittest", short_description="unittest track",
source_root_url="http://example.org",
indices=[track.Index(name="tests", auto_managed=True, types=[track.Type(name="test-type", mapping={})])],
challenges=[
track.Challenge(name="index", description="Index", default=True, index_settings=None, schedule=schedule)
])
race = metrics.Race(rally_version="0.4.4", environment_name="unittest", trial_timestamp=FileRaceStoreTests.TRIAL_TIMESTAMP,
pipeline="from-sources", user_tag="let-me-test", track=t, challenge=t.default_challenge, car="4gheap",
total_laps=12,
cluster=FileRaceStoreTests.DictHolder(
{
"distribution-version": "5.0.0",
"nodes": [
{"node_name": "node0", "ip": "127.0.0.1"}
]
}),
lap_results=[],
results=FileRaceStoreTests.DictHolder(
{
"young_gc_time": 100,
"old_gc_time": 5,
"op_metrics": [
{
"operation": "index",
"throughput": {
"min": 1000,
"median": 1250,
"max": 1500,
"unit": "docs/s"
}
}
]
})
)
self.race_store.store_race(race)
retrieved_race = self.race_store.find_by_timestamp(timestamp=time.to_iso8601(FileRaceStoreTests.TRIAL_TIMESTAMP))
self.assertEqual(race.trial_timestamp, retrieved_race.trial_timestamp)
self.assertEqual(1, len(self.race_store.list()))
def configure_logging(cfg):
start_time = rtime.to_iso8601(cfg.opts("system", "time.start"))
logging_output = cfg.opts("system", "logging.output")
profiling_enabled = cfg.opts("driver", "profiling")
if logging_output == "file":
log_file = application_log_file_path(start_time)
log_dir = os.path.dirname(log_file)
io.ensure_dir(log_dir)
console.info("Writing logs to %s" % log_file)
# there is an old log file lying around -> backup
if os.path.exists(log_file):
os.rename(
log_file,
"%s-bak-%d.log" % (log_file, int(os.path.getctime(log_file))))
ch = logging.FileHandler(filename=log_file, mode="a")
else:
ch = logging.StreamHandler(stream=sys.stdout)
log_level = logging.INFO
ch.setLevel(log_level)
formatter = logging.Formatter(
"%(asctime)s,%(msecs)d PID:%(process)d %(name)s %(levelname)s %(message)s",
datefmt="%Y-%m-%d %H:%M:%S")
formatter.converter = time.gmtime
ch.setFormatter(formatter)
# Remove all handlers associated with the root logger object so we can start over with an entirely fresh log configuration
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.root.addHandler(ch)
logging.getLogger("elasticsearch").setLevel(logging.WARNING)
if profiling_enabled:
profile_file = "%s/profile.log" % application_log_dir_path()
log_dir = os.path.dirname(profile_file)
io.ensure_dir(log_dir)
console.info("Writing driver profiling data to %s" % profile_file)
handler = logging.FileHandler(filename=profile_file, encoding="UTF-8")
handler.setFormatter(formatter)
profile_logger = logging.getLogger("rally.profile")
profile_logger.setLevel(logging.INFO)
profile_logger.addHandler(handler)
def configure_logging(cfg):
start_time = rtime.to_iso8601(cfg.opts("system", "time.start"))
logging_output = cfg.opts("system", "logging.output")
profiling_enabled = cfg.opts("driver", "profiling")
if logging_output == "file":
log_file = application_log_file_path(start_time)
log_dir = os.path.dirname(log_file)
io.ensure_dir(log_dir)
console.info("Writing logs to %s" % log_file)
# there is an old log file lying around -> backup
if os.path.exists(log_file):
os.rename(log_file, "%s-bak-%d.log" % (log_file, int(os.path.getctime(log_file))))
ch = logging.FileHandler(filename=log_file, mode="a")
else:
ch = logging.StreamHandler(stream=sys.stdout)
log_level = logging.INFO
ch.setLevel(log_level)
formatter = logging.Formatter("%(asctime)s,%(msecs)d PID:%(process)d %(name)s %(levelname)s %(message)s", datefmt="%Y-%m-%d %H:%M:%S")
formatter.converter = time.gmtime
ch.setFormatter(formatter)
# Remove all handlers associated with the root logger object so we can start over with an entirely fresh log configuration
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.root.addHandler(ch)
logging.getLogger("elasticsearch").setLevel(logging.WARNING)
if profiling_enabled:
profile_file = "%s/profile.log" % application_log_dir_path()
log_dir = os.path.dirname(profile_file)
io.ensure_dir(log_dir)
console.info("Writing driver profiling data to %s" % profile_file)
handler = logging.FileHandler(filename=profile_file, encoding="UTF-8")
handler.setFormatter(formatter)
profile_logger = logging.getLogger("rally.profile")
profile_logger.setLevel(logging.INFO)
profile_logger.addHandler(handler)
def open(self, invocation, track_name, track_setup_name, create=False):
"""
Opens a metrics store for a specific invocation, track and track setup.
:param invocation: The invocation (timestamp).
:param track_name: Track name.
:param track_setup_name: Track setup name.
:param create: True if an index should be created (if necessary). This is typically True, when attempting to write metrics and
False when it is just opened for reading (as we can assume all necessary indices exist at this point).
"""
self._invocation = time.to_iso8601(invocation)
self._track = track_name
self._track_setup = track_setup_name
self._index = "rally-%04d" % invocation.year
self._docs = []
# reduce a bit of noise in the metrics cluster log
if create and not self._client.exists(index=self._index):
self._client.put_template("rally", self._get_template())
self._client.create_index(index=self._index)
# ensure we can search immediately after opening
self._client.refresh(index=self._index)
self._stop_watch.start()
def open(self, invocation, track_name, track_setup_name, create=False):
"""
Opens a metrics store for a specific invocation, track and track setup.
:param invocation: The invocation (timestamp).
:param track_name: Track name.
:param track_setup_name: Track setup name.
:param create: True if an index should be created (if necessary). This is typically True, when attempting to write metrics and
False when it is just opened for reading (as we can assume all necessary indices exist at this point).
"""
self._invocation = time.to_iso8601(invocation)
self._track = track_name
self._track_setup = track_setup_name
self._index = "rally-%04d" % invocation.year
self._docs = []
# reduce a bit of noise in the metrics cluster log
if create and not self._client.exists(index=self._index):
self._client.put_template("rally", self._get_template())
self._client.create_index(index=self._index)
# ensure we can search immediately after opening
self._client.refresh(index=self._index)
self._stop_watch.start()
def store_race(self, t):
# always update the mapping to the latest version
self.client.put_template("rally", self.index_template_provider.template())
trial_timestamp = self.config.opts("meta", "time.start")
selected_challenge = {}
for challenge in t.challenges:
if challenge.name == self.config.opts("benchmarks", "challenge"):
selected_challenge["name"] = challenge.name
if track.BenchmarkPhase.index in challenge.benchmark:
selected_challenge["benchmark-phase-index"] = True
if track.BenchmarkPhase.stats in challenge.benchmark:
selected_challenge["benchmark-phase-stats"] = {
"sample-size": challenge.benchmark[track.BenchmarkPhase.stats].iteration_count
}
if track.BenchmarkPhase.search in challenge.benchmark:
c = challenge.benchmark[track.BenchmarkPhase.search]
selected_challenge["benchmark-phase-search"] = {
"queries": [q.name for q in c.queries],
"sample-size": c.iteration_count
}
doc = {
"environment": self.environment_name,
"trial-timestamp": time.to_iso8601(trial_timestamp),
"pipeline": self.config.opts("system", "pipeline"),
"revision": self.config.opts("source", "revision"),
"distribution-version": self.config.opts("source", "distribution.version"),
"track": t.name,
"selected-challenge": selected_challenge,
"car": self.config.opts("benchmarks", "car"),
"rounds": self.config.opts("benchmarks", "rounds"),
"target-hosts": self.config.opts("launcher", "external.target.hosts"),
"user-tag": self.config.opts("system", "user.tag")
}
self.client.index(index_name(trial_timestamp), EsRaceStore.RACE_DOC_TYPE, doc)
def test_store_race(self):
from esrally import time
schedule = [
track.Task(track.Operation("index", track.OperationType.Index))
]
t = track.Track(name="unittest",
short_description="unittest track",
source_root_url="http://example.org",
indices=[
track.Index(name="tests",
auto_managed=True,
types=[
track.Type(name="test-type",
mapping_file=None)
])
],
challenges=[
track.Challenge(name="index",
description="Index",
default=True,
index_settings=None,
schedule=schedule)
])
race = metrics.Race(rally_version="0.4.4",
environment_name="unittest",
trial_timestamp=FileRaceStoreTests.TRIAL_TIMESTAMP,
pipeline="from-sources",
user_tag="let-me-test",
track=t,
challenge=t.default_challenge,
car="4gheap",
total_laps=12,
cluster=FileRaceStoreTests.DictHolder({
"distribution-version":
"5.0.0",
"nodes": [{
"node_name": "node0",
"ip": "127.0.0.1"
}]
}),
lap_results=[],
results=FileRaceStoreTests.DictHolder({
"young_gc_time":
100,
"old_gc_time":
5,
"op_metrics": [{
"operation": "index",
"throughput": {
"min": 1000,
"median": 1250,
"max": 1500,
"unit": "docs/s"
}
}]
}))
self.race_store.store_race(race)
retrieved_race = self.race_store.find_by_timestamp(
timestamp=time.to_iso8601(FileRaceStoreTests.TRIAL_TIMESTAMP))
self.assertEqual(race.trial_timestamp, retrieved_race.trial_timestamp)
self.assertEqual(1, len(self.race_store.list()))
def configure_logging(cfg):
start_time = rtime.to_iso8601(cfg.opts("system", "time.start"))
logging_output = cfg.opts("system", "logging.output")
profiling_enabled = cfg.opts("driver", "profiling")
if logging_output == "file":
log_file = application_log_file_path(start_time)
log_dir = os.path.dirname(log_file)
io.ensure_dir(log_dir)
console.info("Writing logs to %s" % log_file)
# there is an old log file lying around -> backup
if os.path.exists(log_file):
os.rename(log_file, "%s-bak-%d.log" % (log_file, int(os.path.getctime(log_file))))
ch = logging.FileHandler(filename=log_file, mode="a")
else:
ch = logging.StreamHandler(stream=sys.stdout)
log_level = logging.INFO
ch.setLevel(log_level)
formatter = logging.Formatter("%(asctime)s,%(msecs)d PID:%(process)d %(name)s %(levelname)s %(message)s", datefmt="%Y-%m-%d %H:%M:%S")
formatter.converter = time.gmtime
ch.setFormatter(formatter)
# Remove all handlers associated with the root logger object so we can start over with an entirely fresh log configuration
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.root.addHandler(ch)
logging.getLogger("elasticsearch").setLevel(logging.WARNING)
# Avoid failures such as the following (shortened a bit):
#
# ---------------------------------------------------------------------------------------------
# "esrally/driver/driver.py", line 220, in create_client
# "thespian-3.8.0-py3.5.egg/thespian/actors.py", line 187, in createActor
# [...]
# "thespian-3.8.0-py3.5.egg/thespian/system/multiprocCommon.py", line 348, in _startChildActor
# "python3.5/multiprocessing/process.py", line 105, in start
# "python3.5/multiprocessing/context.py", line 267, in _Popen
# "python3.5/multiprocessing/popen_fork.py", line 18, in __init__
# sys.stderr.flush()
#
# OSError: [Errno 5] Input/output error
# ---------------------------------------------------------------------------------------------
#
# This is caused by urllib3 wanting to send warnings about insecure SSL connections to stderr when we disable them (in client.py) with:
#
# urllib3.disable_warnings()
#
# The filtering functionality of the warnings module causes the error above on some systems. If we instead redirect the warning output
# to our logs instead of stderr (which is the warnings module's default), we can disable warnings safely.
logging.captureWarnings(True)
if profiling_enabled:
profile_file = "%s/profile.log" % application_log_dir_path()
log_dir = os.path.dirname(profile_file)
io.ensure_dir(log_dir)
console.info("Writing driver profiling data to %s" % profile_file)
handler = logging.FileHandler(filename=profile_file, encoding="UTF-8")
handler.setFormatter(formatter)
profile_logger = logging.getLogger("rally.profile")
profile_logger.setLevel(logging.INFO)
profile_logger.addHandler(handler)