def float_param(self,
params,
name,
default_value,
min_value,
max_value,
min_operator=operator.le):
try:
value = float(params.get(name, default_value))
if min_operator(value, min_value) or value > max_value:
interval_min = "(" if min_operator is operator.le else "["
raise exceptions.InvalidSyntax(
"'{}' must be in the range {}{:.1f}, {:.1f}] but was {:.1f}"
.format(name, interval_min, min_value, max_value, value))
return value
except ValueError:
raise exceptions.InvalidSyntax("'{}' must be numeric".format(name))
def __init__(self, track, params, **kwargs):
super().__init__(track, params, **kwargs)
if len(track.indices) == 1:
default_index = track.indices[0].name
else:
default_index = None
index_name = params.get("index", default_index)
type_name = params.get("type")
request_cache = params.get("cache", False)
query_body = params.get("body", None)
query_body_params = params.get("body-params", None)
pages = params.get("pages", None)
results_per_page = params.get("results-per-page", None)
request_params = params.get("request-params", {})
self.query_params = {
"index": index_name,
"type": type_name,
"cache": request_cache,
# TODO: This is the old name, remove with Rally 1.0
"use_request_cache": request_cache,
"request-params": request_params,
# TODO: This is the old name, remove with Rally 1.0
"request_params": request_params,
"body": query_body
}
if not index_name:
raise exceptions.InvalidSyntax("'index' is mandatory")
if pages:
self.query_params["pages"] = pages
if results_per_page:
self.query_params["results-per-page"] = results_per_page
# TODO: This is the old name, remove with Rally 1.0
self.query_params["items_per_page"] = results_per_page
self.query_body_params = []
if query_body_params:
for param, data in query_body_params.items():
# TODO #365: Strictly check for allowed syntax. Be lenient in the pre-release and only interpret what's safely possible.
# build path based on param
# if not isinstance(data, list):
# raise exceptions.RallyError("%s in body-params defines %s but only lists are allowed. This may be a new syntax "
# "that is not recognized by this version. Please upgrade Rally." % (param, data))
if isinstance(data, list):
query_body_path = param.split(".")
b = self.query_params["body"]
# check early to ensure this path is actually contained in the body
try:
self.get_from_dict(b, query_body_path)
except KeyError:
raise exceptions.RallyError(
"The path %s could not be found within the query body %s."
% (param, b))
self.query_body_params.append((query_body_path, data))
def _render_template(env, variables, file_name):
try:
template = env.get_template(io.basename(file_name))
# force a new line at the end. Jinja seems to remove it.
return template.render(variables) + "\n"
except jinja2.exceptions.TemplateSyntaxError as e:
raise exceptions.InvalidSyntax("%s in %s" % (str(e), file_name))
except BaseException as e:
raise exceptions.SystemSetupError("%s in %s" % (str(e), file_name))
def target_throughput(self):
def numeric(v):
# While booleans can be converted to a number (False -> 0, True -> 1), we don't want to allow that here
return isinstance(v, numbers.Number) and not isinstance(v, bool)
target_throughput = self.params.get("target-throughput")
target_interval = self.params.get("target-interval")
if target_interval is not None and target_throughput is not None:
raise exceptions.InvalidSyntax(
f"Task [{self}] specifies target-interval [{target_interval}] and "
f"target-throughput [{target_throughput}] but only one of them is allowed."
)
value = None
unit = "ops/s"
if target_interval:
if not numeric(target_interval):
raise exceptions.InvalidSyntax(
f"Target interval [{target_interval}] for task [{self}] must be numeric."
)
value = 1 / float(target_interval)
elif target_throughput:
if isinstance(target_throughput, str):
matches = re.match(Task.THROUGHPUT_PATTERN, target_throughput)
if matches:
value = float(matches.group("value"))
unit = matches.group("unit")
else:
raise exceptions.InvalidSyntax(
f"Task [{self}] specifies invalid target throughput [{target_throughput}]."
)
elif numeric(target_throughput):
value = float(target_throughput)
else:
raise exceptions.InvalidSyntax(
f"Target throughput [{target_throughput}] for task [{self}] "
f"must be string or numeric.")
if value:
return Throughput(value, unit)
else:
return None
def ignore_response_error_level(self):
ignore_response_error_level = self.params.get("ignore-response-error-level")
if ignore_response_error_level and ignore_response_error_level not in Task.IGNORE_RESPONSE_ERROR_LEVEL_WHITELIST:
raise exceptions.InvalidSyntax(
f"Task [{self}] specifies ignore-response-error-level to [{ignore_response_error_level}] but "
f"the only allowed values are [{','.join(Task.IGNORE_RESPONSE_ERROR_LEVEL_WHITELIST)}]."
)
return ignore_response_error_level
def _render_template(self, loader, template_name, variables):
try:
env = jinja2.Environment(loader=loader)
for k, v in variables.items():
env.globals[k] = v
template = env.get_template(template_name)
return template.render()
except jinja2.exceptions.TemplateSyntaxError as e:
raise exceptions.InvalidSyntax("%s in %s" % (str(e), template_name))
except BaseException as e:
raise exceptions.SystemSetupError("%s in %s" % (str(e), template_name))
def __init__(self, track, params, **kwargs):
super().__init__(track, params, **kwargs)
self.only_if_exists = params.get("only-if-exists", True)
self.request_params = params.get("request-params", {})
self.template_definitions = []
if track.templates:
filter_template = params.get("template")
for template in track.templates:
if not filter_template or template.name == filter_template:
self.template_definitions.append((template.name, template.delete_matching_indices, template.pattern))
else:
try:
template = params["template"]
except KeyError:
raise exceptions.InvalidSyntax("Please set the property 'template' for the delete-index-template operation")
delete_matching = params.get("delete-matching-indices", False)
try:
index_pattern = params["index-pattern"] if delete_matching else None
except KeyError:
raise exceptions.InvalidSyntax("The property 'index-pattern' is required for delete-index-template if "
"'delete-matching-indices' is true.")
self.template_definitions.append((template, delete_matching, index_pattern))
def __init__(self, track, params, **kwargs):
super().__init__(track, params, **kwargs)
id_conflicts = params.get("conflicts", None)
if not id_conflicts:
self.id_conflicts = IndexIdConflict.NoConflicts
elif id_conflicts == "sequential":
self.id_conflicts = IndexIdConflict.SequentialConflicts
elif id_conflicts == "random":
self.id_conflicts = IndexIdConflict.RandomConflicts
else:
raise exceptions.InvalidSyntax("Unknown 'conflicts' setting [%s]" % id_conflicts)
self.corpora = self.used_corpora(track, params)
for corpus in self.corpora:
for document_set in corpus.documents:
if document_set.includes_action_and_meta_data and self.id_conflicts != IndexIdConflict.NoConflicts:
file_name = document_set.document_archive if document_set.has_compressed_corpus() else document_set.document_file
raise exceptions.InvalidSyntax("Cannot generate id conflicts [%s] as [%s] in document corpus [%s] already contains an "
"action and meta-data line." % (id_conflicts, file_name, corpus))
self.pipeline = params.get("pipeline", None)
try:
self.bulk_size = int(params["bulk-size"])
if self.bulk_size <= 0:
raise exceptions.InvalidSyntax("'bulk-size' must be positive but was %d" % self.bulk_size)
except KeyError:
raise exceptions.InvalidSyntax("Mandatory parameter 'bulk-size' is missing")
except ValueError:
raise exceptions.InvalidSyntax("'bulk-size' must be numeric")
try:
self.batch_size = int(params.get("batch-size", self.bulk_size))
if self.batch_size <= 0:
raise exceptions.InvalidSyntax("'batch-size' must be positive but was %d" % self.batch_size)
if self.batch_size < self.bulk_size:
raise exceptions.InvalidSyntax("'batch-size' must be greater than or equal to 'bulk-size'")
if self.batch_size % self.bulk_size != 0:
raise exceptions.InvalidSyntax("'batch-size' must be a multiple of 'bulk-size'")
except ValueError:
raise exceptions.InvalidSyntax("'batch-size' must be numeric")
def __init__(self, track, params, **kwargs):
super().__init__(track, params, **kwargs)
self.request_params = params.get("request-params", {})
self.index_definitions = []
if track.indices:
filter_idx = params.get("index")
if isinstance(filter_idx, str):
filter_idx = [filter_idx]
settings = params.get("settings")
for idx in track.indices:
if not filter_idx or idx.name in filter_idx:
body = idx.body
if body and settings:
if "settings" in body:
# merge (and potentially override)
body["settings"].update(settings)
else:
body["settings"] = settings
elif not body:
# this is just needed because we will output this in the middle of the benchmark and will thus write
# this on the same line as the progress message.
console.println("")
console.warn(
"Creating index %s based on deprecated type mappings. Please specify an index body instead. "
"For details please see the migration guide in the docs."
% idx.name,
logger=logger)
# TODO #366: Deprecate this syntax. We should only specify all mappings in the body property.
# check all types and merge their mappings
body = {"mappings": {}}
if settings:
body["settings"] = settings
for t in idx.types:
body["mappings"].update(t.mapping)
self.index_definitions.append((idx.name, body))
else:
try:
# only 'index' is mandatory, the body is optional (may be ok to create an index without a body)
idx = params["index"]
body = params.get("body")
if isinstance(idx, str):
idx = [idx]
for i in idx:
self.index_definitions.append((i, body))
except KeyError:
raise exceptions.InvalidSyntax(
"Please set the property 'index' for the create-index operation"
)
def __init__(self, indices, params):
super().__init__(indices, params)
id_conflicts = params.get("conflicts", None)
if not id_conflicts:
self.id_conflicts = IndexIdConflict.NoConflicts
elif id_conflicts == "sequential":
self.id_conflicts = IndexIdConflict.SequentialConflicts
elif id_conflicts == "random":
self.id_conflicts = IndexIdConflict.RandomConflicts
else:
raise exceptions.InvalidSyntax(
"Unknown index id conflict type [%s]." % id_conflicts)
self.pipeline = params.get("pipeline", None)
try:
self.bulk_size = int(params["bulk-size"])
if self.bulk_size <= 0:
raise exceptions.InvalidSyntax(
"'bulk-size' must be positive but was %d" % self.bulk_size)
except KeyError:
raise exceptions.InvalidSyntax(
"Mandatory parameter 'bulk-size' is missing")
except ValueError:
raise exceptions.InvalidSyntax("'bulk-size' must be numeric")
def __init__(self, track, params, **kwargs):
super().__init__(track, params, **kwargs)
self.request_params = params.get("request-params", {})
self.only_if_exists = params.get("only-if-exists", True)
self.index_definitions = []
target_index = params.get("index")
if target_index:
# TODO: Should we allow to delete multiple indices at once?
self.index_definitions.append(target_index)
elif track.indices:
for idx in track.indices:
self.index_definitions.append(idx.name)
else:
raise exceptions.InvalidSyntax(
"delete-index operation targets no index")
def __init__(self, track, params, **kwargs):
super().__init__(track, params, **kwargs)
self.request_params = params.get("request-params", {})
self.only_if_exists = params.get("only-if-exists", True)
self.index_definitions = []
target_index = params.get("index")
if target_index:
if isinstance(target_index, str):
target_index = [target_index]
for idx in target_index:
self.index_definitions.append(idx)
elif track.indices:
for idx in track.indices:
self.index_definitions.append(idx.name)
else:
raise exceptions.InvalidSyntax("delete-index operation targets no index")
def components(version):
"""
Determines components of a version string.
:param version: A version string in the format major.minor.path-suffix (suffix is optional)
:return: A dict with the keys "major", "minor", "patch" and optionally "suffix".
"""
matches = VERSIONS.match(version)
if matches:
version_components = {
"major": matches.group(1),
"minor": matches.group(2),
"patch": matches.group(3)
}
if matches.start(4) > 0:
version_components["suffix"] = matches.group(4)
return version_components
else:
raise exceptions.InvalidSyntax(
"version string '%s' does not conform to pattern '%s'" %
(version, VERSIONS.pattern))
def components(version, strict=True):
"""
Determines components of a version string.
:param version: A version string in the format major.minor.path-suffix (suffix is optional)
:param strict: Determines whether versions need to have at least "major", "minor" and "patch" defined. Default: True
:return: A tuple with four components determining "major", "minor", "patch" and "suffix" (any part except "major" may be `None`)
"""
versions_pattern = _versions_pattern(strict)
matches = versions_pattern.match(version)
if matches:
if matches.start(4) > 0:
return int(matches.group(1)), int(matches.group(2)), int(matches.group(3)), matches.group(4)
elif matches.start(3) > 0:
return int(matches.group(1)), int(matches.group(2)), int(matches.group(3)), None
elif matches.start(2) > 0:
return int(matches.group(1)), int(matches.group(2)), None, None
elif matches.start(1) > 0:
return int(matches.group(1)), None, None, None
else:
return int(version), None, None, None
raise exceptions.InvalidSyntax("version string '%s' does not conform to pattern '%s'" % (version, versions_pattern.pattern))
def __init__(self, track, params, **kwargs):
super().__init__(track, params, **kwargs)
self.request_params = params.get("request-params", {})
self.template_definitions = []
if track.templates:
filter_template = params.get("template")
settings = params.get("settings")
for template in track.templates:
if not filter_template or template.name == filter_template:
body = template.content
if body and settings:
if "settings" in body:
# merge (and potentially override)
body["settings"].update(settings)
else:
body["settings"] = settings
self.template_definitions.append((template.name, body))
else:
try:
self.template_definitions.append((params["template"], params["body"]))
except KeyError:
raise exceptions.InvalidSyntax("Please set the properties 'template' and 'body' for the create-index-template operation")
def __init__(self, indices, params):
super().__init__(indices, params)
id_conflicts = params.get("conflicts", None)
if not id_conflicts:
self.id_conflicts = IndexIdConflict.NoConflicts
elif id_conflicts == "sequential":
self.id_conflicts = IndexIdConflict.SequentialConflicts
elif id_conflicts == "random":
self.id_conflicts = IndexIdConflict.RandomConflicts
else:
raise exceptions.InvalidSyntax("Unknown 'conflicts' setting [%s]" %
id_conflicts)
action_metadata = params.get("action-and-meta-data", "generate")
if action_metadata == "generate":
self.action_metadata = ActionMetaData.Generate
elif action_metadata == "none":
self.action_metadata = ActionMetaData.NoMetaData
elif action_metadata == "sourcefile":
self.action_metadata = ActionMetaData.SourceFile
else:
raise exceptions.InvalidSyntax(
"Unknown 'action-and-meta-data' setting [%s]" %
action_metadata)
if self.action_metadata != ActionMetaData.Generate and self.id_conflicts != IndexIdConflict.NoConflicts:
raise exceptions.InvalidSyntax(
"Cannot generate id conflicts [%s] when 'action-and-meta-data' is [%s]."
% (id_conflicts, action_metadata))
self.pipeline = params.get("pipeline", None)
try:
self.bulk_size = int(params["bulk-size"])
if self.bulk_size <= 0:
raise exceptions.InvalidSyntax(
"'bulk-size' must be positive but was %d" % self.bulk_size)
except KeyError:
raise exceptions.InvalidSyntax(
"Mandatory parameter 'bulk-size' is missing")
except ValueError:
raise exceptions.InvalidSyntax("'bulk-size' must be numeric")
try:
self.batch_size = int(params.get("batch-size", self.bulk_size))
if self.batch_size <= 0:
raise exceptions.InvalidSyntax(
"'batch-size' must be positive but was %d" %
self.batch_size)
if self.batch_size < self.bulk_size:
raise exceptions.InvalidSyntax(
"'batch-size' must be greater than or equal to 'bulk-size'"
)
if self.batch_size % self.bulk_size != 0:
raise exceptions.InvalidSyntax(
"'batch-size' must be a multiple of 'bulk-size'")
except ValueError:
raise exceptions.InvalidSyntax("'batch-size' must be numeric")
if len(indices) == 1 and len(indices[0].types) == 1:
default_index = indices[0].name
else:
default_index = None
self.index_name = params.get("index", default_index)
def __init__(self, track, params, **kwargs):
super().__init__(track, params, **kwargs)
id_conflicts = params.get("conflicts", None)
if not id_conflicts:
self.id_conflicts = IndexIdConflict.NoConflicts
elif id_conflicts == "sequential":
self.id_conflicts = IndexIdConflict.SequentialConflicts
elif id_conflicts == "random":
self.id_conflicts = IndexIdConflict.RandomConflicts
else:
raise exceptions.InvalidSyntax("Unknown 'conflicts' setting [%s]" %
id_conflicts)
if self.id_conflicts != IndexIdConflict.NoConflicts:
self.conflict_probability = self.float_param(
params,
name="conflict-probability",
default_value=25,
min_value=0,
max_value=100,
min_operator=operator.lt)
self.on_conflict = params.get("on-conflict", "index")
if self.on_conflict not in ["index", "update"]:
raise exceptions.InvalidSyntax(
"Unknown 'on-conflict' setting [{}]".format(
self.on_conflict))
self.recency = self.float_param(params,
name="recency",
default_value=0,
min_value=0,
max_value=1,
min_operator=operator.lt)
else:
self.conflict_probability = None
self.on_conflict = None
self.recency = None
self.corpora = self.used_corpora(track, params)
for corpus in self.corpora:
for document_set in corpus.documents:
if document_set.includes_action_and_meta_data and self.id_conflicts != IndexIdConflict.NoConflicts:
file_name = document_set.document_archive if document_set.has_compressed_corpus(
) else document_set.document_file
raise exceptions.InvalidSyntax(
"Cannot generate id conflicts [%s] as [%s] in document corpus [%s] already contains an "
"action and meta-data line." %
(id_conflicts, file_name, corpus))
self.pipeline = params.get("pipeline", None)
try:
self.bulk_size = int(params["bulk-size"])
if self.bulk_size <= 0:
raise exceptions.InvalidSyntax(
"'bulk-size' must be positive but was %d" % self.bulk_size)
except KeyError:
raise exceptions.InvalidSyntax(
"Mandatory parameter 'bulk-size' is missing")
except ValueError:
raise exceptions.InvalidSyntax("'bulk-size' must be numeric")
try:
self.batch_size = int(params.get("batch-size", self.bulk_size))
if self.batch_size <= 0:
raise exceptions.InvalidSyntax(
"'batch-size' must be positive but was %d" %
self.batch_size)
if self.batch_size < self.bulk_size:
raise exceptions.InvalidSyntax(
"'batch-size' must be greater than or equal to 'bulk-size'"
)
if self.batch_size % self.bulk_size != 0:
raise exceptions.InvalidSyntax(
"'batch-size' must be a multiple of 'bulk-size'")
except ValueError:
raise exceptions.InvalidSyntax("'batch-size' must be numeric")
self.ingest_percentage = self.float_param(params,
name="ingest-percentage",
default_value=100,
min_value=0,
max_value=100)
