def float_param(self, params, name, default_value, min_value, max_value, min_operator=operator.le): try: value = float(params.get(name, default_value)) if min_operator(value, min_value) or value > max_value: interval_min = "(" if min_operator is operator.le else "[" raise exceptions.InvalidSyntax( "'{}' must be in the range {}{:.1f}, {:.1f}] but was {:.1f}" .format(name, interval_min, min_value, max_value, value)) return value except ValueError: raise exceptions.InvalidSyntax("'{}' must be numeric".format(name))
def __init__(self, track, params, **kwargs): super().__init__(track, params, **kwargs) if len(track.indices) == 1: default_index = track.indices[0].name else: default_index = None index_name = params.get("index", default_index) type_name = params.get("type") request_cache = params.get("cache", False) query_body = params.get("body", None) query_body_params = params.get("body-params", None) pages = params.get("pages", None) results_per_page = params.get("results-per-page", None) request_params = params.get("request-params", {}) self.query_params = { "index": index_name, "type": type_name, "cache": request_cache, # TODO: This is the old name, remove with Rally 1.0 "use_request_cache": request_cache, "request-params": request_params, # TODO: This is the old name, remove with Rally 1.0 "request_params": request_params, "body": query_body } if not index_name: raise exceptions.InvalidSyntax("'index' is mandatory") if pages: self.query_params["pages"] = pages if results_per_page: self.query_params["results-per-page"] = results_per_page # TODO: This is the old name, remove with Rally 1.0 self.query_params["items_per_page"] = results_per_page self.query_body_params = [] if query_body_params: for param, data in query_body_params.items(): # TODO #365: Strictly check for allowed syntax. Be lenient in the pre-release and only interpret what's safely possible. # build path based on param # if not isinstance(data, list): # raise exceptions.RallyError("%s in body-params defines %s but only lists are allowed. This may be a new syntax " # "that is not recognized by this version. Please upgrade Rally." % (param, data)) if isinstance(data, list): query_body_path = param.split(".") b = self.query_params["body"] # check early to ensure this path is actually contained in the body try: self.get_from_dict(b, query_body_path) except KeyError: raise exceptions.RallyError( "The path %s could not be found within the query body %s." % (param, b)) self.query_body_params.append((query_body_path, data))
def _render_template(env, variables, file_name): try: template = env.get_template(io.basename(file_name)) # force a new line at the end. Jinja seems to remove it. return template.render(variables) + "\n" except jinja2.exceptions.TemplateSyntaxError as e: raise exceptions.InvalidSyntax("%s in %s" % (str(e), file_name)) except BaseException as e: raise exceptions.SystemSetupError("%s in %s" % (str(e), file_name))
def target_throughput(self): def numeric(v): # While booleans can be converted to a number (False -> 0, True -> 1), we don't want to allow that here return isinstance(v, numbers.Number) and not isinstance(v, bool) target_throughput = self.params.get("target-throughput") target_interval = self.params.get("target-interval") if target_interval is not None and target_throughput is not None: raise exceptions.InvalidSyntax( f"Task [{self}] specifies target-interval [{target_interval}] and " f"target-throughput [{target_throughput}] but only one of them is allowed." ) value = None unit = "ops/s" if target_interval: if not numeric(target_interval): raise exceptions.InvalidSyntax( f"Target interval [{target_interval}] for task [{self}] must be numeric." ) value = 1 / float(target_interval) elif target_throughput: if isinstance(target_throughput, str): matches = re.match(Task.THROUGHPUT_PATTERN, target_throughput) if matches: value = float(matches.group("value")) unit = matches.group("unit") else: raise exceptions.InvalidSyntax( f"Task [{self}] specifies invalid target throughput [{target_throughput}]." ) elif numeric(target_throughput): value = float(target_throughput) else: raise exceptions.InvalidSyntax( f"Target throughput [{target_throughput}] for task [{self}] " f"must be string or numeric.") if value: return Throughput(value, unit) else: return None
def ignore_response_error_level(self): ignore_response_error_level = self.params.get("ignore-response-error-level") if ignore_response_error_level and ignore_response_error_level not in Task.IGNORE_RESPONSE_ERROR_LEVEL_WHITELIST: raise exceptions.InvalidSyntax( f"Task [{self}] specifies ignore-response-error-level to [{ignore_response_error_level}] but " f"the only allowed values are [{','.join(Task.IGNORE_RESPONSE_ERROR_LEVEL_WHITELIST)}]." ) return ignore_response_error_level
def _render_template(self, loader, template_name, variables): try: env = jinja2.Environment(loader=loader) for k, v in variables.items(): env.globals[k] = v template = env.get_template(template_name) return template.render() except jinja2.exceptions.TemplateSyntaxError as e: raise exceptions.InvalidSyntax("%s in %s" % (str(e), template_name)) except BaseException as e: raise exceptions.SystemSetupError("%s in %s" % (str(e), template_name))
def __init__(self, track, params, **kwargs): super().__init__(track, params, **kwargs) self.only_if_exists = params.get("only-if-exists", True) self.request_params = params.get("request-params", {}) self.template_definitions = [] if track.templates: filter_template = params.get("template") for template in track.templates: if not filter_template or template.name == filter_template: self.template_definitions.append((template.name, template.delete_matching_indices, template.pattern)) else: try: template = params["template"] except KeyError: raise exceptions.InvalidSyntax("Please set the property 'template' for the delete-index-template operation") delete_matching = params.get("delete-matching-indices", False) try: index_pattern = params["index-pattern"] if delete_matching else None except KeyError: raise exceptions.InvalidSyntax("The property 'index-pattern' is required for delete-index-template if " "'delete-matching-indices' is true.") self.template_definitions.append((template, delete_matching, index_pattern))
def __init__(self, track, params, **kwargs): super().__init__(track, params, **kwargs) id_conflicts = params.get("conflicts", None) if not id_conflicts: self.id_conflicts = IndexIdConflict.NoConflicts elif id_conflicts == "sequential": self.id_conflicts = IndexIdConflict.SequentialConflicts elif id_conflicts == "random": self.id_conflicts = IndexIdConflict.RandomConflicts else: raise exceptions.InvalidSyntax("Unknown 'conflicts' setting [%s]" % id_conflicts) self.corpora = self.used_corpora(track, params) for corpus in self.corpora: for document_set in corpus.documents: if document_set.includes_action_and_meta_data and self.id_conflicts != IndexIdConflict.NoConflicts: file_name = document_set.document_archive if document_set.has_compressed_corpus() else document_set.document_file raise exceptions.InvalidSyntax("Cannot generate id conflicts [%s] as [%s] in document corpus [%s] already contains an " "action and meta-data line." % (id_conflicts, file_name, corpus)) self.pipeline = params.get("pipeline", None) try: self.bulk_size = int(params["bulk-size"]) if self.bulk_size <= 0: raise exceptions.InvalidSyntax("'bulk-size' must be positive but was %d" % self.bulk_size) except KeyError: raise exceptions.InvalidSyntax("Mandatory parameter 'bulk-size' is missing") except ValueError: raise exceptions.InvalidSyntax("'bulk-size' must be numeric") try: self.batch_size = int(params.get("batch-size", self.bulk_size)) if self.batch_size <= 0: raise exceptions.InvalidSyntax("'batch-size' must be positive but was %d" % self.batch_size) if self.batch_size < self.bulk_size: raise exceptions.InvalidSyntax("'batch-size' must be greater than or equal to 'bulk-size'") if self.batch_size % self.bulk_size != 0: raise exceptions.InvalidSyntax("'batch-size' must be a multiple of 'bulk-size'") except ValueError: raise exceptions.InvalidSyntax("'batch-size' must be numeric")
def __init__(self, track, params, **kwargs): super().__init__(track, params, **kwargs) self.request_params = params.get("request-params", {}) self.index_definitions = [] if track.indices: filter_idx = params.get("index") if isinstance(filter_idx, str): filter_idx = [filter_idx] settings = params.get("settings") for idx in track.indices: if not filter_idx or idx.name in filter_idx: body = idx.body if body and settings: if "settings" in body: # merge (and potentially override) body["settings"].update(settings) else: body["settings"] = settings elif not body: # this is just needed because we will output this in the middle of the benchmark and will thus write # this on the same line as the progress message. console.println("") console.warn( "Creating index %s based on deprecated type mappings. Please specify an index body instead. " "For details please see the migration guide in the docs." % idx.name, logger=logger) # TODO #366: Deprecate this syntax. We should only specify all mappings in the body property. # check all types and merge their mappings body = {"mappings": {}} if settings: body["settings"] = settings for t in idx.types: body["mappings"].update(t.mapping) self.index_definitions.append((idx.name, body)) else: try: # only 'index' is mandatory, the body is optional (may be ok to create an index without a body) idx = params["index"] body = params.get("body") if isinstance(idx, str): idx = [idx] for i in idx: self.index_definitions.append((i, body)) except KeyError: raise exceptions.InvalidSyntax( "Please set the property 'index' for the create-index operation" )
def __init__(self, indices, params): super().__init__(indices, params) id_conflicts = params.get("conflicts", None) if not id_conflicts: self.id_conflicts = IndexIdConflict.NoConflicts elif id_conflicts == "sequential": self.id_conflicts = IndexIdConflict.SequentialConflicts elif id_conflicts == "random": self.id_conflicts = IndexIdConflict.RandomConflicts else: raise exceptions.InvalidSyntax( "Unknown index id conflict type [%s]." % id_conflicts) self.pipeline = params.get("pipeline", None) try: self.bulk_size = int(params["bulk-size"]) if self.bulk_size <= 0: raise exceptions.InvalidSyntax( "'bulk-size' must be positive but was %d" % self.bulk_size) except KeyError: raise exceptions.InvalidSyntax( "Mandatory parameter 'bulk-size' is missing") except ValueError: raise exceptions.InvalidSyntax("'bulk-size' must be numeric")
def __init__(self, track, params, **kwargs): super().__init__(track, params, **kwargs) self.request_params = params.get("request-params", {}) self.only_if_exists = params.get("only-if-exists", True) self.index_definitions = [] target_index = params.get("index") if target_index: # TODO: Should we allow to delete multiple indices at once? self.index_definitions.append(target_index) elif track.indices: for idx in track.indices: self.index_definitions.append(idx.name) else: raise exceptions.InvalidSyntax( "delete-index operation targets no index")
def __init__(self, track, params, **kwargs): super().__init__(track, params, **kwargs) self.request_params = params.get("request-params", {}) self.only_if_exists = params.get("only-if-exists", True) self.index_definitions = [] target_index = params.get("index") if target_index: if isinstance(target_index, str): target_index = [target_index] for idx in target_index: self.index_definitions.append(idx) elif track.indices: for idx in track.indices: self.index_definitions.append(idx.name) else: raise exceptions.InvalidSyntax("delete-index operation targets no index")
def components(version): """ Determines components of a version string. :param version: A version string in the format major.minor.path-suffix (suffix is optional) :return: A dict with the keys "major", "minor", "patch" and optionally "suffix". """ matches = VERSIONS.match(version) if matches: version_components = { "major": matches.group(1), "minor": matches.group(2), "patch": matches.group(3) } if matches.start(4) > 0: version_components["suffix"] = matches.group(4) return version_components else: raise exceptions.InvalidSyntax( "version string '%s' does not conform to pattern '%s'" % (version, VERSIONS.pattern))
def components(version, strict=True): """ Determines components of a version string. :param version: A version string in the format major.minor.path-suffix (suffix is optional) :param strict: Determines whether versions need to have at least "major", "minor" and "patch" defined. Default: True :return: A tuple with four components determining "major", "minor", "patch" and "suffix" (any part except "major" may be `None`) """ versions_pattern = _versions_pattern(strict) matches = versions_pattern.match(version) if matches: if matches.start(4) > 0: return int(matches.group(1)), int(matches.group(2)), int(matches.group(3)), matches.group(4) elif matches.start(3) > 0: return int(matches.group(1)), int(matches.group(2)), int(matches.group(3)), None elif matches.start(2) > 0: return int(matches.group(1)), int(matches.group(2)), None, None elif matches.start(1) > 0: return int(matches.group(1)), None, None, None else: return int(version), None, None, None raise exceptions.InvalidSyntax("version string '%s' does not conform to pattern '%s'" % (version, versions_pattern.pattern))
def __init__(self, track, params, **kwargs): super().__init__(track, params, **kwargs) self.request_params = params.get("request-params", {}) self.template_definitions = [] if track.templates: filter_template = params.get("template") settings = params.get("settings") for template in track.templates: if not filter_template or template.name == filter_template: body = template.content if body and settings: if "settings" in body: # merge (and potentially override) body["settings"].update(settings) else: body["settings"] = settings self.template_definitions.append((template.name, body)) else: try: self.template_definitions.append((params["template"], params["body"])) except KeyError: raise exceptions.InvalidSyntax("Please set the properties 'template' and 'body' for the create-index-template operation")
def __init__(self, indices, params): super().__init__(indices, params) id_conflicts = params.get("conflicts", None) if not id_conflicts: self.id_conflicts = IndexIdConflict.NoConflicts elif id_conflicts == "sequential": self.id_conflicts = IndexIdConflict.SequentialConflicts elif id_conflicts == "random": self.id_conflicts = IndexIdConflict.RandomConflicts else: raise exceptions.InvalidSyntax("Unknown 'conflicts' setting [%s]" % id_conflicts) action_metadata = params.get("action-and-meta-data", "generate") if action_metadata == "generate": self.action_metadata = ActionMetaData.Generate elif action_metadata == "none": self.action_metadata = ActionMetaData.NoMetaData elif action_metadata == "sourcefile": self.action_metadata = ActionMetaData.SourceFile else: raise exceptions.InvalidSyntax( "Unknown 'action-and-meta-data' setting [%s]" % action_metadata) if self.action_metadata != ActionMetaData.Generate and self.id_conflicts != IndexIdConflict.NoConflicts: raise exceptions.InvalidSyntax( "Cannot generate id conflicts [%s] when 'action-and-meta-data' is [%s]." % (id_conflicts, action_metadata)) self.pipeline = params.get("pipeline", None) try: self.bulk_size = int(params["bulk-size"]) if self.bulk_size <= 0: raise exceptions.InvalidSyntax( "'bulk-size' must be positive but was %d" % self.bulk_size) except KeyError: raise exceptions.InvalidSyntax( "Mandatory parameter 'bulk-size' is missing") except ValueError: raise exceptions.InvalidSyntax("'bulk-size' must be numeric") try: self.batch_size = int(params.get("batch-size", self.bulk_size)) if self.batch_size <= 0: raise exceptions.InvalidSyntax( "'batch-size' must be positive but was %d" % self.batch_size) if self.batch_size < self.bulk_size: raise exceptions.InvalidSyntax( "'batch-size' must be greater than or equal to 'bulk-size'" ) if self.batch_size % self.bulk_size != 0: raise exceptions.InvalidSyntax( "'batch-size' must be a multiple of 'bulk-size'") except ValueError: raise exceptions.InvalidSyntax("'batch-size' must be numeric") if len(indices) == 1 and len(indices[0].types) == 1: default_index = indices[0].name else: default_index = None self.index_name = params.get("index", default_index)
def __init__(self, track, params, **kwargs): super().__init__(track, params, **kwargs) id_conflicts = params.get("conflicts", None) if not id_conflicts: self.id_conflicts = IndexIdConflict.NoConflicts elif id_conflicts == "sequential": self.id_conflicts = IndexIdConflict.SequentialConflicts elif id_conflicts == "random": self.id_conflicts = IndexIdConflict.RandomConflicts else: raise exceptions.InvalidSyntax("Unknown 'conflicts' setting [%s]" % id_conflicts) if self.id_conflicts != IndexIdConflict.NoConflicts: self.conflict_probability = self.float_param( params, name="conflict-probability", default_value=25, min_value=0, max_value=100, min_operator=operator.lt) self.on_conflict = params.get("on-conflict", "index") if self.on_conflict not in ["index", "update"]: raise exceptions.InvalidSyntax( "Unknown 'on-conflict' setting [{}]".format( self.on_conflict)) self.recency = self.float_param(params, name="recency", default_value=0, min_value=0, max_value=1, min_operator=operator.lt) else: self.conflict_probability = None self.on_conflict = None self.recency = None self.corpora = self.used_corpora(track, params) for corpus in self.corpora: for document_set in corpus.documents: if document_set.includes_action_and_meta_data and self.id_conflicts != IndexIdConflict.NoConflicts: file_name = document_set.document_archive if document_set.has_compressed_corpus( ) else document_set.document_file raise exceptions.InvalidSyntax( "Cannot generate id conflicts [%s] as [%s] in document corpus [%s] already contains an " "action and meta-data line." % (id_conflicts, file_name, corpus)) self.pipeline = params.get("pipeline", None) try: self.bulk_size = int(params["bulk-size"]) if self.bulk_size <= 0: raise exceptions.InvalidSyntax( "'bulk-size' must be positive but was %d" % self.bulk_size) except KeyError: raise exceptions.InvalidSyntax( "Mandatory parameter 'bulk-size' is missing") except ValueError: raise exceptions.InvalidSyntax("'bulk-size' must be numeric") try: self.batch_size = int(params.get("batch-size", self.bulk_size)) if self.batch_size <= 0: raise exceptions.InvalidSyntax( "'batch-size' must be positive but was %d" % self.batch_size) if self.batch_size < self.bulk_size: raise exceptions.InvalidSyntax( "'batch-size' must be greater than or equal to 'bulk-size'" ) if self.batch_size % self.bulk_size != 0: raise exceptions.InvalidSyntax( "'batch-size' must be a multiple of 'bulk-size'") except ValueError: raise exceptions.InvalidSyntax("'batch-size' must be numeric") self.ingest_percentage = self.float_param(params, name="ingest-percentage", default_value=100, min_value=0, max_value=100)