def build_operating_system_filter(field_name, operating_system, field_filter): # field name is unused but here because the generic filter builders need it and this has # to have the same interface os_filters = [] for name in operating_system: if isinstance(operating_system[name], dict) and operating_system[name].get("version"): os_filters_for_current_name = [] version_dict = operating_system[name]["version"] # Check that there is an operation at all. No default it wouldn't make sense for operation in version_dict: if operation in lookup_operations("range"): os_value = version_dict[operation] if isinstance(os_value, list): # Make a list os_filter_for_name = _build_operating_system_version_filter_list( os_value, name, operation) else: os_filter_for_name = _build_filter_from_version_string( os_value, name, operation) os_filters_for_current_name.append(os_filter_for_name) else: raise ValidationException( f"Specified operation '{operation}' is not on [operating_system][version] field" ) os_filters.append({"AND": os_filters_for_current_name}) else: raise ValidationException( f"Incomplete path provided: {operating_system} ") return ({"OR": os_filters}, )
def _get_identity(host, metadata): # rhsm reporter does not provide identity. Set identity type to system for access the host in future. if metadata and "b64_identity" in metadata: identity = _decode_id(metadata["b64_identity"]) else: if host.get("reporter") == "rhsm-conduit" and host.get( "subscription_manager_id"): identity = deepcopy(SYSTEM_IDENTITY) identity["account_number"] = host.get("account") identity["system"]["cn"] = _formatted_uuid( host.get("subscription_manager_id")) elif metadata: raise ValueError( "When identity is not provided, reporter MUST be rhsm-conduit with a subscription_manager_id.\n" f"Host Data: {host}") else: raise ValidationException("platform_metadata is mandatory") if host.get("account") != identity["account_number"]: raise ValidationException( "The account number in identity does not match the number in the host." ) identity = Identity(identity) return identity
def _make_deep_object(k, v): """consumes keys, value pairs like (a[foo][bar], "baz") returns (a, {"foo": {"bar": "baz"}}}, is_deep_object) """ root_key = k.split("[", 1)[0] if k == root_key: return (k, v, False) key_path = re.findall(r"\[([^\[\]]*)\]", k) root = prev = node = {} if root_key == "fields": return custom_fields_parser(root_key, key_path, v) prev_k = root_key for k in key_path: if k != "": # skip [] to avoid creating dict with empty string as key node[k] = {} prev = node node = node[k] prev_k = k # if the final value of k is '' it corresponds to a path ending with [] # this indicates an array parameter. # in this case we want to add all the values, not just the 0th if k == "": prev[prev_k] = v else: if len(v) > 1: raise ValidationException( f"Param {root_key} must be appended with [] to accept multiple values." ) prev[k] = v[0] return (root_key, [root], True)
def _object_filter_builder(input_object, spec): object_filter = {} if not isinstance(input_object, dict): raise ValidationException("Invalid filter value") for name in input_object: _check_field_in_spec(spec["children"], name) child_spec = spec["children"][name] child_filter = child_spec["filter"] child_format = child_spec["format"] child_is_array = child_spec["is_array"] if child_filter == "object": object_filter[name] = _object_filter_builder(input_object[name], spec=child_spec) else: field_value, operation = _get_field_value_and_operation( input_object[name], child_filter, child_format, child_is_array ) object_filter.update( _generic_filter_builder( _get_builder_function(child_filter, child_format), name, field_value, child_filter, operation, child_spec, )[0] ) return object_filter
def _base_filter_builder(builder_function, field_name, field_value, field_filter, operation, spec=None): xjoin_field_name = field_name if spec else f"spf_{field_name}" base_value = _get_object_base_value(field_value, field_filter) if isinstance(base_value, list): logger.debug("filter value is a list") foo_list = [] for value in base_value: isolated_expression = _isolate_object_filter_expression( field_value, value) foo_list.append( builder_function(xjoin_field_name, isolated_expression, field_filter, operation, spec)[0]) list_operator = _get_list_operator(field_name, field_filter) field_filter = ({list_operator: foo_list}, ) elif isinstance(base_value, str): logger.debug("filter value is a string") isolated_expression = _isolate_object_filter_expression( field_value, base_value) field_filter = builder_function(xjoin_field_name, isolated_expression, field_filter, operation, spec) else: logger.debug("filter value is bad") raise ValidationException(f"wrong type for {field_value} filter") return field_filter
def _base_object_filter_builder(builder_function, field_name, field_value, field_filter, spec=None): if not isinstance(field_value, dict): raise ValidationException(f"value '{field_value}'' not valid for field '{field_name}'") filter_list = [] for key, val in field_value.items(): filter_list += _base_filter_builder(builder_function, field_name, {key: val}, field_filter, spec) return ({"AND": filter_list},)
def get_sap_system(tags=None, page=None, per_page=None, staleness=None, registered_with=None, filter=None): if not xjoin_enabled(): logger.error("xjoin-search not enabled") flask.abort(503) limit, offset = pagination_params(page, per_page) variables = { "hostFilter": { # we're not indexing null timestamps in ES "OR": list(staleness_filter(staleness)) }, "limit": limit, "offset": offset, } hostfilter_and_variables = () if tags: hostfilter_and_variables = build_tag_query_dict_tuple(tags) if registered_with: hostfilter_and_variables += build_registered_with_filter( registered_with) if filter: for key in filter: if key == "system_profile": hostfilter_and_variables += build_system_profile_filter( filter["system_profile"]) else: raise ValidationException("filter key is invalid") current_identity = get_current_identity() if current_identity.identity_type == IdentityType.SYSTEM: hostfilter_and_variables += owner_id_filter() if hostfilter_and_variables != (): variables["hostFilter"]["AND"] = hostfilter_and_variables response = graphql_query(SAP_SYSTEM_QUERY, variables, log_get_sap_system_failed) data = response["hostSystemProfile"] check_pagination(offset, data["sap_system"]["meta"]["total"]) log_get_sap_system_succeeded(logger, data) return flask_json_response( build_collection_response(data["sap_system"]["data"], page, per_page, data["sap_system"]["meta"]["total"]))
def get_sap_sids(search=None, tags=None, page=None, per_page=None, staleness=None, registered_with=None, filter=None): if not xjoin_enabled(): logger.error("xjoin-search not enabled") flask.abort(503) limit, offset = pagination_params(page, per_page) variables = { "hostFilter": { # we're not indexing null timestamps in ES "OR": list(staleness_filter(staleness)) }, "limit": limit, "offset": offset, } hostfilter_and_variables = () if tags: hostfilter_and_variables = build_tag_query_dict_tuple(tags) if registered_with: variables["hostFilter"]["NOT"] = {"insights_id": {"eq": None}} if search: variables["filter"] = { # Escaped so that the string literals are not interpreted as regex "search": {"regex": f".*{custom_escape(search)}.*"} } if filter: for key in filter: if key == "system_profile": hostfilter_and_variables += build_system_profile_filter(filter["system_profile"]) else: raise ValidationException("filter key is invalid") current_identity = get_current_identity() if current_identity.identity_type == IdentityType.SYSTEM and current_identity.auth_type != AuthType.CLASSIC: hostfilter_and_variables += owner_id_filter() if hostfilter_and_variables != (): variables["hostFilter"]["AND"] = hostfilter_and_variables response = graphql_query(SAP_SIDS_QUERY, variables, log_get_sap_sids_failed) data = response["hostSystemProfile"] check_pagination(offset, data["sap_sids"]["meta"]["total"]) log_get_sap_sids_succeeded(logger, data) return flask_json_response( build_collection_response(data["sap_sids"]["data"], page, per_page, data["sap_sids"]["meta"]["total"]) )
def get_operating_system( tags=None, page: Optional[int] = None, per_page: Optional[int] = None, staleness: Optional[str] = None, registered_with: Optional[str] = None, filter=None, ): limit, offset = pagination_params(page, per_page) variables = { "hostFilter": { # we're not indexing null timestamps in ES "OR": list(staleness_filter(staleness)) }, "limit": limit, "offset": offset, } hostfilter_and_variables = () if tags: hostfilter_and_variables = build_tag_query_dict_tuple(tags) if registered_with: variables["hostFilter"]["NOT"] = {"insights_id": {"eq": None}} if filter: for key in filter: if key == "system_profile": hostfilter_and_variables += build_system_profile_filter( filter["system_profile"]) else: raise ValidationException("filter key is invalid") current_identity = get_current_identity() if current_identity.identity_type == IdentityType.SYSTEM: hostfilter_and_variables += owner_id_filter() if hostfilter_and_variables != (): variables["hostFilter"]["AND"] = hostfilter_and_variables response = graphql_query(OPERATING_SYSTEM_QUERY, variables, log_get_operating_system_failed) data = response["hostSystemProfile"] check_pagination(offset, data["operating_system"]["meta"]["total"]) log_get_operating_system_succeeded(logger, data) return flask_json_response( build_collection_response(data["operating_system"]["data"], page, per_page, data["operating_system"]["meta"]["total"]))
def _build_filter_from_version_string(os_value, name, operation): os_value_split = os_value.split(".") major_version = int(os_value_split[0]) minor_version = int(os_value_split[1]) if len(os_value_split) > 1 else 0 if len(os_value_split) > 2: raise ValidationException( "operating_system filter can only have a major and minor version.") return _build_operating_system_version_filter(major_version, minor_version, name, operation)
def deserialize_host(raw_data, schema=HostSchema, system_profile_spec=None): try: validated_data = schema( system_profile_schema=system_profile_spec).load(raw_data) except ValidationError as e: raise ValidationException(str(e.messages)) from None canonical_facts = _deserialize_canonical_facts(validated_data) facts = _deserialize_facts(validated_data.get("facts")) tags = _deserialize_tags(validated_data.get("tags")) return schema.build_model(validated_data, canonical_facts, facts, tags)
def _get_field_value_and_operation(field_value, field_filter, field_format, is_array): # Selecting 0th element from lookup_operations because it is always eq or equivalent operation = None if field_filter == "object" else lookup_operations(field_filter, field_format, is_array)[0] if isinstance(field_value, dict) and field_filter != "object": for key in field_value: # check if the operation is valid for the field. if key not in lookup_operations(field_filter, field_format, is_array): raise ValidationException(f"invalid operation for {field_filter}") operation = key field_value = field_value[key] return (field_value, operation)
def _create_object_existence_query(field_name, field_value): # TODO: this will break with more than one level of object nesting # The plan is to make the interface in xjoin simpler to offload the complexity # before something gets added to the system profile that will cause an issue if not isinstance(field_value, str) or field_value not in (NIL_STRING, NOT_NIL_STRING): raise ValidationException( f"value '{field_value}'' not valid for field '{field_name}'") nil_query = _create_object_nil_query(field_name) return {"NOT": nil_query} if field_value == NOT_NIL_STRING else nil_query
def from_string(self, string_tag): namespace, key, value = self._split_string_tag(string_tag) if namespace is not None: namespace = urllib.parse.unquote(namespace) if len(namespace) > 255: raise ValidationException( "namespace is longer than 255 characters") self.namespace = namespace key = urllib.parse.unquote(key) if len(key) > 255: raise ValidationException("key is longer than 255 characters") self.key = key if value is not None: value = urllib.parse.unquote(value) if len(value) > 255: raise ValidationException( "value is longer than 255 characters") self.value = value return self
def from_string(string_tag): match = re.match(r"((?P<namespace>[^=/]+)/)?(?P<key>[^=/]+)(=(?P<value>[^=/]+))?$", string_tag) encoded_tag_data = match.groupdict() decoded_tag_data = {} for k, v in encoded_tag_data.items(): if v is None: decoded_tag_data[k] = None else: decoded_tag_data[k] = urllib.parse.unquote(v) if len(decoded_tag_data[k]) > 255: raise ValidationException(f"{k} is longer than 255 characters") return Tag(**decoded_tag_data)
def _set_owner(host, identity): cn = identity.system.get("cn") if "system_profile" not in host: host["system_profile"] = {} host["system_profile"]["owner_id"] = cn elif not host["system_profile"].get("owner_id"): host["system_profile"]["owner_id"] = cn else: if host.get("reporter") == "rhsm-conduit" and host.get( "subscription_manager_id"): host["system_profile"]["owner_id"] = _formatted_uuid( host.get("subscription_manager_id")) else: if host["system_profile"]["owner_id"] != cn: raise ValidationException( "The owner in host does not match the owner in identity") return host
def deserialize_host(raw_data): try: validated_data = HostSchema(strict=True).load(raw_data).data except ValidationError as e: raise ValidationException(str(e.messages)) from None canonical_facts = _deserialize_canonical_facts(validated_data) facts = _deserialize_facts(validated_data.get("facts")) tags = _deserialize_tags(validated_data.get("tags")) return Host( canonical_facts, validated_data.get("display_name"), validated_data.get("ansible_host"), validated_data.get("account"), facts, tags, validated_data.get("system_profile", {}), validated_data.get("stale_timestamp"), validated_data.get("reporter"), )
def deserialize_canonical_facts(raw_data): try: validated_data = CanonicalFactsSchema().load(raw_data) except ValidationError as e: raise ValidationException(str(e.messages)) from None return _deserialize_canonical_facts(validated_data)
def _check_field_in_spec(spec, field_name): if field_name not in spec.keys(): raise ValidationException(f"invalid filter field: {field_name}")
def _invalid_value_error(field_name, field_value): raise ValidationException(f"{field_value} is an invalid value for field {field_name}")
def query_filters( fqdn, display_name, hostname_or_id, insights_id, provider_id, provider_type, tags, staleness, registered_with, filter, ): num_ids = 0 for id_param in [fqdn, display_name, hostname_or_id, insights_id]: if id_param: num_ids += 1 if num_ids > 1: raise ValidationException( "Only one of [fqdn, display_name, hostname_or_id, insights_id] may be provided at a time." ) if fqdn: query_filters = ({"fqdn": {"eq": fqdn.casefold()}},) elif display_name: query_filters = ({"display_name": string_contains_lc(display_name)},) elif hostname_or_id: contains_lc = string_contains_lc(hostname_or_id) hostname_or_id_filters = ({"display_name": contains_lc}, {"fqdn": contains_lc}) try: id = UUID(hostname_or_id) except ValueError: # Do not filter using the id logger.debug("The hostname (%s) could not be converted into a UUID", hostname_or_id, exc_info=True) else: logger.debug("Adding id (uuid) to the filter list") hostname_or_id_filters += ({"id": {"eq": str(id)}},) query_filters = ({"OR": hostname_or_id_filters},) elif insights_id: query_filters = ({"insights_id": {"eq": insights_id.casefold()}},) else: query_filters = () if tags: query_filters += build_tag_query_dict_tuple(tags) if staleness: staleness_filters = tuple(staleness_filter(staleness)) query_filters += ({"OR": staleness_filters},) if registered_with: query_filters += ({"NOT": {"insights_id": {"eq": None}}},) if provider_type: query_filters += ({"provider_type": {"eq": provider_type.casefold()}},) if provider_id: query_filters += ({"provider_id": {"eq": provider_id.casefold()}},) for key in filter: if key == "system_profile": query_filters += build_system_profile_filter(filter["system_profile"]) else: raise ValidationException("filter key is invalid") logger.debug(query_filters) return query_filters