def build_operating_system_filter(field_name, operating_system, field_filter):
# field name is unused but here because the generic filter builders need it and this has
# to have the same interface
os_filters = []
for name in operating_system:
if isinstance(operating_system[name],
dict) and operating_system[name].get("version"):
os_filters_for_current_name = []
version_dict = operating_system[name]["version"]
# Check that there is an operation at all. No default it wouldn't make sense
for operation in version_dict:
if operation in lookup_operations("range"):
os_value = version_dict[operation]
if isinstance(os_value, list):
# Make a list
os_filter_for_name = _build_operating_system_version_filter_list(
os_value, name, operation)
else:
os_filter_for_name = _build_filter_from_version_string(
os_value, name, operation)
os_filters_for_current_name.append(os_filter_for_name)
else:
raise ValidationException(
f"Specified operation '{operation}' is not on [operating_system][version] field"
)
os_filters.append({"AND": os_filters_for_current_name})
else:
raise ValidationException(
f"Incomplete path provided: {operating_system} ")
return ({"OR": os_filters}, )
def _get_identity(host, metadata):
# rhsm reporter does not provide identity. Set identity type to system for access the host in future.
if metadata and "b64_identity" in metadata:
identity = _decode_id(metadata["b64_identity"])
else:
if host.get("reporter") == "rhsm-conduit" and host.get(
"subscription_manager_id"):
identity = deepcopy(SYSTEM_IDENTITY)
identity["account_number"] = host.get("account")
identity["system"]["cn"] = _formatted_uuid(
host.get("subscription_manager_id"))
elif metadata:
raise ValueError(
"When identity is not provided, reporter MUST be rhsm-conduit with a subscription_manager_id.\n"
f"Host Data: {host}")
else:
raise ValidationException("platform_metadata is mandatory")
if host.get("account") != identity["account_number"]:
raise ValidationException(
"The account number in identity does not match the number in the host."
)
identity = Identity(identity)
return identity
def _make_deep_object(k, v):
"""consumes keys, value pairs like (a[foo][bar], "baz")
returns (a, {"foo": {"bar": "baz"}}}, is_deep_object)
"""
root_key = k.split("[", 1)[0]
if k == root_key:
return (k, v, False)
key_path = re.findall(r"\[([^\[\]]*)\]", k)
root = prev = node = {}
if root_key == "fields":
return custom_fields_parser(root_key, key_path, v)
prev_k = root_key
for k in key_path:
if k != "": # skip [] to avoid creating dict with empty string as key
node[k] = {}
prev = node
node = node[k]
prev_k = k
# if the final value of k is '' it corresponds to a path ending with []
# this indicates an array parameter.
# in this case we want to add all the values, not just the 0th
if k == "":
prev[prev_k] = v
else:
if len(v) > 1:
raise ValidationException(
f"Param {root_key} must be appended with [] to accept multiple values."
)
prev[k] = v[0]
return (root_key, [root], True)
def _object_filter_builder(input_object, spec):
object_filter = {}
if not isinstance(input_object, dict):
raise ValidationException("Invalid filter value")
for name in input_object:
_check_field_in_spec(spec["children"], name)
child_spec = spec["children"][name]
child_filter = child_spec["filter"]
child_format = child_spec["format"]
child_is_array = child_spec["is_array"]
if child_filter == "object":
object_filter[name] = _object_filter_builder(input_object[name], spec=child_spec)
else:
field_value, operation = _get_field_value_and_operation(
input_object[name], child_filter, child_format, child_is_array
)
object_filter.update(
_generic_filter_builder(
_get_builder_function(child_filter, child_format),
name,
field_value,
child_filter,
operation,
child_spec,
)[0]
)
return object_filter
def _base_filter_builder(builder_function,
field_name,
field_value,
field_filter,
operation,
spec=None):
xjoin_field_name = field_name if spec else f"spf_{field_name}"
base_value = _get_object_base_value(field_value, field_filter)
if isinstance(base_value, list):
logger.debug("filter value is a list")
foo_list = []
for value in base_value:
isolated_expression = _isolate_object_filter_expression(
field_value, value)
foo_list.append(
builder_function(xjoin_field_name, isolated_expression,
field_filter, operation, spec)[0])
list_operator = _get_list_operator(field_name, field_filter)
field_filter = ({list_operator: foo_list}, )
elif isinstance(base_value, str):
logger.debug("filter value is a string")
isolated_expression = _isolate_object_filter_expression(
field_value, base_value)
field_filter = builder_function(xjoin_field_name, isolated_expression,
field_filter, operation, spec)
else:
logger.debug("filter value is bad")
raise ValidationException(f"wrong type for {field_value} filter")
return field_filter
def _base_object_filter_builder(builder_function, field_name, field_value, field_filter, spec=None):
if not isinstance(field_value, dict):
raise ValidationException(f"value '{field_value}'' not valid for field '{field_name}'")
filter_list = []
for key, val in field_value.items():
filter_list += _base_filter_builder(builder_function, field_name, {key: val}, field_filter, spec)
return ({"AND": filter_list},)
def get_sap_system(tags=None,
page=None,
per_page=None,
staleness=None,
registered_with=None,
filter=None):
if not xjoin_enabled():
logger.error("xjoin-search not enabled")
flask.abort(503)
limit, offset = pagination_params(page, per_page)
variables = {
"hostFilter": {
# we're not indexing null timestamps in ES
"OR": list(staleness_filter(staleness))
},
"limit": limit,
"offset": offset,
}
hostfilter_and_variables = ()
if tags:
hostfilter_and_variables = build_tag_query_dict_tuple(tags)
if registered_with:
hostfilter_and_variables += build_registered_with_filter(
registered_with)
if filter:
for key in filter:
if key == "system_profile":
hostfilter_and_variables += build_system_profile_filter(
filter["system_profile"])
else:
raise ValidationException("filter key is invalid")
current_identity = get_current_identity()
if current_identity.identity_type == IdentityType.SYSTEM:
hostfilter_and_variables += owner_id_filter()
if hostfilter_and_variables != ():
variables["hostFilter"]["AND"] = hostfilter_and_variables
response = graphql_query(SAP_SYSTEM_QUERY, variables,
log_get_sap_system_failed)
data = response["hostSystemProfile"]
check_pagination(offset, data["sap_system"]["meta"]["total"])
log_get_sap_system_succeeded(logger, data)
return flask_json_response(
build_collection_response(data["sap_system"]["data"], page, per_page,
data["sap_system"]["meta"]["total"]))
def get_sap_sids(search=None, tags=None, page=None, per_page=None, staleness=None, registered_with=None, filter=None):
if not xjoin_enabled():
logger.error("xjoin-search not enabled")
flask.abort(503)
limit, offset = pagination_params(page, per_page)
variables = {
"hostFilter": {
# we're not indexing null timestamps in ES
"OR": list(staleness_filter(staleness))
},
"limit": limit,
"offset": offset,
}
hostfilter_and_variables = ()
if tags:
hostfilter_and_variables = build_tag_query_dict_tuple(tags)
if registered_with:
variables["hostFilter"]["NOT"] = {"insights_id": {"eq": None}}
if search:
variables["filter"] = {
# Escaped so that the string literals are not interpreted as regex
"search": {"regex": f".*{custom_escape(search)}.*"}
}
if filter:
for key in filter:
if key == "system_profile":
hostfilter_and_variables += build_system_profile_filter(filter["system_profile"])
else:
raise ValidationException("filter key is invalid")
current_identity = get_current_identity()
if current_identity.identity_type == IdentityType.SYSTEM and current_identity.auth_type != AuthType.CLASSIC:
hostfilter_and_variables += owner_id_filter()
if hostfilter_and_variables != ():
variables["hostFilter"]["AND"] = hostfilter_and_variables
response = graphql_query(SAP_SIDS_QUERY, variables, log_get_sap_sids_failed)
data = response["hostSystemProfile"]
check_pagination(offset, data["sap_sids"]["meta"]["total"])
log_get_sap_sids_succeeded(logger, data)
return flask_json_response(
build_collection_response(data["sap_sids"]["data"], page, per_page, data["sap_sids"]["meta"]["total"])
)
def get_operating_system(
tags=None,
page: Optional[int] = None,
per_page: Optional[int] = None,
staleness: Optional[str] = None,
registered_with: Optional[str] = None,
filter=None,
):
limit, offset = pagination_params(page, per_page)
variables = {
"hostFilter": {
# we're not indexing null timestamps in ES
"OR": list(staleness_filter(staleness))
},
"limit": limit,
"offset": offset,
}
hostfilter_and_variables = ()
if tags:
hostfilter_and_variables = build_tag_query_dict_tuple(tags)
if registered_with:
variables["hostFilter"]["NOT"] = {"insights_id": {"eq": None}}
if filter:
for key in filter:
if key == "system_profile":
hostfilter_and_variables += build_system_profile_filter(
filter["system_profile"])
else:
raise ValidationException("filter key is invalid")
current_identity = get_current_identity()
if current_identity.identity_type == IdentityType.SYSTEM:
hostfilter_and_variables += owner_id_filter()
if hostfilter_and_variables != ():
variables["hostFilter"]["AND"] = hostfilter_and_variables
response = graphql_query(OPERATING_SYSTEM_QUERY, variables,
log_get_operating_system_failed)
data = response["hostSystemProfile"]
check_pagination(offset, data["operating_system"]["meta"]["total"])
log_get_operating_system_succeeded(logger, data)
return flask_json_response(
build_collection_response(data["operating_system"]["data"], page,
per_page,
data["operating_system"]["meta"]["total"]))
def _build_filter_from_version_string(os_value, name, operation):
os_value_split = os_value.split(".")
major_version = int(os_value_split[0])
minor_version = int(os_value_split[1]) if len(os_value_split) > 1 else 0
if len(os_value_split) > 2:
raise ValidationException(
"operating_system filter can only have a major and minor version.")
return _build_operating_system_version_filter(major_version, minor_version,
name, operation)
def deserialize_host(raw_data, schema=HostSchema, system_profile_spec=None):
try:
validated_data = schema(
system_profile_schema=system_profile_spec).load(raw_data)
except ValidationError as e:
raise ValidationException(str(e.messages)) from None
canonical_facts = _deserialize_canonical_facts(validated_data)
facts = _deserialize_facts(validated_data.get("facts"))
tags = _deserialize_tags(validated_data.get("tags"))
return schema.build_model(validated_data, canonical_facts, facts, tags)
def _get_field_value_and_operation(field_value, field_filter, field_format, is_array):
# Selecting 0th element from lookup_operations because it is always eq or equivalent
operation = None if field_filter == "object" else lookup_operations(field_filter, field_format, is_array)[0]
if isinstance(field_value, dict) and field_filter != "object":
for key in field_value:
# check if the operation is valid for the field.
if key not in lookup_operations(field_filter, field_format, is_array):
raise ValidationException(f"invalid operation for {field_filter}")
operation = key
field_value = field_value[key]
return (field_value, operation)
def _create_object_existence_query(field_name, field_value):
# TODO: this will break with more than one level of object nesting
# The plan is to make the interface in xjoin simpler to offload the complexity
# before something gets added to the system profile that will cause an issue
if not isinstance(field_value, str) or field_value not in (NIL_STRING,
NOT_NIL_STRING):
raise ValidationException(
f"value '{field_value}'' not valid for field '{field_name}'")
nil_query = _create_object_nil_query(field_name)
return {"NOT": nil_query} if field_value == NOT_NIL_STRING else nil_query
def from_string(self, string_tag):
namespace, key, value = self._split_string_tag(string_tag)
if namespace is not None:
namespace = urllib.parse.unquote(namespace)
if len(namespace) > 255:
raise ValidationException(
"namespace is longer than 255 characters")
self.namespace = namespace
key = urllib.parse.unquote(key)
if len(key) > 255:
raise ValidationException("key is longer than 255 characters")
self.key = key
if value is not None:
value = urllib.parse.unquote(value)
if len(value) > 255:
raise ValidationException(
"value is longer than 255 characters")
self.value = value
return self
def from_string(string_tag):
match = re.match(r"((?P<namespace>[^=/]+)/)?(?P<key>[^=/]+)(=(?P<value>[^=/]+))?$", string_tag)
encoded_tag_data = match.groupdict()
decoded_tag_data = {}
for k, v in encoded_tag_data.items():
if v is None:
decoded_tag_data[k] = None
else:
decoded_tag_data[k] = urllib.parse.unquote(v)
if len(decoded_tag_data[k]) > 255:
raise ValidationException(f"{k} is longer than 255 characters")
return Tag(**decoded_tag_data)
def _set_owner(host, identity):
cn = identity.system.get("cn")
if "system_profile" not in host:
host["system_profile"] = {}
host["system_profile"]["owner_id"] = cn
elif not host["system_profile"].get("owner_id"):
host["system_profile"]["owner_id"] = cn
else:
if host.get("reporter") == "rhsm-conduit" and host.get(
"subscription_manager_id"):
host["system_profile"]["owner_id"] = _formatted_uuid(
host.get("subscription_manager_id"))
else:
if host["system_profile"]["owner_id"] != cn:
raise ValidationException(
"The owner in host does not match the owner in identity")
return host
def deserialize_host(raw_data):
try:
validated_data = HostSchema(strict=True).load(raw_data).data
except ValidationError as e:
raise ValidationException(str(e.messages)) from None
canonical_facts = _deserialize_canonical_facts(validated_data)
facts = _deserialize_facts(validated_data.get("facts"))
tags = _deserialize_tags(validated_data.get("tags"))
return Host(
canonical_facts,
validated_data.get("display_name"),
validated_data.get("ansible_host"),
validated_data.get("account"),
facts,
tags,
validated_data.get("system_profile", {}),
validated_data.get("stale_timestamp"),
validated_data.get("reporter"),
)
def deserialize_canonical_facts(raw_data):
try:
validated_data = CanonicalFactsSchema().load(raw_data)
except ValidationError as e:
raise ValidationException(str(e.messages)) from None
return _deserialize_canonical_facts(validated_data)
def _check_field_in_spec(spec, field_name):
if field_name not in spec.keys():
raise ValidationException(f"invalid filter field: {field_name}")
def _invalid_value_error(field_name, field_value):
raise ValidationException(f"{field_value} is an invalid value for field {field_name}")
def query_filters(
fqdn,
display_name,
hostname_or_id,
insights_id,
provider_id,
provider_type,
tags,
staleness,
registered_with,
filter,
):
num_ids = 0
for id_param in [fqdn, display_name, hostname_or_id, insights_id]:
if id_param:
num_ids += 1
if num_ids > 1:
raise ValidationException(
"Only one of [fqdn, display_name, hostname_or_id, insights_id] may be provided at a time."
)
if fqdn:
query_filters = ({"fqdn": {"eq": fqdn.casefold()}},)
elif display_name:
query_filters = ({"display_name": string_contains_lc(display_name)},)
elif hostname_or_id:
contains_lc = string_contains_lc(hostname_or_id)
hostname_or_id_filters = ({"display_name": contains_lc}, {"fqdn": contains_lc})
try:
id = UUID(hostname_or_id)
except ValueError:
# Do not filter using the id
logger.debug("The hostname (%s) could not be converted into a UUID", hostname_or_id, exc_info=True)
else:
logger.debug("Adding id (uuid) to the filter list")
hostname_or_id_filters += ({"id": {"eq": str(id)}},)
query_filters = ({"OR": hostname_or_id_filters},)
elif insights_id:
query_filters = ({"insights_id": {"eq": insights_id.casefold()}},)
else:
query_filters = ()
if tags:
query_filters += build_tag_query_dict_tuple(tags)
if staleness:
staleness_filters = tuple(staleness_filter(staleness))
query_filters += ({"OR": staleness_filters},)
if registered_with:
query_filters += ({"NOT": {"insights_id": {"eq": None}}},)
if provider_type:
query_filters += ({"provider_type": {"eq": provider_type.casefold()}},)
if provider_id:
query_filters += ({"provider_id": {"eq": provider_id.casefold()}},)
for key in filter:
if key == "system_profile":
query_filters += build_system_profile_filter(filter["system_profile"])
else:
raise ValidationException("filter key is invalid")
logger.debug(query_filters)
return query_filters
