def get_rbac_permissions():
request_header = {
IDENTITY_HEADER:
request.headers[IDENTITY_HEADER],
REQUEST_ID_HEADER:
request.headers.get(REQUEST_ID_HEADER, UNKNOWN_REQUEST_ID_VALUE),
}
request_session = Session()
retry_config = Retry(total=inventory_config().rbac_retries,
backoff_factor=1,
status_forcelist=RETRY_STATUSES)
request_session.mount(rbac_url(), HTTPAdapter(max_retries=retry_config))
try:
with outbound_http_metric.time():
rbac_response = request_session.get(
url=rbac_url(),
headers=request_header,
timeout=inventory_config().rbac_timeout)
except Exception as e:
rbac_failure(logger, e)
abort(503,
"Failed to reach RBAC endpoint, request cannot be fulfilled")
finally:
request_session.close()
resp_data = rbac_response.json()
logger.debug("Fetched RBAC Data", extra=resp_data)
return resp_data["data"]
def get_bulk_query_source():
if XJOIN_HEADER in connexion.request.headers:
if connexion.request.headers[XJOIN_HEADER].lower() == "xjoin":
return BulkQuerySource.xjoin
elif connexion.request.headers[XJOIN_HEADER].lower() == "db":
return BulkQuerySource.db
if REFERAL_HEADER in connexion.request.headers:
if "/beta" in connexion.request.headers[REFERAL_HEADER]:
return inventory_config().bulk_query_source_beta
return inventory_config().bulk_query_source
def _delete_filtered_hosts(host_id_list):
current_identity = get_current_identity()
payload_tracker = get_payload_tracker(
account=current_identity.account_number,
request_id=threadctx.request_id)
with PayloadTrackerContext(payload_tracker,
received_status_message="delete operation",
current_operation="delete"):
query = _get_host_list_by_id_list(host_id_list)
if not query.count():
flask.abort(status.HTTP_404_NOT_FOUND)
deletion_count = 0
for host_id, deleted in delete_hosts(
query, current_app.event_producer,
inventory_config().host_delete_chunk_size):
if deleted:
log_host_delete_succeeded(logger, host_id, get_control_rule())
tracker_message = "deleted host"
deletion_count += 1
else:
log_host_delete_failed(logger, host_id, get_control_rule())
tracker_message = "not deleted host"
with PayloadTrackerProcessingContext(
payload_tracker, processing_status_message=tracker_message
) as payload_tracker_processing_ctx:
payload_tracker_processing_ctx.inventory_id = host_id
return deletion_count
def modified_func(*args, **kwargs):
if not inventory_config().rbac_enforced:
return func(*args, **kwargs)
if get_current_identity().identity_type != CHECKED_TYPE:
return func(*args, **kwargs)
# track that RBAC is being used to control access
g.access_control_rule = "RBAC"
logger.debug("access_control_rule set")
rbac_data = get_rbac_permissions()
permission_type = required_permission.value.split(":")[2]
for rbac_permission in rbac_data:
if (rbac_permission["permission"]
== Permission.ADMIN.value # inventory:*:*
or rbac_permission["permission"]
== Permission.HOSTS_ALL.value # inventory:hosts:*
or rbac_permission["permission"] ==
f"inventory:*:{permission_type}" # inventory:*:(read | write)
or rbac_permission["permission"] == required_permission
.value # inventory:hosts:(read | write)
):
return func(*args, **kwargs)
rbac_permission_denied(logger, required_permission.value,
rbac_data)
abort(status.HTTP_403_FORBIDDEN)
def handle_message(message, event_producer):
validated_operation_msg = parse_operation_message(message)
platform_metadata = validated_operation_msg.get("platform_metadata") or {}
# create a dummy identity for working around the identity requirement for CRUD operations
identity = Identity(USER_IDENTITY)
# set account_number in dummy idenity to the actual account_number received in the payload
identity.account_number = validated_operation_msg["data"]["account"]
request_id = platform_metadata.get("request_id", "-1")
initialize_thread_local_storage(request_id)
payload_tracker = get_payload_tracker(request_id=request_id)
with PayloadTrackerContext(payload_tracker,
received_status_message="message received",
current_operation="handle_message"):
output_host, host_id, insights_id, add_results = add_host(
validated_operation_msg["data"], identity)
event_type = add_host_results_to_event_type(add_results)
event = build_event(event_type,
output_host,
platform_metadata=platform_metadata)
headers = message_headers(add_results, insights_id)
event_producer.write_event(event, str(host_id), headers, Topic.egress)
# for transition to platform.inventory.events
if inventory_config().secondary_topic_enabled:
event_producer.write_event(event, str(host_id), headers,
Topic.events)
def update_system_profile(host_data, platform_metadata):
payload_tracker = get_payload_tracker(request_id=threadctx.request_id)
with PayloadTrackerProcessingContext(
payload_tracker,
processing_status_message="updating host system profile",
current_operation="updating host system profile",
) as payload_tracker_processing_ctx:
try:
input_host = deserialize_host(host_data, schema=LimitedHostSchema)
input_host.id = host_data.get("id")
staleness_timestamps = Timestamps.from_config(inventory_config())
identity = create_mock_identity_with_account(input_host.account)
output_host, host_id, insights_id, update_result = host_repository.update_system_profile(
input_host, identity, staleness_timestamps, EGRESS_HOST_FIELDS)
log_update_system_profile_success(logger, output_host)
payload_tracker_processing_ctx.inventory_id = output_host["id"]
return output_host, host_id, insights_id, update_result
except ValidationException:
metrics.update_system_profile_failure.labels(
"ValidationException").inc()
raise
except InventoryException:
log_update_system_profile_failure(logger, host_data)
raise
except OperationalError as oe:
log_db_access_failure(logger, f"Could not access DB {str(oe)}",
host_data)
raise oe
except Exception:
logger.exception("Error while updating host system profile",
extra={"host": host_data})
metrics.update_system_profile_failure.labels("Exception").inc()
raise
def delete_by_id(host_id_list):
payload_tracker = get_payload_tracker(account=current_identity.account_number, request_id=threadctx.request_id)
with PayloadTrackerContext(
payload_tracker, received_status_message="delete operation", current_operation="delete"
):
query = _get_host_list_by_id_list(current_identity.account_number, host_id_list)
if not query.count():
flask.abort(status.HTTP_404_NOT_FOUND)
for host_id, deleted in delete_hosts(
query, current_app.event_producer, inventory_config().host_delete_chunk_size
):
if deleted:
logger.info("Deleted host: %s", host_id)
tracker_message = "deleted host"
else:
logger.info("Host %s already deleted. Delete event not emitted.", host_id)
tracker_message = "not deleted host"
with PayloadTrackerProcessingContext(
payload_tracker, processing_status_message=tracker_message
) as payload_tracker_processing_ctx:
payload_tracker_processing_ctx.inventory_id = host_id
return flask.Response(None, status.HTTP_200_OK)
def find_hosts_by_staleness(staleness, query):
logger.debug("find_hosts_by_staleness(%s)", staleness)
config = inventory_config()
staleness_conditions = tuple(staleness_to_conditions(config, staleness, stale_timestamp_filter))
if "unknown" in staleness:
staleness_conditions += (Host.stale_timestamp == NULL,)
return query.filter(or_(*staleness_conditions))
def add_host(host_data):
payload_tracker = get_payload_tracker(request_id=threadctx.request_id)
with PayloadTrackerProcessingContext(
payload_tracker, processing_status_message="adding/updating host"
) as payload_tracker_processing_ctx:
try:
input_host = deserialize_host(host_data)
staleness_timestamps = Timestamps.from_config(inventory_config())
logger.info(
"Attempting to add host",
extra={
"input_host": {
"account": input_host.account,
"display_name": input_host.display_name,
"canonical_facts": input_host.canonical_facts,
"reporter": input_host.reporter,
"stale_timestamp":
input_host.stale_timestamp.isoformat(),
"tags": input_host.tags,
}
},
)
(output_host,
add_results) = host_repository.add_host(input_host,
staleness_timestamps,
fields=EGRESS_HOST_FIELDS)
metrics.add_host_success.labels(
add_results.name,
host_data.get("reporter", "null")).inc() # created vs updated
# log all the incoming host data except facts and system_profile b/c they can be quite large
logger.info(
"Host %s",
add_results.name,
extra={
"host": {
i: output_host[i]
for i in output_host
if i not in ("facts", "system_profile")
}
},
)
payload_tracker_processing_ctx.inventory_id = output_host["id"]
return (output_host, add_results)
except InventoryException:
logger.exception("Error adding host ", extra={"host": host_data})
metrics.add_host_failure.labels("InventoryException",
host_data.get("reporter",
"null")).inc()
raise
except Exception:
logger.exception("Error while adding host",
extra={"host": host_data})
metrics.add_host_failure.labels("Exception",
host_data.get("reporter",
"null")).inc()
raise
def add_host(host_data, platform_metadata):
payload_tracker = get_payload_tracker(request_id=threadctx.request_id)
with PayloadTrackerProcessingContext(
payload_tracker,
processing_status_message="adding/updating host",
current_operation="adding/updating host"
) as payload_tracker_processing_ctx:
try:
identity = _get_identity(host_data, platform_metadata)
# basic-auth does not need owner_id
if identity.identity_type == IdentityType.SYSTEM:
host_data = _set_owner(host_data, identity)
input_host = deserialize_host(host_data)
staleness_timestamps = Timestamps.from_config(inventory_config())
log_add_host_attempt(logger, input_host)
output_host, host_id, insights_id, add_result = host_repository.add_host(
input_host,
identity,
staleness_timestamps,
fields=EGRESS_HOST_FIELDS)
log_add_update_host_succeeded(logger, add_result, host_data,
output_host)
payload_tracker_processing_ctx.inventory_id = output_host["id"]
return output_host, host_id, insights_id, add_result
except ValidationException:
metrics.add_host_failure.labels("ValidationException",
host_data.get("reporter",
"null")).inc()
raise
except InventoryException as ie:
log_add_host_failure(logger, str(ie.detail), host_data)
raise
except OperationalError as oe:
log_db_access_failure(logger, f"Could not access DB {str(oe)}",
host_data)
raise oe
except Exception:
logger.exception("Error while adding host",
extra={"host": host_data})
metrics.add_host_failure.labels("Exception",
host_data.get("reporter",
"null")).inc()
raise
def modified_func(*args, **kwargs):
if not inventory_config().rbac_enforced:
return func(*args, **kwargs)
if current_identity.identity_type != CHECKED_TYPE:
return func(*args, **kwargs)
rbac_data = get_rbac_permissions()
for rbac_permission in rbac_data:
if (
rbac_permission["permission"] == Permission.ADMIN.value
or rbac_permission["permission"] == Permission.HOSTS_ALL.value
or rbac_permission["permission"] == required_permission.value
):
return func(*args, **kwargs)
rbac_permission_denied(logger, required_permission.value, rbac_data)
abort(status.HTTP_403_FORBIDDEN)
def handle_message(message, event_producer):
validated_operation_msg = parse_operation_message(message)
platform_metadata = validated_operation_msg.get("platform_metadata") or {}
request_id = platform_metadata.get("request_id", "-1")
initialize_thread_local_storage(request_id)
payload_tracker = get_payload_tracker(request_id=request_id)
with PayloadTrackerContext(
payload_tracker, received_status_message="message received", current_operation="handle_message"
):
(output_host, add_results) = add_host(validated_operation_msg["data"])
event_type = add_host_results_to_event_type(add_results)
event = build_event(event_type, output_host, platform_metadata=platform_metadata)
event_producer.write_event(event, output_host["id"], message_headers(add_results), Topic.egress)
# for transition to platform.inventory.events
if inventory_config().secondary_topic_enabled:
event_producer.write_event(event, output_host["id"], message_headers(add_results), Topic.events)
def modified_func(*args, **kwargs):
if not inventory_config().rbac_enforced:
return func(*args, **kwargs)
if current_identity.identity_type != CHECKED_TYPE:
return func(*args, **kwargs)
rbac_data = get_rbac_permissions()
permission_type = required_permission.value.split(":")[2]
for rbac_permission in rbac_data:
if (
rbac_permission["permission"] == Permission.ADMIN.value # inventory:*:*
or rbac_permission["permission"] == Permission.HOSTS_ALL.value # inventory:hosts:*
or rbac_permission["permission"] == f"inventory:*:{permission_type}" # inventory:*:(read | write)
or rbac_permission["permission"] == required_permission.value # inventory:hosts:(read | write)
):
return func(*args, **kwargs)
rbac_permission_denied(logger, required_permission.value, rbac_data)
abort(status.HTTP_403_FORBIDDEN)
def build_registered_with_filter(registered_with):
reg_with_copy = deepcopy(registered_with)
prs_list = []
if "insights" in reg_with_copy:
prs_list.append({"NOT": {"insights_id": {"eq": None}}})
reg_with_copy.remove("insights")
if reg_with_copy:
for item in reg_with_copy:
prs_list.append({
"per_reporter_staleness": {
"reporter": {
"eq": item
},
"stale_timestamp": {
"gt":
str((datetime.now(timezone.utc) -
inventory_config().culling_culled_offset_delta
).isoformat())
},
},
})
return ({"OR": prs_list}, )
def add_host(host_data):
payload_tracker = get_payload_tracker(payload_id=threadctx.request_id)
with PayloadTrackerProcessingContext(
payload_tracker, processing_status_message="adding/updating host"
) as payload_tracker_processing_ctx:
try:
logger.info("Attempting to add host...")
input_host = deserialize_host(host_data)
staleness_timestamps = Timestamps.from_config(inventory_config())
(output_host,
add_results) = host_repository.add_host(input_host,
staleness_timestamps,
fields=EGRESS_HOST_FIELDS)
metrics.add_host_success.labels(
add_results.name,
host_data.get("reporter", "null")).inc() # created vs updated
logger.info(
"Host added"
) # This definitely needs to be more specific (added vs updated?)
payload_tracker_processing_ctx.inventory_id = output_host["id"]
return (output_host, add_results)
except InventoryException:
logger.exception("Error adding host ", extra={"host": host_data})
metrics.add_host_failure.labels("InventoryException",
host_data.get("reporter",
"null")).inc()
raise
except Exception:
logger.exception("Error while adding host",
extra={"host": host_data})
metrics.add_host_failure.labels("Exception",
host_data.get("reporter",
"null")).inc()
raise
def rbac_url():
return inventory_config().rbac_endpoint + ROUTE
def staleness_filter(staleness):
config = inventory_config()
return staleness_to_conditions(config, staleness, _stale_timestamp_filter)
# print(query_results)
if args.id:
host_id_list = [args.id]
print("looking up host using id")
query_results = Host.query.filter(Host.id.in_(host_id_list)).all()
elif args.hostname:
print("looking up host using display_name, fqdn")
query_results = Host.query.filter(
Host.display_name.comparator.contains(args.hostname)
| Host.canonical_facts["fqdn"].astext.contains(args.hostname)).all(
)
elif args.insights_id:
print("looking up host using insights_id")
query_results = Host.query.filter(
Host.canonical_facts.comparator.contains(
{"insights_id": args.insights_id})).all()
elif args.account_number:
query_results = Host.query.filter(
Host.account == args.account_number).all()
staleness_timestamps = Timestamps.from_config(inventory_config())
json_host_list = [
serialize_host(host, staleness_timestamps) for host in query_results
]
if args.no_pp:
print(json_host_list)
else:
pp = pprint.PrettyPrinter(indent=4)
pp.pprint(json_host_list)
def staleness_timestamps():
return Timestamps.from_config(inventory_config())
def add_host_list(body):
if not inventory_config().rest_post_enabled:
return flask_json_response(
{
"detail": "The method is not allowed for the requested URL.",
"status": 405,
"title": "Method Not Allowed",
"type": "about:blank",
},
status=405,
)
reporter = None
response_host_list = []
number_of_errors = 0
payload_tracker = get_payload_tracker(account=current_identity.account_number, request_id=threadctx.request_id)
with PayloadTrackerContext(
payload_tracker, received_status_message="add host operation", current_operation="add host"
):
for host in body:
try:
with PayloadTrackerProcessingContext(
payload_tracker,
processing_status_message="adding/updating host",
current_operation="adding/updating host",
) as payload_tracker_processing_ctx:
if host.get("tags"):
tags_ignored_from_http_count.inc()
logger.info("Tags from an HTTP request were ignored")
input_host = deserialize_host_http(host)
output_host, host_id, _, add_result = _add_host(input_host)
status_code = _convert_host_results_to_http_status(add_result)
response_host_list.append({"status": status_code, "host": output_host})
payload_tracker_processing_ctx.inventory_id = host_id
reporter = host.get("reporter")
except ValidationException as e:
number_of_errors += 1
logger.exception("Input validation error while adding host", extra={"host": host})
response_host_list.append({**e.to_json(), "title": "Bad Request", "host": host})
except InventoryException as e:
number_of_errors += 1
logger.exception("Error adding host", extra={"host": host})
response_host_list.append({**e.to_json(), "host": host})
except Exception:
number_of_errors += 1
logger.exception("Error adding host", extra={"host": host})
response_host_list.append(
{
"status": 500,
"title": "Error",
"type": "unknown",
"detail": "Could not complete operation",
"host": host,
}
)
rest_post_request_count.labels(reporter=reporter).inc()
response = {"total": len(response_host_list), "errors": number_of_errors, "data": response_host_list}
return flask_json_response(response, status=207)