def delete_by_id(host_id_list):
payload_tracker = get_payload_tracker(account=current_identity.account_number, request_id=threadctx.request_id)
with PayloadTrackerContext(
payload_tracker, received_status_message="delete operation", current_operation="delete"
):
query = _get_host_list_by_id_list(current_identity.account_number, host_id_list)
if not query.count():
flask.abort(status.HTTP_404_NOT_FOUND)
for host_id, deleted in delete_hosts(query, current_app.event_producer):
if deleted:
logger.info("Deleted host: %s", host_id)
tracker_message = "deleted host"
else:
logger.info("Host %s already deleted. Delete event not emitted.", host_id)
tracker_message = "not deleted host"
with PayloadTrackerProcessingContext(
payload_tracker, processing_status_message=tracker_message
) as payload_tracker_processing_ctx:
payload_tracker_processing_ctx.inventory_id = host_id
return flask.Response(None, status.HTTP_200_OK)
def delete_by_id(host_id_list):
current_identity = get_current_identity()
payload_tracker = get_payload_tracker(
account=current_identity.account_number,
request_id=threadctx.request_id)
with PayloadTrackerContext(payload_tracker,
received_status_message="delete operation",
current_operation="delete"):
query = _get_host_list_by_id_list(host_id_list)
if not query.count():
flask.abort(status.HTTP_404_NOT_FOUND)
for host_id, deleted in delete_hosts(
query, current_app.event_producer,
inventory_config().host_delete_chunk_size):
if deleted:
log_host_delete_succeeded(logger, host_id, get_control_rule())
tracker_message = "deleted host"
else:
log_host_delete_failed(logger, host_id)
tracker_message = "not deleted host"
with PayloadTrackerProcessingContext(
payload_tracker, processing_status_message=tracker_message
) as payload_tracker_processing_ctx:
payload_tracker_processing_ctx.inventory_id = host_id
return flask.Response(None, status.HTTP_200_OK)
def add_host(host_data):
payload_tracker = get_payload_tracker(payload_id=threadctx.request_id)
with PayloadTrackerProcessingContext(
payload_tracker, processing_status_message="adding/updating host"
) as payload_tracker_processing_ctx:
try:
logger.info("Attempting to add host...")
input_host = deserialize_host(host_data)
(output_host,
add_results) = host_repository.add_host(input_host,
staleness_offset())
metrics.add_host_success.labels(
add_results.name,
host_data.get("reporter", "null")).inc() # created vs updated
logger.info(
"Host added"
) # This definitely needs to be more specific (added vs updated?)
payload_tracker_processing_ctx.inventory_id = output_host["id"]
return (output_host, add_results)
except InventoryException:
logger.exception("Error adding host ", extra={"host": host_data})
metrics.add_host_failure.labels("InventoryException",
host_data.get("reporter",
"null")).inc()
raise
except Exception:
logger.exception("Error while adding host",
extra={"host": host_data})
metrics.add_host_failure.labels("Exception",
host_data.get("reporter",
"null")).inc()
raise
def update_system_profile(host_data, platform_metadata):
payload_tracker = get_payload_tracker(request_id=threadctx.request_id)
with PayloadTrackerProcessingContext(
payload_tracker,
processing_status_message="updating host system profile",
current_operation="updating host system profile",
) as payload_tracker_processing_ctx:
try:
input_host = deserialize_host(host_data, schema=LimitedHostSchema)
input_host.id = host_data.get("id")
staleness_timestamps = Timestamps.from_config(inventory_config())
identity = create_mock_identity_with_account(input_host.account)
output_host, host_id, insights_id, update_result = host_repository.update_system_profile(
input_host, identity, staleness_timestamps, EGRESS_HOST_FIELDS)
log_update_system_profile_success(logger, output_host)
payload_tracker_processing_ctx.inventory_id = output_host["id"]
return output_host, host_id, insights_id, update_result
except ValidationException:
metrics.update_system_profile_failure.labels(
"ValidationException").inc()
raise
except InventoryException:
log_update_system_profile_failure(logger, host_data)
raise
except OperationalError as oe:
log_db_access_failure(logger, f"Could not access DB {str(oe)}",
host_data)
raise oe
except Exception:
logger.exception("Error while updating host system profile",
extra={"host": host_data})
metrics.update_system_profile_failure.labels("Exception").inc()
raise
def handle_message(message, event_producer):
validated_operation_msg = parse_operation_message(message)
platform_metadata = validated_operation_msg.get("platform_metadata") or {}
# create a dummy identity for working around the identity requirement for CRUD operations
identity = Identity(USER_IDENTITY)
# set account_number in dummy idenity to the actual account_number received in the payload
identity.account_number = validated_operation_msg["data"]["account"]
request_id = platform_metadata.get("request_id", "-1")
initialize_thread_local_storage(request_id)
payload_tracker = get_payload_tracker(request_id=request_id)
with PayloadTrackerContext(payload_tracker,
received_status_message="message received",
current_operation="handle_message"):
output_host, host_id, insights_id, add_results = add_host(
validated_operation_msg["data"], identity)
event_type = add_host_results_to_event_type(add_results)
event = build_event(event_type,
output_host,
platform_metadata=platform_metadata)
headers = message_headers(add_results, insights_id)
event_producer.write_event(event, str(host_id), headers, Topic.egress)
# for transition to platform.inventory.events
if inventory_config().secondary_topic_enabled:
event_producer.write_event(event, str(host_id), headers,
Topic.events)
def add_host(host_data):
payload_tracker = get_payload_tracker(request_id=threadctx.request_id)
with PayloadTrackerProcessingContext(
payload_tracker, processing_status_message="adding/updating host"
) as payload_tracker_processing_ctx:
try:
input_host = deserialize_host(host_data)
staleness_timestamps = Timestamps.from_config(inventory_config())
logger.info(
"Attempting to add host",
extra={
"input_host": {
"account": input_host.account,
"display_name": input_host.display_name,
"canonical_facts": input_host.canonical_facts,
"reporter": input_host.reporter,
"stale_timestamp":
input_host.stale_timestamp.isoformat(),
"tags": input_host.tags,
}
},
)
(output_host,
add_results) = host_repository.add_host(input_host,
staleness_timestamps,
fields=EGRESS_HOST_FIELDS)
metrics.add_host_success.labels(
add_results.name,
host_data.get("reporter", "null")).inc() # created vs updated
# log all the incoming host data except facts and system_profile b/c they can be quite large
logger.info(
"Host %s",
add_results.name,
extra={
"host": {
i: output_host[i]
for i in output_host
if i not in ("facts", "system_profile")
}
},
)
payload_tracker_processing_ctx.inventory_id = output_host["id"]
return (output_host, add_results)
except InventoryException:
logger.exception("Error adding host ", extra={"host": host_data})
metrics.add_host_failure.labels("InventoryException",
host_data.get("reporter",
"null")).inc()
raise
except Exception:
logger.exception("Error while adding host",
extra={"host": host_data})
metrics.add_host_failure.labels("Exception",
host_data.get("reporter",
"null")).inc()
raise
def add_host_list(host_list):
response_host_list = []
number_of_errors = 0
payload_tracker = get_payload_tracker(
account=current_identity.account_number,
payload_id=threadctx.request_id)
with PayloadTrackerContext(payload_tracker,
received_status_message="add host operation"):
for host in host_list:
try:
with PayloadTrackerProcessingContext(
payload_tracker,
processing_status_message="adding/updating host"
) as payload_tracker_processing_ctx:
input_host = deserialize_host(host)
(output_host, add_result) = _add_host(input_host)
status_code = _convert_host_results_to_http_status(
add_result)
response_host_list.append({
"status": status_code,
"host": output_host
})
payload_tracker_processing_ctx.inventory_id = output_host[
"id"]
except ValidationException as e:
number_of_errors += 1
logger.exception("Input validation error while adding host",
extra={"host": host})
response_host_list.append({
**e.to_json(), "title": "Bad Request",
"host": host
})
except InventoryException as e:
number_of_errors += 1
logger.exception("Error adding host", extra={"host": host})
response_host_list.append({**e.to_json(), "host": host})
except Exception:
number_of_errors += 1
logger.exception("Error adding host", extra={"host": host})
response_host_list.append({
"status": 500,
"title": "Error",
"type": "unknown",
"detail": "Could not complete operation",
"host": host,
})
response = {
"total": len(response_host_list),
"errors": number_of_errors,
"data": response_host_list
}
return _build_json_response(response, status=207)
def handle_message(message, event_producer):
validated_operation_msg = parse_operation_message(message)
platform_metadata = validated_operation_msg.get("platform_metadata") or {}
initialize_thread_local_storage(platform_metadata)
payload_tracker = get_payload_tracker(request_id=threadctx.request_id)
with PayloadTrackerContext(payload_tracker, received_status_message="message received"):
(output_host, add_results) = add_host(validated_operation_msg["data"])
event = build_egress_topic_event(add_results.name, output_host, platform_metadata)
event_producer.write_event(event, output_host["id"], message_headers(add_results.name))
def add_host(host_data, platform_metadata):
payload_tracker = get_payload_tracker(request_id=threadctx.request_id)
with PayloadTrackerProcessingContext(
payload_tracker,
processing_status_message="adding/updating host",
current_operation="adding/updating host"
) as payload_tracker_processing_ctx:
try:
identity = _get_identity(host_data, platform_metadata)
# basic-auth does not need owner_id
if identity.identity_type == IdentityType.SYSTEM:
host_data = _set_owner(host_data, identity)
input_host = deserialize_host(host_data)
staleness_timestamps = Timestamps.from_config(inventory_config())
log_add_host_attempt(logger, input_host)
output_host, host_id, insights_id, add_result = host_repository.add_host(
input_host,
identity,
staleness_timestamps,
fields=EGRESS_HOST_FIELDS)
log_add_update_host_succeeded(logger, add_result, host_data,
output_host)
payload_tracker_processing_ctx.inventory_id = output_host["id"]
return output_host, host_id, insights_id, add_result
except ValidationException:
metrics.add_host_failure.labels("ValidationException",
host_data.get("reporter",
"null")).inc()
raise
except InventoryException as ie:
log_add_host_failure(logger, str(ie.detail), host_data)
raise
except OperationalError as oe:
log_db_access_failure(logger, f"Could not access DB {str(oe)}",
host_data)
raise oe
except Exception:
logger.exception("Error while adding host",
extra={"host": host_data})
metrics.add_host_failure.labels("Exception",
host_data.get("reporter",
"null")).inc()
raise
def handle_message(message, event_producer):
validated_operation_msg = parse_operation_message(message)
platform_metadata = validated_operation_msg.get("platform_metadata") or {}
request_id = platform_metadata.get("request_id", "-1")
initialize_thread_local_storage(request_id)
payload_tracker = get_payload_tracker(request_id=request_id)
with PayloadTrackerContext(
payload_tracker, received_status_message="message received", current_operation="handle_message"
):
(output_host, add_results) = add_host(validated_operation_msg["data"])
event_type = add_host_results_to_event_type(add_results)
event = build_event(event_type, output_host, platform_metadata=platform_metadata)
event_producer.write_event(event, output_host["id"], message_headers(add_results), Topic.egress)
# for transition to platform.inventory.events
if inventory_config().secondary_topic_enabled:
event_producer.write_event(event, output_host["id"], message_headers(add_results), Topic.events)
def handle_message(message, event_producer):
validated_operation_msg = parse_operation_message(message)
metadata = validated_operation_msg.get("platform_metadata") or {}
initialize_thread_local_storage(metadata)
payload_tracker = get_payload_tracker(payload_id=threadctx.request_id)
with PayloadTrackerContext(payload_tracker,
received_status_message="message received"):
# FIXME: verify operation type
(output_host, add_results) = add_host(validated_operation_msg["data"])
if add_results == host_repository.AddHostResults.created:
event_type = "created"
else:
event_type = "updated"
event = build_event(event_type, output_host, metadata)
event_producer.write_event(event, output_host["id"])
def delete_by_id(host_id_list):
payload_tracker = get_payload_tracker(
account=current_identity.account_number,
payload_id=threadctx.request_id)
with PayloadTrackerContext(payload_tracker,
received_status_message="delete operation"):
query = _get_host_list_by_id_list(current_identity.account_number,
host_id_list)
hosts_to_delete = query.all()
if not hosts_to_delete:
return flask.abort(status.HTTP_404_NOT_FOUND)
with metrics.delete_host_processing_time.time():
query.delete(synchronize_session="fetch")
db.session.commit()
metrics.delete_host_count.inc(len(hosts_to_delete))
# This process of checking for an already deleted host relies
# on checking the session after it has been updated by the commit()
# function and marked the deleted hosts as expired. It is after this
# change that the host is called by a new query and, if deleted by a
# different process, triggers the ObjectDeletedError and is not emited.
for deleted_host in hosts_to_delete:
# Prevents ObjectDeletedError from being raised.
if instance_state(deleted_host).expired:
# Can’t log the Host ID. Accessing an attribute raises ObjectDeletedError.
logger.info("Host already deleted. Delete event not emitted.")
else:
with PayloadTrackerProcessingContext(
payload_tracker,
processing_status_message="deleted host"
) as payload_tracker_processing_ctx:
logger.debug("Deleted host: %s", deleted_host)
emit_event(events.delete(deleted_host))
payload_tracker_processing_ctx.inventory_id = deleted_host.id
def handle_message(message, event_producer, message_operation=add_host):
validated_operation_msg = parse_operation_message(message)
platform_metadata = validated_operation_msg.get("platform_metadata", {})
request_id = platform_metadata.get("request_id", UNKNOWN_REQUEST_ID_VALUE)
initialize_thread_local_storage(request_id)
payload_tracker = get_payload_tracker(request_id=request_id)
with PayloadTrackerContext(payload_tracker,
received_status_message="message received",
current_operation="handle_message"):
try:
host = validated_operation_msg["data"]
output_host, host_id, insights_id, operation_result = message_operation(
host, platform_metadata)
event_type = operation_results_to_event_type(operation_result)
event = build_event(event_type,
output_host,
platform_metadata=platform_metadata)
headers = message_headers(operation_result, insights_id)
event_producer.write_event(event, str(host_id), headers)
except ValidationException as ve:
logger.error(
"Validation error while adding or updating host: %s",
ve,
extra={"host": {
"reporter": host.get("reporter")
}},
)
raise
except ValueError as ve:
logger.error("Value error while adding or updating host: %s",
ve,
extra={"reporter": host.get("reporter")})
raise
def _get_tracker(self, account=None, payload_id=None, producer=None):
config = Config(RuntimeEnvironment.server)
init_payload_tracker(config, producer=producer)
return get_payload_tracker(account=account, payload_id=payload_id)
def add_host_list(body):
if not inventory_config().rest_post_enabled:
return flask_json_response(
{
"detail": "The method is not allowed for the requested URL.",
"status": 405,
"title": "Method Not Allowed",
"type": "about:blank",
},
status=405,
)
reporter = None
response_host_list = []
number_of_errors = 0
payload_tracker = get_payload_tracker(account=current_identity.account_number, request_id=threadctx.request_id)
with PayloadTrackerContext(
payload_tracker, received_status_message="add host operation", current_operation="add host"
):
for host in body:
try:
with PayloadTrackerProcessingContext(
payload_tracker,
processing_status_message="adding/updating host",
current_operation="adding/updating host",
) as payload_tracker_processing_ctx:
if host.get("tags"):
tags_ignored_from_http_count.inc()
logger.info("Tags from an HTTP request were ignored")
input_host = deserialize_host_http(host)
output_host, host_id, _, add_result = _add_host(input_host)
status_code = _convert_host_results_to_http_status(add_result)
response_host_list.append({"status": status_code, "host": output_host})
payload_tracker_processing_ctx.inventory_id = host_id
reporter = host.get("reporter")
except ValidationException as e:
number_of_errors += 1
logger.exception("Input validation error while adding host", extra={"host": host})
response_host_list.append({**e.to_json(), "title": "Bad Request", "host": host})
except InventoryException as e:
number_of_errors += 1
logger.exception("Error adding host", extra={"host": host})
response_host_list.append({**e.to_json(), "host": host})
except Exception:
number_of_errors += 1
logger.exception("Error adding host", extra={"host": host})
response_host_list.append(
{
"status": 500,
"title": "Error",
"type": "unknown",
"detail": "Could not complete operation",
"host": host,
}
)
rest_post_request_count.labels(reporter=reporter).inc()
response = {"total": len(response_host_list), "errors": number_of_errors, "data": response_host_list}
return flask_json_response(response, status=207)
def _get_tracker(self, account=None, payload_id=None, producer=None):
config = Config()
init_payload_tracker(config, producer=producer)
return get_payload_tracker(account=account, payload_id=payload_id)
def _payload_tracker(account=None, request_id=None, producer=None):
config = Config(RuntimeEnvironment.SERVER)
init_payload_tracker(config, producer=producer)
return get_payload_tracker(account=account, request_id=request_id)
