def update_system_profile(host_data, platform_metadata):
payload_tracker = get_payload_tracker(request_id=threadctx.request_id)
with PayloadTrackerProcessingContext(
payload_tracker,
processing_status_message="updating host system profile",
current_operation="updating host system profile",
) as payload_tracker_processing_ctx:
try:
input_host = deserialize_host(host_data, schema=LimitedHostSchema)
input_host.id = host_data.get("id")
staleness_timestamps = Timestamps.from_config(inventory_config())
identity = create_mock_identity_with_account(input_host.account)
output_host, host_id, insights_id, update_result = host_repository.update_system_profile(
input_host, identity, staleness_timestamps, EGRESS_HOST_FIELDS)
log_update_system_profile_success(logger, output_host)
payload_tracker_processing_ctx.inventory_id = output_host["id"]
return output_host, host_id, insights_id, update_result
except ValidationException:
metrics.update_system_profile_failure.labels(
"ValidationException").inc()
raise
except InventoryException:
log_update_system_profile_failure(logger, host_data)
raise
except OperationalError as oe:
log_db_access_failure(logger, f"Could not access DB {str(oe)}",
host_data)
raise oe
except Exception:
logger.exception("Error while updating host system profile",
extra={"host": host_data})
metrics.update_system_profile_failure.labels("Exception").inc()
raise
def delete_by_id(host_id_list):
payload_tracker = get_payload_tracker(account=current_identity.account_number, request_id=threadctx.request_id)
with PayloadTrackerContext(
payload_tracker, received_status_message="delete operation", current_operation="delete"
):
query = _get_host_list_by_id_list(current_identity.account_number, host_id_list)
if not query.count():
flask.abort(status.HTTP_404_NOT_FOUND)
for host_id, deleted in delete_hosts(query, current_app.event_producer):
if deleted:
logger.info("Deleted host: %s", host_id)
tracker_message = "deleted host"
else:
logger.info("Host %s already deleted. Delete event not emitted.", host_id)
tracker_message = "not deleted host"
with PayloadTrackerProcessingContext(
payload_tracker, processing_status_message=tracker_message
) as payload_tracker_processing_ctx:
payload_tracker_processing_ctx.inventory_id = host_id
return flask.Response(None, status.HTTP_200_OK)
def delete_by_id(host_id_list):
current_identity = get_current_identity()
payload_tracker = get_payload_tracker(
account=current_identity.account_number,
request_id=threadctx.request_id)
with PayloadTrackerContext(payload_tracker,
received_status_message="delete operation",
current_operation="delete"):
query = _get_host_list_by_id_list(host_id_list)
if not query.count():
flask.abort(status.HTTP_404_NOT_FOUND)
for host_id, deleted in delete_hosts(
query, current_app.event_producer,
inventory_config().host_delete_chunk_size):
if deleted:
log_host_delete_succeeded(logger, host_id, get_control_rule())
tracker_message = "deleted host"
else:
log_host_delete_failed(logger, host_id)
tracker_message = "not deleted host"
with PayloadTrackerProcessingContext(
payload_tracker, processing_status_message=tracker_message
) as payload_tracker_processing_ctx:
payload_tracker_processing_ctx.inventory_id = host_id
return flask.Response(None, status.HTTP_200_OK)
def add_host(host_data):
payload_tracker = get_payload_tracker(payload_id=threadctx.request_id)
with PayloadTrackerProcessingContext(
payload_tracker, processing_status_message="adding/updating host"
) as payload_tracker_processing_ctx:
try:
logger.info("Attempting to add host...")
input_host = deserialize_host(host_data)
(output_host,
add_results) = host_repository.add_host(input_host,
staleness_offset())
metrics.add_host_success.labels(
add_results.name,
host_data.get("reporter", "null")).inc() # created vs updated
logger.info(
"Host added"
) # This definitely needs to be more specific (added vs updated?)
payload_tracker_processing_ctx.inventory_id = output_host["id"]
return (output_host, add_results)
except InventoryException:
logger.exception("Error adding host ", extra={"host": host_data})
metrics.add_host_failure.labels("InventoryException",
host_data.get("reporter",
"null")).inc()
raise
except Exception:
logger.exception("Error while adding host",
extra={"host": host_data})
metrics.add_host_failure.labels("Exception",
host_data.get("reporter",
"null")).inc()
raise
def add_host(host_data):
payload_tracker = get_payload_tracker(request_id=threadctx.request_id)
with PayloadTrackerProcessingContext(
payload_tracker, processing_status_message="adding/updating host"
) as payload_tracker_processing_ctx:
try:
input_host = deserialize_host(host_data)
staleness_timestamps = Timestamps.from_config(inventory_config())
logger.info(
"Attempting to add host",
extra={
"input_host": {
"account": input_host.account,
"display_name": input_host.display_name,
"canonical_facts": input_host.canonical_facts,
"reporter": input_host.reporter,
"stale_timestamp":
input_host.stale_timestamp.isoformat(),
"tags": input_host.tags,
}
},
)
(output_host,
add_results) = host_repository.add_host(input_host,
staleness_timestamps,
fields=EGRESS_HOST_FIELDS)
metrics.add_host_success.labels(
add_results.name,
host_data.get("reporter", "null")).inc() # created vs updated
# log all the incoming host data except facts and system_profile b/c they can be quite large
logger.info(
"Host %s",
add_results.name,
extra={
"host": {
i: output_host[i]
for i in output_host
if i not in ("facts", "system_profile")
}
},
)
payload_tracker_processing_ctx.inventory_id = output_host["id"]
return (output_host, add_results)
except InventoryException:
logger.exception("Error adding host ", extra={"host": host_data})
metrics.add_host_failure.labels("InventoryException",
host_data.get("reporter",
"null")).inc()
raise
except Exception:
logger.exception("Error while adding host",
extra={"host": host_data})
metrics.add_host_failure.labels("Exception",
host_data.get("reporter",
"null")).inc()
raise
def add_host_list(host_list):
response_host_list = []
number_of_errors = 0
payload_tracker = get_payload_tracker(
account=current_identity.account_number,
payload_id=threadctx.request_id)
with PayloadTrackerContext(payload_tracker,
received_status_message="add host operation"):
for host in host_list:
try:
with PayloadTrackerProcessingContext(
payload_tracker,
processing_status_message="adding/updating host"
) as payload_tracker_processing_ctx:
input_host = deserialize_host(host)
(output_host, add_result) = _add_host(input_host)
status_code = _convert_host_results_to_http_status(
add_result)
response_host_list.append({
"status": status_code,
"host": output_host
})
payload_tracker_processing_ctx.inventory_id = output_host[
"id"]
except ValidationException as e:
number_of_errors += 1
logger.exception("Input validation error while adding host",
extra={"host": host})
response_host_list.append({
**e.to_json(), "title": "Bad Request",
"host": host
})
except InventoryException as e:
number_of_errors += 1
logger.exception("Error adding host", extra={"host": host})
response_host_list.append({**e.to_json(), "host": host})
except Exception:
number_of_errors += 1
logger.exception("Error adding host", extra={"host": host})
response_host_list.append({
"status": 500,
"title": "Error",
"type": "unknown",
"detail": "Could not complete operation",
"host": host,
})
response = {
"total": len(response_host_list),
"errors": number_of_errors,
"data": response_host_list
}
return _build_json_response(response, status=207)
def test_payload_tracker_processing_context_error(self, datetime_mock):
expected_request_id = "REQUEST_ID"
expected_processing_status = "processing"
expected_processing_status_msg = "ima processing msg"
expected_inventory_id = uuid.uuid4()
producer = Mock()
tracker = self._get_tracker(request_id=expected_request_id,
producer=producer)
operations = [None, "test operation"]
for current_operation in operations:
with self.subTest(current_operation=current_operation):
with self.assertRaises(ValueError):
with PayloadTrackerProcessingContext(
payload_tracker=tracker,
processing_status_message=
expected_processing_status_msg,
current_operation=current_operation,
) as processing_context:
expected_msg = self._build_expected_message(
status=expected_processing_status,
status_msg=expected_processing_status_msg,
request_id=expected_request_id,
datetime_mock=datetime_mock,
)
self._verify_mock_send_call(producer,
self.DEFAULT_TOPIC,
expected_msg)
producer.reset_mock()
processing_context.inventory_id = expected_inventory_id
method_to_raise_exception()
expected_msg = self._build_expected_message(
status="processing_error",
status_msg=self.
_build_payload_tracker_context_error_message(
"ValueError", current_operation,
"something bad happened!"),
request_id=expected_request_id,
datetime_mock=datetime_mock,
)
expected_msg["inventory_id"] = str(expected_inventory_id)
self.assertIsNone(tracker.inventory_id)
self._verify_mock_send_call(producer, self.DEFAULT_TOPIC,
expected_msg)
producer.reset_mock()
def test_payload_tracker_processing_context_error(self, datetime_mock):
expected_payload_id = "REQUEST_ID"
expected_processing_status = "processing"
expected_processing_status_msg = "ima processing msg"
expected_inventory_id = uuid.uuid4()
producer = Mock()
tracker = self._get_tracker(payload_id=expected_payload_id,
producer=producer)
error_status_msgs = [None, "ima error status msg"]
for error_status_msg in error_status_msgs:
with self.subTest(error_status_msg=error_status_msg):
with self.assertRaises(ValueError):
with PayloadTrackerProcessingContext(
payload_tracker=tracker,
processing_status_message=
expected_processing_status_msg,
error_status_message=error_status_msg,
) as processing_context:
expected_msg = self._build_expected_message(
status=expected_processing_status,
status_msg=expected_processing_status_msg,
payload_id=expected_payload_id,
datetime_mock=datetime_mock,
)
self._verify_mock_send_call(producer,
self.DEFAULT_TOPIC,
expected_msg)
producer.reset_mock()
processing_context.inventory_id = expected_inventory_id
method_to_raise_exception()
expected_msg = self._build_expected_message(
status="processing_error",
status_msg=error_status_msg,
payload_id=expected_payload_id,
datetime_mock=datetime_mock,
)
expected_msg["inventory_id"] = str(expected_inventory_id)
self.assertIsNone(tracker.inventory_id)
self._verify_mock_send_call(producer, self.DEFAULT_TOPIC,
expected_msg)
producer.reset_mock()
def test_payload_tracker_processing_context_success(self, datetime_mock):
expected_request_id = "REQUEST_ID"
expected_processing_status = "processing"
expected_processing_status_msg = "ima processing msg"
expected_inventory_id = uuid.uuid4()
producer = Mock()
tracker = self._get_tracker(request_id=expected_request_id,
producer=producer)
success_status_msgs = [None, "ima success status msg"]
for success_status_msg in success_status_msgs:
with self.subTest(success_status_msg=success_status_msg):
with PayloadTrackerProcessingContext(
payload_tracker=tracker,
processing_status_message=
expected_processing_status_msg,
success_status_message=success_status_msg,
) as processing_context:
expected_msg = self._build_expected_message(
status=expected_processing_status,
status_msg=expected_processing_status_msg,
request_id=expected_request_id,
datetime_mock=datetime_mock,
)
self._verify_mock_send_call(producer, self.DEFAULT_TOPIC,
expected_msg)
processing_context.inventory_id = expected_inventory_id
producer.reset_mock()
expected_msg = self._build_expected_message(
status="processing_success",
status_msg=success_status_msg,
request_id=expected_request_id,
datetime_mock=datetime_mock,
)
expected_msg["inventory_id"] = str(expected_inventory_id)
self.assertIsNone(tracker.inventory_id)
self._verify_mock_send_call(producer, self.DEFAULT_TOPIC,
expected_msg)
producer.reset_mock()
def add_host(host_data, platform_metadata):
payload_tracker = get_payload_tracker(request_id=threadctx.request_id)
with PayloadTrackerProcessingContext(
payload_tracker,
processing_status_message="adding/updating host",
current_operation="adding/updating host"
) as payload_tracker_processing_ctx:
try:
identity = _get_identity(host_data, platform_metadata)
# basic-auth does not need owner_id
if identity.identity_type == IdentityType.SYSTEM:
host_data = _set_owner(host_data, identity)
input_host = deserialize_host(host_data)
staleness_timestamps = Timestamps.from_config(inventory_config())
log_add_host_attempt(logger, input_host)
output_host, host_id, insights_id, add_result = host_repository.add_host(
input_host,
identity,
staleness_timestamps,
fields=EGRESS_HOST_FIELDS)
log_add_update_host_succeeded(logger, add_result, host_data,
output_host)
payload_tracker_processing_ctx.inventory_id = output_host["id"]
return output_host, host_id, insights_id, add_result
except ValidationException:
metrics.add_host_failure.labels("ValidationException",
host_data.get("reporter",
"null")).inc()
raise
except InventoryException as ie:
log_add_host_failure(logger, str(ie.detail), host_data)
raise
except OperationalError as oe:
log_db_access_failure(logger, f"Could not access DB {str(oe)}",
host_data)
raise oe
except Exception:
logger.exception("Error while adding host",
extra={"host": host_data})
metrics.add_host_failure.labels("Exception",
host_data.get("reporter",
"null")).inc()
raise
def delete_by_id(host_id_list):
payload_tracker = get_payload_tracker(
account=current_identity.account_number,
payload_id=threadctx.request_id)
with PayloadTrackerContext(payload_tracker,
received_status_message="delete operation"):
query = _get_host_list_by_id_list(current_identity.account_number,
host_id_list)
hosts_to_delete = query.all()
if not hosts_to_delete:
return flask.abort(status.HTTP_404_NOT_FOUND)
with metrics.delete_host_processing_time.time():
query.delete(synchronize_session="fetch")
db.session.commit()
metrics.delete_host_count.inc(len(hosts_to_delete))
# This process of checking for an already deleted host relies
# on checking the session after it has been updated by the commit()
# function and marked the deleted hosts as expired. It is after this
# change that the host is called by a new query and, if deleted by a
# different process, triggers the ObjectDeletedError and is not emited.
for deleted_host in hosts_to_delete:
# Prevents ObjectDeletedError from being raised.
if instance_state(deleted_host).expired:
# Can’t log the Host ID. Accessing an attribute raises ObjectDeletedError.
logger.info("Host already deleted. Delete event not emitted.")
else:
with PayloadTrackerProcessingContext(
payload_tracker,
processing_status_message="deleted host"
) as payload_tracker_processing_ctx:
logger.debug("Deleted host: %s", deleted_host)
emit_event(events.delete(deleted_host))
payload_tracker_processing_ctx.inventory_id = deleted_host.id
def add_host_list(body):
if not inventory_config().rest_post_enabled:
return flask_json_response(
{
"detail": "The method is not allowed for the requested URL.",
"status": 405,
"title": "Method Not Allowed",
"type": "about:blank",
},
status=405,
)
reporter = None
response_host_list = []
number_of_errors = 0
payload_tracker = get_payload_tracker(account=current_identity.account_number, request_id=threadctx.request_id)
with PayloadTrackerContext(
payload_tracker, received_status_message="add host operation", current_operation="add host"
):
for host in body:
try:
with PayloadTrackerProcessingContext(
payload_tracker,
processing_status_message="adding/updating host",
current_operation="adding/updating host",
) as payload_tracker_processing_ctx:
if host.get("tags"):
tags_ignored_from_http_count.inc()
logger.info("Tags from an HTTP request were ignored")
input_host = deserialize_host_http(host)
output_host, host_id, _, add_result = _add_host(input_host)
status_code = _convert_host_results_to_http_status(add_result)
response_host_list.append({"status": status_code, "host": output_host})
payload_tracker_processing_ctx.inventory_id = host_id
reporter = host.get("reporter")
except ValidationException as e:
number_of_errors += 1
logger.exception("Input validation error while adding host", extra={"host": host})
response_host_list.append({**e.to_json(), "title": "Bad Request", "host": host})
except InventoryException as e:
number_of_errors += 1
logger.exception("Error adding host", extra={"host": host})
response_host_list.append({**e.to_json(), "host": host})
except Exception:
number_of_errors += 1
logger.exception("Error adding host", extra={"host": host})
response_host_list.append(
{
"status": 500,
"title": "Error",
"type": "unknown",
"detail": "Could not complete operation",
"host": host,
}
)
rest_post_request_count.labels(reporter=reporter).inc()
response = {"total": len(response_host_list), "errors": number_of_errors, "data": response_host_list}
return flask_json_response(response, status=207)