def get_rbac_permissions(): request_header = { IDENTITY_HEADER: request.headers[IDENTITY_HEADER], REQUEST_ID_HEADER: request.headers.get(REQUEST_ID_HEADER, UNKNOWN_REQUEST_ID_VALUE), } request_session = Session() retry_config = Retry(total=inventory_config().rbac_retries, backoff_factor=1, status_forcelist=RETRY_STATUSES) request_session.mount(rbac_url(), HTTPAdapter(max_retries=retry_config)) try: with outbound_http_metric.time(): rbac_response = request_session.get( url=rbac_url(), headers=request_header, timeout=inventory_config().rbac_timeout) except Exception as e: rbac_failure(logger, e) abort(503, "Failed to reach RBAC endpoint, request cannot be fulfilled") finally: request_session.close() resp_data = rbac_response.json() logger.debug("Fetched RBAC Data", extra=resp_data) return resp_data["data"]
def get_bulk_query_source(): if XJOIN_HEADER in connexion.request.headers: if connexion.request.headers[XJOIN_HEADER].lower() == "xjoin": return BulkQuerySource.xjoin elif connexion.request.headers[XJOIN_HEADER].lower() == "db": return BulkQuerySource.db if REFERAL_HEADER in connexion.request.headers: if "/beta" in connexion.request.headers[REFERAL_HEADER]: return inventory_config().bulk_query_source_beta return inventory_config().bulk_query_source
def _delete_filtered_hosts(host_id_list): current_identity = get_current_identity() payload_tracker = get_payload_tracker( account=current_identity.account_number, request_id=threadctx.request_id) with PayloadTrackerContext(payload_tracker, received_status_message="delete operation", current_operation="delete"): query = _get_host_list_by_id_list(host_id_list) if not query.count(): flask.abort(status.HTTP_404_NOT_FOUND) deletion_count = 0 for host_id, deleted in delete_hosts( query, current_app.event_producer, inventory_config().host_delete_chunk_size): if deleted: log_host_delete_succeeded(logger, host_id, get_control_rule()) tracker_message = "deleted host" deletion_count += 1 else: log_host_delete_failed(logger, host_id, get_control_rule()) tracker_message = "not deleted host" with PayloadTrackerProcessingContext( payload_tracker, processing_status_message=tracker_message ) as payload_tracker_processing_ctx: payload_tracker_processing_ctx.inventory_id = host_id return deletion_count
def modified_func(*args, **kwargs): if not inventory_config().rbac_enforced: return func(*args, **kwargs) if get_current_identity().identity_type != CHECKED_TYPE: return func(*args, **kwargs) # track that RBAC is being used to control access g.access_control_rule = "RBAC" logger.debug("access_control_rule set") rbac_data = get_rbac_permissions() permission_type = required_permission.value.split(":")[2] for rbac_permission in rbac_data: if (rbac_permission["permission"] == Permission.ADMIN.value # inventory:*:* or rbac_permission["permission"] == Permission.HOSTS_ALL.value # inventory:hosts:* or rbac_permission["permission"] == f"inventory:*:{permission_type}" # inventory:*:(read | write) or rbac_permission["permission"] == required_permission .value # inventory:hosts:(read | write) ): return func(*args, **kwargs) rbac_permission_denied(logger, required_permission.value, rbac_data) abort(status.HTTP_403_FORBIDDEN)
def handle_message(message, event_producer): validated_operation_msg = parse_operation_message(message) platform_metadata = validated_operation_msg.get("platform_metadata") or {} # create a dummy identity for working around the identity requirement for CRUD operations identity = Identity(USER_IDENTITY) # set account_number in dummy idenity to the actual account_number received in the payload identity.account_number = validated_operation_msg["data"]["account"] request_id = platform_metadata.get("request_id", "-1") initialize_thread_local_storage(request_id) payload_tracker = get_payload_tracker(request_id=request_id) with PayloadTrackerContext(payload_tracker, received_status_message="message received", current_operation="handle_message"): output_host, host_id, insights_id, add_results = add_host( validated_operation_msg["data"], identity) event_type = add_host_results_to_event_type(add_results) event = build_event(event_type, output_host, platform_metadata=platform_metadata) headers = message_headers(add_results, insights_id) event_producer.write_event(event, str(host_id), headers, Topic.egress) # for transition to platform.inventory.events if inventory_config().secondary_topic_enabled: event_producer.write_event(event, str(host_id), headers, Topic.events)
def update_system_profile(host_data, platform_metadata): payload_tracker = get_payload_tracker(request_id=threadctx.request_id) with PayloadTrackerProcessingContext( payload_tracker, processing_status_message="updating host system profile", current_operation="updating host system profile", ) as payload_tracker_processing_ctx: try: input_host = deserialize_host(host_data, schema=LimitedHostSchema) input_host.id = host_data.get("id") staleness_timestamps = Timestamps.from_config(inventory_config()) identity = create_mock_identity_with_account(input_host.account) output_host, host_id, insights_id, update_result = host_repository.update_system_profile( input_host, identity, staleness_timestamps, EGRESS_HOST_FIELDS) log_update_system_profile_success(logger, output_host) payload_tracker_processing_ctx.inventory_id = output_host["id"] return output_host, host_id, insights_id, update_result except ValidationException: metrics.update_system_profile_failure.labels( "ValidationException").inc() raise except InventoryException: log_update_system_profile_failure(logger, host_data) raise except OperationalError as oe: log_db_access_failure(logger, f"Could not access DB {str(oe)}", host_data) raise oe except Exception: logger.exception("Error while updating host system profile", extra={"host": host_data}) metrics.update_system_profile_failure.labels("Exception").inc() raise
def delete_by_id(host_id_list): payload_tracker = get_payload_tracker(account=current_identity.account_number, request_id=threadctx.request_id) with PayloadTrackerContext( payload_tracker, received_status_message="delete operation", current_operation="delete" ): query = _get_host_list_by_id_list(current_identity.account_number, host_id_list) if not query.count(): flask.abort(status.HTTP_404_NOT_FOUND) for host_id, deleted in delete_hosts( query, current_app.event_producer, inventory_config().host_delete_chunk_size ): if deleted: logger.info("Deleted host: %s", host_id) tracker_message = "deleted host" else: logger.info("Host %s already deleted. Delete event not emitted.", host_id) tracker_message = "not deleted host" with PayloadTrackerProcessingContext( payload_tracker, processing_status_message=tracker_message ) as payload_tracker_processing_ctx: payload_tracker_processing_ctx.inventory_id = host_id return flask.Response(None, status.HTTP_200_OK)
def find_hosts_by_staleness(staleness, query): logger.debug("find_hosts_by_staleness(%s)", staleness) config = inventory_config() staleness_conditions = tuple(staleness_to_conditions(config, staleness, stale_timestamp_filter)) if "unknown" in staleness: staleness_conditions += (Host.stale_timestamp == NULL,) return query.filter(or_(*staleness_conditions))
def add_host(host_data): payload_tracker = get_payload_tracker(request_id=threadctx.request_id) with PayloadTrackerProcessingContext( payload_tracker, processing_status_message="adding/updating host" ) as payload_tracker_processing_ctx: try: input_host = deserialize_host(host_data) staleness_timestamps = Timestamps.from_config(inventory_config()) logger.info( "Attempting to add host", extra={ "input_host": { "account": input_host.account, "display_name": input_host.display_name, "canonical_facts": input_host.canonical_facts, "reporter": input_host.reporter, "stale_timestamp": input_host.stale_timestamp.isoformat(), "tags": input_host.tags, } }, ) (output_host, add_results) = host_repository.add_host(input_host, staleness_timestamps, fields=EGRESS_HOST_FIELDS) metrics.add_host_success.labels( add_results.name, host_data.get("reporter", "null")).inc() # created vs updated # log all the incoming host data except facts and system_profile b/c they can be quite large logger.info( "Host %s", add_results.name, extra={ "host": { i: output_host[i] for i in output_host if i not in ("facts", "system_profile") } }, ) payload_tracker_processing_ctx.inventory_id = output_host["id"] return (output_host, add_results) except InventoryException: logger.exception("Error adding host ", extra={"host": host_data}) metrics.add_host_failure.labels("InventoryException", host_data.get("reporter", "null")).inc() raise except Exception: logger.exception("Error while adding host", extra={"host": host_data}) metrics.add_host_failure.labels("Exception", host_data.get("reporter", "null")).inc() raise
def add_host(host_data, platform_metadata): payload_tracker = get_payload_tracker(request_id=threadctx.request_id) with PayloadTrackerProcessingContext( payload_tracker, processing_status_message="adding/updating host", current_operation="adding/updating host" ) as payload_tracker_processing_ctx: try: identity = _get_identity(host_data, platform_metadata) # basic-auth does not need owner_id if identity.identity_type == IdentityType.SYSTEM: host_data = _set_owner(host_data, identity) input_host = deserialize_host(host_data) staleness_timestamps = Timestamps.from_config(inventory_config()) log_add_host_attempt(logger, input_host) output_host, host_id, insights_id, add_result = host_repository.add_host( input_host, identity, staleness_timestamps, fields=EGRESS_HOST_FIELDS) log_add_update_host_succeeded(logger, add_result, host_data, output_host) payload_tracker_processing_ctx.inventory_id = output_host["id"] return output_host, host_id, insights_id, add_result except ValidationException: metrics.add_host_failure.labels("ValidationException", host_data.get("reporter", "null")).inc() raise except InventoryException as ie: log_add_host_failure(logger, str(ie.detail), host_data) raise except OperationalError as oe: log_db_access_failure(logger, f"Could not access DB {str(oe)}", host_data) raise oe except Exception: logger.exception("Error while adding host", extra={"host": host_data}) metrics.add_host_failure.labels("Exception", host_data.get("reporter", "null")).inc() raise
def modified_func(*args, **kwargs): if not inventory_config().rbac_enforced: return func(*args, **kwargs) if current_identity.identity_type != CHECKED_TYPE: return func(*args, **kwargs) rbac_data = get_rbac_permissions() for rbac_permission in rbac_data: if ( rbac_permission["permission"] == Permission.ADMIN.value or rbac_permission["permission"] == Permission.HOSTS_ALL.value or rbac_permission["permission"] == required_permission.value ): return func(*args, **kwargs) rbac_permission_denied(logger, required_permission.value, rbac_data) abort(status.HTTP_403_FORBIDDEN)
def handle_message(message, event_producer): validated_operation_msg = parse_operation_message(message) platform_metadata = validated_operation_msg.get("platform_metadata") or {} request_id = platform_metadata.get("request_id", "-1") initialize_thread_local_storage(request_id) payload_tracker = get_payload_tracker(request_id=request_id) with PayloadTrackerContext( payload_tracker, received_status_message="message received", current_operation="handle_message" ): (output_host, add_results) = add_host(validated_operation_msg["data"]) event_type = add_host_results_to_event_type(add_results) event = build_event(event_type, output_host, platform_metadata=platform_metadata) event_producer.write_event(event, output_host["id"], message_headers(add_results), Topic.egress) # for transition to platform.inventory.events if inventory_config().secondary_topic_enabled: event_producer.write_event(event, output_host["id"], message_headers(add_results), Topic.events)
def modified_func(*args, **kwargs): if not inventory_config().rbac_enforced: return func(*args, **kwargs) if current_identity.identity_type != CHECKED_TYPE: return func(*args, **kwargs) rbac_data = get_rbac_permissions() permission_type = required_permission.value.split(":")[2] for rbac_permission in rbac_data: if ( rbac_permission["permission"] == Permission.ADMIN.value # inventory:*:* or rbac_permission["permission"] == Permission.HOSTS_ALL.value # inventory:hosts:* or rbac_permission["permission"] == f"inventory:*:{permission_type}" # inventory:*:(read | write) or rbac_permission["permission"] == required_permission.value # inventory:hosts:(read | write) ): return func(*args, **kwargs) rbac_permission_denied(logger, required_permission.value, rbac_data) abort(status.HTTP_403_FORBIDDEN)
def build_registered_with_filter(registered_with): reg_with_copy = deepcopy(registered_with) prs_list = [] if "insights" in reg_with_copy: prs_list.append({"NOT": {"insights_id": {"eq": None}}}) reg_with_copy.remove("insights") if reg_with_copy: for item in reg_with_copy: prs_list.append({ "per_reporter_staleness": { "reporter": { "eq": item }, "stale_timestamp": { "gt": str((datetime.now(timezone.utc) - inventory_config().culling_culled_offset_delta ).isoformat()) }, }, }) return ({"OR": prs_list}, )
def add_host(host_data): payload_tracker = get_payload_tracker(payload_id=threadctx.request_id) with PayloadTrackerProcessingContext( payload_tracker, processing_status_message="adding/updating host" ) as payload_tracker_processing_ctx: try: logger.info("Attempting to add host...") input_host = deserialize_host(host_data) staleness_timestamps = Timestamps.from_config(inventory_config()) (output_host, add_results) = host_repository.add_host(input_host, staleness_timestamps, fields=EGRESS_HOST_FIELDS) metrics.add_host_success.labels( add_results.name, host_data.get("reporter", "null")).inc() # created vs updated logger.info( "Host added" ) # This definitely needs to be more specific (added vs updated?) payload_tracker_processing_ctx.inventory_id = output_host["id"] return (output_host, add_results) except InventoryException: logger.exception("Error adding host ", extra={"host": host_data}) metrics.add_host_failure.labels("InventoryException", host_data.get("reporter", "null")).inc() raise except Exception: logger.exception("Error while adding host", extra={"host": host_data}) metrics.add_host_failure.labels("Exception", host_data.get("reporter", "null")).inc() raise
def rbac_url(): return inventory_config().rbac_endpoint + ROUTE
def staleness_filter(staleness): config = inventory_config() return staleness_to_conditions(config, staleness, _stale_timestamp_filter)
# print(query_results) if args.id: host_id_list = [args.id] print("looking up host using id") query_results = Host.query.filter(Host.id.in_(host_id_list)).all() elif args.hostname: print("looking up host using display_name, fqdn") query_results = Host.query.filter( Host.display_name.comparator.contains(args.hostname) | Host.canonical_facts["fqdn"].astext.contains(args.hostname)).all( ) elif args.insights_id: print("looking up host using insights_id") query_results = Host.query.filter( Host.canonical_facts.comparator.contains( {"insights_id": args.insights_id})).all() elif args.account_number: query_results = Host.query.filter( Host.account == args.account_number).all() staleness_timestamps = Timestamps.from_config(inventory_config()) json_host_list = [ serialize_host(host, staleness_timestamps) for host in query_results ] if args.no_pp: print(json_host_list) else: pp = pprint.PrettyPrinter(indent=4) pp.pprint(json_host_list)
def staleness_timestamps(): return Timestamps.from_config(inventory_config())
def add_host_list(body): if not inventory_config().rest_post_enabled: return flask_json_response( { "detail": "The method is not allowed for the requested URL.", "status": 405, "title": "Method Not Allowed", "type": "about:blank", }, status=405, ) reporter = None response_host_list = [] number_of_errors = 0 payload_tracker = get_payload_tracker(account=current_identity.account_number, request_id=threadctx.request_id) with PayloadTrackerContext( payload_tracker, received_status_message="add host operation", current_operation="add host" ): for host in body: try: with PayloadTrackerProcessingContext( payload_tracker, processing_status_message="adding/updating host", current_operation="adding/updating host", ) as payload_tracker_processing_ctx: if host.get("tags"): tags_ignored_from_http_count.inc() logger.info("Tags from an HTTP request were ignored") input_host = deserialize_host_http(host) output_host, host_id, _, add_result = _add_host(input_host) status_code = _convert_host_results_to_http_status(add_result) response_host_list.append({"status": status_code, "host": output_host}) payload_tracker_processing_ctx.inventory_id = host_id reporter = host.get("reporter") except ValidationException as e: number_of_errors += 1 logger.exception("Input validation error while adding host", extra={"host": host}) response_host_list.append({**e.to_json(), "title": "Bad Request", "host": host}) except InventoryException as e: number_of_errors += 1 logger.exception("Error adding host", extra={"host": host}) response_host_list.append({**e.to_json(), "host": host}) except Exception: number_of_errors += 1 logger.exception("Error adding host", extra={"host": host}) response_host_list.append( { "status": 500, "title": "Error", "type": "unknown", "detail": "Could not complete operation", "host": host, } ) rest_post_request_count.labels(reporter=reporter).inc() response = {"total": len(response_host_list), "errors": number_of_errors, "data": response_host_list} return flask_json_response(response, status=207)