def get_host_list( display_name=None, fqdn=None, hostname_or_id=None, insights_id=None, tags=None, page=1, per_page=100, order_by=None, order_how=None, staleness=None, ): total = 0 host_list = () bulk_query_source = get_bulk_query_source() get_host_list = GET_HOST_LIST_FUNCTIONS[bulk_query_source] try: host_list, total = get_host_list(display_name, fqdn, hostname_or_id, insights_id, tags, page, per_page, order_by, order_how, staleness) except ValueError as e: flask.abort(400, str(e)) json_data = build_paginated_host_list_response(total, page, per_page, host_list) return flask_json_response(json_data)
def delete_all_hosts(confirm_delete_all=None): if not confirm_delete_all: logger.error( "To delete all hosts, provide confirm_delete_all=true in the request." ) flask.abort( 400, "To delete all hosts, provide confirm_delete_all=true in the request." ) try: # get all hosts from the DB; bypasses xjoin-search, which limits the number hosts to 10 by default. ids_list = get_all_hosts() except ValueError as err: log_get_host_list_failed(logger) flask.abort(400, str(err)) except ConnectionError: logger.error("xjoin-search not accessible") flask.abort(503) if len(ids_list) == 0: flask.abort(status.HTTP_404_NOT_FOUND, "No hosts found for deletion.") delete_count = _delete_filtered_hosts(ids_list) json_data = {"hosts_found": len(ids_list), "hosts_deleted": delete_count} return flask_json_response(json_data, status.HTTP_202_ACCEPTED)
def get_tags(search=None, tags=None, order_by=None, order_how=None, page=None, per_page=None, staleness=None): if not xjoin_enabled(): flask.abort(503) limit, offset = pagination_params(page, per_page) variables = { "order_by": order_by, "order_how": order_how, "limit": limit, "offset": offset, "hostFilter": { # we're not indexing null timestamps in ES "OR": list(staleness_filter(staleness)) }, } if search: variables["filter"] = { # Escaped to prevent ReDoS "name": f".*{re.escape(url_quote(search, safe=''))}.*" } if tags: variables["hostFilter"]["AND"] = [{"tag": Tag().from_string(tag).data()} for tag in tags] response = graphql_query(TAGS_QUERY, variables) data = response["hostTags"] check_pagination(offset, data["meta"]["total"]) return flask_json_response(build_collection_response(data["data"], page, per_page, data["meta"]["total"]))
def get_host_by_id(host_id_list, page=1, per_page=100, order_by=None, order_how=None, staleness=None): query = _get_host_list_by_id_list(current_identity.account_number, host_id_list) # The staleness check is currently disabled when getting a single host by id # to ease onboarding (RHCLOUD-3562) # if staleness: # query = find_hosts_by_staleness(staleness, query) try: order_by = params_to_order_by(order_by, order_how) except ValueError as e: flask.abort(400, str(e)) else: query = query.order_by(*order_by) query_results = query.paginate(page, per_page, True) logger.debug("Found hosts: %s", query_results.items) json_data = build_paginated_host_list_response(query_results.total, page, per_page, query_results.items) return flask_json_response(json_data)
def get_host_system_profile_by_id(host_id_list, page=1, per_page=100, order_by=None, order_how=None, fields=None): if fields: if not get_bulk_query_source() == BulkQuerySource.xjoin: logger.error("xjoin-search not accessible") flask.abort(503) total, response_list = get_sparse_system_profile( host_id_list, page, per_page, order_by, order_how, fields) else: query = _get_host_list_by_id_list(host_id_list) try: order_by = params_to_order_by(order_by, order_how) except ValueError as e: flask.abort(400, str(e)) else: query = query.order_by(*order_by) query_results = query.paginate(page, per_page, True) total = query_results.total response_list = [ serialize_host_system_profile(host) for host in query_results.items ] json_output = build_collection_response(response_list, page, per_page, total) return flask_json_response(json_output)
def validate_schema(repo_fork="RedHatInsights", repo_branch="master", days=1, max_messages=10000): # Use the identity header to make sure the user is someone from our team. config = Config(RuntimeEnvironment.SERVICE) identity = get_current_identity() if not hasattr(identity, "user") or identity.user.get("username") not in config.sp_authorized_users: flask.abort(403, "This endpoint is restricted to HBI Admins.") consumer = KafkaConsumer( bootstrap_servers=config.bootstrap_servers, api_version=(0, 10, 1), value_deserializer=lambda m: m.decode(), **config.validator_kafka_consumer, ) try: response = validate_sp_for_branch( consumer, {config.host_ingress_topic, config.additional_validation_topic}, repo_fork, repo_branch, days, max_messages, ) consumer.close() return flask_json_response(response) except (ValueError, AttributeError) as e: consumer.close() flask.abort(400, str(e))
def delete_host_list( display_name=None, fqdn=None, hostname_or_id=None, insights_id=None, provider_id=None, provider_type=None, registered_with=None, staleness=None, tags=None, filter=None, ): if not any([ display_name, fqdn, hostname_or_id, insights_id, provider_id, provider_type, registered_with, staleness, tags, filter, ]): logger.error( "bulk-delete operation needs at least one input property to filter on." ) flask.abort( 400, "bulk-delete operation needs at least one input property to filter on." ) try: ids_list = get_host_ids_list_xjoin( display_name, fqdn, hostname_or_id, insights_id, provider_id, provider_type, registered_with, staleness, tags, filter, ) except ValueError as err: log_get_host_list_failed(logger) flask.abort(400, str(err)) except ConnectionError: logger.error("xjoin-search not accessible") flask.abort(503) if not len(ids_list): flask.abort(status.HTTP_404_NOT_FOUND, "No hosts found for deletion.") delete_count = _delete_filtered_hosts(ids_list) json_data = {"hosts_found": len(ids_list), "hosts_deleted": delete_count} return flask_json_response(json_data, status.HTTP_202_ACCEPTED)
def get_tags( search=None, tags=None, order_by=None, order_how=None, page=None, per_page=None, staleness=None, registered_with=None, filter=None, ): if not xjoin_enabled(): flask.abort(503) limit, offset = pagination_params(page, per_page) variables = { "order_by": order_by, "order_how": order_how, "limit": limit, "offset": offset, "hostFilter": { # we're not indexing null timestamps in ES "OR": list(staleness_filter(staleness)) }, } hostfilter_and_variables = () if search: variables["filter"] = { # Escaped so that the string literals are not interpreted as regex "search": {"regex": f".*{re.escape(search)}.*"} } if tags: hostfilter_and_variables = build_tag_query_dict_tuple(tags) if registered_with: variables["hostFilter"]["NOT"] = {"insights_id": {"eq": None}} if filter: if filter.get("system_profile"): if filter["system_profile"].get("sap_system"): hostfilter_and_variables += build_sap_system_filters(filter["system_profile"].get("sap_system")) if filter["system_profile"].get("sap_sids"): hostfilter_and_variables += build_sap_sids_filter(filter["system_profile"]["sap_sids"]) if hostfilter_and_variables != (): variables["hostFilter"]["AND"] = hostfilter_and_variables response = graphql_query(TAGS_QUERY, variables) data = response["hostTags"] check_pagination(offset, data["meta"]["total"]) return flask_json_response(build_collection_response(data["data"], page, per_page, data["meta"]["total"]))
def get_sap_system(tags=None, page=None, per_page=None, staleness=None, registered_with=None, filter=None): if not xjoin_enabled(): flask.abort(503) limit, offset = pagination_params(page, per_page) variables = { "hostFilter": { # we're not indexing null timestamps in ES "OR": list(staleness_filter(staleness)) } } hostfilter_and_variables = () if tags: hostfilter_and_variables = build_tag_query_dict_tuple(tags) if registered_with: variables["hostFilter"]["NOT"] = {"insights_id": {"eq": None}} if filter: if filter.get("system_profile"): if filter["system_profile"].get("sap_system"): hostfilter_and_variables += build_sap_system_filters( filter["system_profile"].get("sap_system")) if filter["system_profile"].get("sap_sids"): hostfilter_and_variables += build_sap_sids_filter( filter["system_profile"]["sap_sids"]) # TODO enable owner_id filtering after all hosts've been updated with "owner_id" # current_identity = get_current_identity() # if ( # current_identity.identity_type == "System" # and current_identity.auth_type != "classic-proxy" # and current_identity.system["cert_type"] == "system" # ): # hostfilter_and_variables += owner_id_filter() if hostfilter_and_variables != (): variables["hostFilter"]["AND"] = hostfilter_and_variables response = graphql_query(SAP_SYSTEM_QUERY, variables, log_get_sap_system_failed) data = response["hostSystemProfile"] check_pagination(offset, data["sap_system"]["meta"]["total"]) log_get_sap_system_succeeded(logger, data) return flask_json_response( build_collection_response(data["sap_system"]["data"], page, per_page, data["sap_system"]["meta"]["total"]))
def add_host_list(host_list): response_host_list = [] number_of_errors = 0 payload_tracker = get_payload_tracker( account=current_identity.account_number, payload_id=threadctx.request_id) with PayloadTrackerContext(payload_tracker, received_status_message="add host operation"): for host in host_list: try: with PayloadTrackerProcessingContext( payload_tracker, processing_status_message="adding/updating host" ) as payload_tracker_processing_ctx: input_host = deserialize_host(host) (output_host, add_result) = _add_host(input_host) status_code = _convert_host_results_to_http_status( add_result) response_host_list.append({ "status": status_code, "host": output_host }) payload_tracker_processing_ctx.inventory_id = output_host[ "id"] except ValidationException as e: number_of_errors += 1 logger.exception("Input validation error while adding host", extra={"host": host}) response_host_list.append({ **e.to_json(), "title": "Bad Request", "host": host }) except InventoryException as e: number_of_errors += 1 logger.exception("Error adding host", extra={"host": host}) response_host_list.append({**e.to_json(), "host": host}) except Exception: number_of_errors += 1 logger.exception("Error adding host", extra={"host": host}) response_host_list.append({ "status": 500, "title": "Error", "type": "unknown", "detail": "Could not complete operation", "host": host, }) response = { "total": len(response_host_list), "errors": number_of_errors, "data": response_host_list } return flask_json_response(response, status=207)
def get_sap_system(tags=None, page=None, per_page=None, staleness=None, registered_with=None, filter=None): if not xjoin_enabled(): logger.error("xjoin-search not enabled") flask.abort(503) limit, offset = pagination_params(page, per_page) variables = { "hostFilter": { # we're not indexing null timestamps in ES "OR": list(staleness_filter(staleness)) }, "limit": limit, "offset": offset, } hostfilter_and_variables = () if tags: hostfilter_and_variables = build_tag_query_dict_tuple(tags) if registered_with: hostfilter_and_variables += build_registered_with_filter( registered_with) if filter: for key in filter: if key == "system_profile": hostfilter_and_variables += build_system_profile_filter( filter["system_profile"]) else: raise ValidationException("filter key is invalid") current_identity = get_current_identity() if current_identity.identity_type == IdentityType.SYSTEM: hostfilter_and_variables += owner_id_filter() if hostfilter_and_variables != (): variables["hostFilter"]["AND"] = hostfilter_and_variables response = graphql_query(SAP_SYSTEM_QUERY, variables, log_get_sap_system_failed) data = response["hostSystemProfile"] check_pagination(offset, data["sap_system"]["meta"]["total"]) log_get_sap_system_succeeded(logger, data) return flask_json_response( build_collection_response(data["sap_system"]["data"], page, per_page, data["sap_system"]["meta"]["total"]))
def host_checkin(body): canonical_facts = deserialize_canonical_facts(body) existing_host = find_existing_host(current_identity.account_number, canonical_facts) if existing_host: existing_host._update_modified_date() db.session.commit() serialized_host = serialize_host(existing_host, staleness_timestamps(), EGRESS_HOST_FIELDS) _emit_patch_event(serialized_host, existing_host.id, existing_host.canonical_facts.get("insights_id")) return flask_json_response(serialized_host, 201) else: return flask_json_response( { "detail": "No hosts match the provided canonical facts.", "status": 404, "title": "Not Found", "type": "about:blank", }, status=404, )
def get_operating_system( tags=None, page: Optional[int] = None, per_page: Optional[int] = None, staleness: Optional[str] = None, registered_with: Optional[str] = None, filter=None, ): limit, offset = pagination_params(page, per_page) variables = { "hostFilter": { # we're not indexing null timestamps in ES "OR": list(staleness_filter(staleness)) }, "limit": limit, "offset": offset, } hostfilter_and_variables = () if tags: hostfilter_and_variables = build_tag_query_dict_tuple(tags) if registered_with: variables["hostFilter"]["NOT"] = {"insights_id": {"eq": None}} if filter: for key in filter: if key == "system_profile": hostfilter_and_variables += build_system_profile_filter( filter["system_profile"]) else: raise ValidationException("filter key is invalid") current_identity = get_current_identity() if current_identity.identity_type == IdentityType.SYSTEM: hostfilter_and_variables += owner_id_filter() if hostfilter_and_variables != (): variables["hostFilter"]["AND"] = hostfilter_and_variables response = graphql_query(OPERATING_SYSTEM_QUERY, variables, log_get_operating_system_failed) data = response["hostSystemProfile"] check_pagination(offset, data["operating_system"]["meta"]["total"]) log_get_operating_system_succeeded(logger, data) return flask_json_response( build_collection_response(data["operating_system"]["data"], page, per_page, data["operating_system"]["meta"]["total"]))
def get_sap_sids(search=None, tags=None, page=None, per_page=None, staleness=None, registered_with=None, filter=None): if not xjoin_enabled(): logger.error("xjoin-search not enabled") flask.abort(503) limit, offset = pagination_params(page, per_page) variables = { "hostFilter": { # we're not indexing null timestamps in ES "OR": list(staleness_filter(staleness)) }, "limit": limit, "offset": offset, } hostfilter_and_variables = () if tags: hostfilter_and_variables = build_tag_query_dict_tuple(tags) if registered_with: variables["hostFilter"]["NOT"] = {"insights_id": {"eq": None}} if search: variables["filter"] = { # Escaped so that the string literals are not interpreted as regex "search": {"regex": f".*{custom_escape(search)}.*"} } if filter: for key in filter: if key == "system_profile": hostfilter_and_variables += build_system_profile_filter(filter["system_profile"]) else: raise ValidationException("filter key is invalid") current_identity = get_current_identity() if current_identity.identity_type == IdentityType.SYSTEM and current_identity.auth_type != AuthType.CLASSIC: hostfilter_and_variables += owner_id_filter() if hostfilter_and_variables != (): variables["hostFilter"]["AND"] = hostfilter_and_variables response = graphql_query(SAP_SIDS_QUERY, variables, log_get_sap_sids_failed) data = response["hostSystemProfile"] check_pagination(offset, data["sap_sids"]["meta"]["total"]) log_get_sap_sids_succeeded(logger, data) return flask_json_response( build_collection_response(data["sap_sids"]["data"], page, per_page, data["sap_sids"]["meta"]["total"]) )
def get_host_list( display_name=None, fqdn=None, hostname_or_id=None, insights_id=None, provider_id=None, provider_type=None, tags=None, page=1, per_page=100, order_by=None, order_how=None, staleness=None, registered_with=None, filter=None, fields=None, ): total = 0 host_list = () bulk_query_source = get_bulk_query_source() get_host_list = GET_HOST_LIST_FUNCTIONS[bulk_query_source] try: host_list, total, additional_fields = get_host_list( display_name, fqdn, hostname_or_id, insights_id, provider_id, provider_type, tags, page, per_page, order_by, order_how, staleness, registered_with, filter, fields, ) except ValueError as e: log_get_host_list_failed(logger) flask.abort(400, str(e)) json_data = build_paginated_host_list_response(total, page, per_page, host_list, additional_fields) return flask_json_response(json_data)
def get_host_system_profile_by_id(host_id_list, page=1, per_page=100, order_by=None, order_how=None): query = _get_host_list_by_id_list(current_identity.account_number, host_id_list) try: order_by = params_to_order_by(order_by, order_how) except ValueError as e: flask.abort(400, str(e)) else: query = query.order_by(*order_by) query_results = query.paginate(page, per_page, True) response_list = [serialize_host_system_profile(host) for host in query_results.items] json_output = build_collection_response(response_list, page, per_page, query_results.total) return flask_json_response(json_output)
def get_host_by_id(host_id_list, page=1, per_page=100, order_by=None, order_how=None): query = _get_host_list_by_id_list(current_identity.account_number, host_id_list) try: order_by = params_to_order_by(order_by, order_how) except ValueError as e: flask.abort(400, str(e)) else: query = query.order_by(*order_by) query_results = query.paginate(page, per_page, True) logger.debug("Found hosts: %s", query_results.items) json_data = build_paginated_host_list_response(query_results.total, page, per_page, query_results.items) return flask_json_response(json_data)
def host_checkin(body): current_identity = get_current_identity() canonical_facts = deserialize_canonical_facts(body) existing_host = find_existing_host(current_identity, canonical_facts) if existing_host: existing_host._update_modified_date() db.session.commit() serialized_host = serialize_host(existing_host, staleness_timestamps(), EGRESS_HOST_FIELDS) _emit_patch_event(serialized_host, existing_host.id, existing_host.canonical_facts.get("insights_id")) return flask_json_response(serialized_host, 201) else: flask.abort(404, "No hosts match the provided canonical facts.")
def validate_schema(repo_fork="RedHatInsights", repo_branch="master", days=1): config = Config(RuntimeEnvironment.SERVICE) consumer = KafkaConsumer( group_id=config.host_ingress_consumer_group, bootstrap_servers=config.bootstrap_servers, api_version=(0, 10, 1), value_deserializer=lambda m: m.decode(), **config.kafka_consumer, ) try: response = validate_sp_for_branch(config, consumer, repo_fork, repo_branch, days) consumer.close() return flask_json_response(response) except (ValueError, AttributeError) as e: consumer.close() flask.abort(400, str(e))
def get_sap_system(tags=None, page=None, per_page=None, staleness=None, registered_with=None, filter=None): if not xjoin_enabled(): flask.abort(503) limit, offset = pagination_params(page, per_page) variables = { "hostFilter": { # we're not indexing null timestamps in ES "OR": list(staleness_filter(staleness)) } } hostfilter_and_variables = () if tags: hostfilter_and_variables = build_tag_query_dict_tuple(tags) if registered_with: variables["hostFilter"]["NOT"] = {"insights_id": {"eq": None}} if filter: if filter.get("system_profile"): if filter["system_profile"].get("sap_system"): hostfilter_and_variables += build_sap_system_filters( filter["system_profile"].get("sap_system")) if filter["system_profile"].get("sap_sids"): hostfilter_and_variables += build_sap_sids_filter( filter["system_profile"]["sap_sids"]) if hostfilter_and_variables != (): variables["hostFilter"]["AND"] = hostfilter_and_variables response = graphql_query(SAP_SYSTEM_QUERY, variables) data = response["hostSystemProfile"] check_pagination(offset, data["sap_system"]["meta"]["total"]) return flask_json_response( build_collection_response(data["sap_system"]["data"], page, per_page, data["sap_system"]["meta"]["total"]))
def get_host_by_id(host_id_list, page=1, per_page=100, order_by=None, order_how=None): query = _get_host_list_by_id_list(host_id_list) try: order_by = params_to_order_by(order_by, order_how) except ValueError as e: flask.abort(400, str(e)) else: query = query.order_by(*order_by) query_results = query.paginate(page, per_page, True) log_get_host_list_succeeded(logger, query_results.items) json_data = build_paginated_host_list_response(query_results.total, page, per_page, query_results.items) return flask_json_response(json_data)
def get_tags( search=None, tags=None, order_by=None, order_how=None, page=None, per_page=None, staleness=None, registered_with=None, filter=None, ): if not xjoin_enabled(): flask.abort(503) limit, offset = pagination_params(page, per_page) variables = { "order_by": order_by, "order_how": order_how, "limit": limit, "offset": offset, "hostFilter": { # we're not indexing null timestamps in ES "OR": list(staleness_filter(staleness)) }, } hostfilter_and_variables = () if search: variables["filter"] = { # Escaped so that the string literals are not interpreted as regex "search": { "regex": f".*{re.escape(search)}.*" } } if tags: hostfilter_and_variables = build_tag_query_dict_tuple(tags) if registered_with: variables["hostFilter"]["NOT"] = {"insights_id": {"eq": None}} if filter: if filter.get("system_profile"): if filter["system_profile"].get("sap_system"): hostfilter_and_variables += build_sap_system_filters( filter["system_profile"].get("sap_system")) if filter["system_profile"].get("sap_sids"): hostfilter_and_variables += build_sap_sids_filter( filter["system_profile"]["sap_sids"]) # TODO enable owner_id filtering after all hosts've been updated with "owner_id" # current_identity = get_current_identity() # if ( # current_identity.identity_type == "System" # and current_identity.auth_type != "classic-proxy" # and current_identity.system["cert_type"] == "system" # ): # hostfilter_and_variables += owner_id_filter() if hostfilter_and_variables != (): variables["hostFilter"]["AND"] = hostfilter_and_variables response = graphql_query(TAGS_QUERY, variables, log_get_tags_failed) data = response["hostTags"] check_pagination(offset, data["meta"]["total"]) log_get_tags_succeeded(logger, data) return flask_json_response( build_collection_response(data["data"], page, per_page, data["meta"]["total"]))
def add_host_list(body): if not inventory_config().rest_post_enabled: return flask_json_response( { "detail": "The method is not allowed for the requested URL.", "status": 405, "title": "Method Not Allowed", "type": "about:blank", }, status=405, ) reporter = None response_host_list = [] number_of_errors = 0 payload_tracker = get_payload_tracker(account=current_identity.account_number, request_id=threadctx.request_id) with PayloadTrackerContext( payload_tracker, received_status_message="add host operation", current_operation="add host" ): for host in body: try: with PayloadTrackerProcessingContext( payload_tracker, processing_status_message="adding/updating host", current_operation="adding/updating host", ) as payload_tracker_processing_ctx: if host.get("tags"): tags_ignored_from_http_count.inc() logger.info("Tags from an HTTP request were ignored") input_host = deserialize_host_http(host) output_host, host_id, _, add_result = _add_host(input_host) status_code = _convert_host_results_to_http_status(add_result) response_host_list.append({"status": status_code, "host": output_host}) payload_tracker_processing_ctx.inventory_id = host_id reporter = host.get("reporter") except ValidationException as e: number_of_errors += 1 logger.exception("Input validation error while adding host", extra={"host": host}) response_host_list.append({**e.to_json(), "title": "Bad Request", "host": host}) except InventoryException as e: number_of_errors += 1 logger.exception("Error adding host", extra={"host": host}) response_host_list.append({**e.to_json(), "host": host}) except Exception: number_of_errors += 1 logger.exception("Error adding host", extra={"host": host}) response_host_list.append( { "status": 500, "title": "Error", "type": "unknown", "detail": "Could not complete operation", "host": host, } ) rest_post_request_count.labels(reporter=reporter).inc() response = {"total": len(response_host_list), "errors": number_of_errors, "data": response_host_list} return flask_json_response(response, status=207)
def get_tags( search=None, tags=None, display_name=None, fqdn=None, hostname_or_id=None, insights_id=None, provider_id=None, provider_type=None, order_by=None, order_how=None, page=None, per_page=None, staleness=None, registered_with=None, filter=None, ): if not xjoin_enabled(): logger.error("xjoin-search not enabled") flask.abort(503) limit, offset = pagination_params(page, per_page) variables = { "order_by": order_by, "order_how": order_how, "limit": limit, "offset": offset, "hostFilter": { # we're not indexing null timestamps in ES "OR": list(staleness_filter(staleness)) }, } hostfilter_and_variables = query_filters( fqdn, display_name, hostname_or_id, insights_id, provider_id, provider_type, tags, None, registered_with, filter, ) if search: variables["filter"] = { # Escaped so that the string literals are not interpreted as regex "search": { "regex": f".*{custom_escape(search)}.*" } } current_identity = get_current_identity() if current_identity.identity_type == IdentityType.SYSTEM: hostfilter_and_variables += owner_id_filter() if hostfilter_and_variables != (): variables["hostFilter"]["AND"] = hostfilter_and_variables response = graphql_query(TAGS_QUERY, variables, log_get_tags_failed) data = response["hostTags"] check_pagination(offset, data["meta"]["total"]) log_get_tags_succeeded(logger, data) return flask_json_response( build_collection_response(data["data"], page, per_page, data["meta"]["total"]))
def _build_paginated_host_tags_response(total, page, per_page, tags_list): json_output = build_collection_response(tags_list, page, per_page, total) return flask_json_response(json_output)