async def get_sid_from_query(auth_header, search=None, input_tokens=None, owner=None, app_name=None, async_splunk_client=None, sid=None): """ This will create a search job based of a regular search query :param auth_header: :param search: :param input_tokens: :param owner: :param app_name: :param async_splunk_client: :param sid: Optional override for a search job id :return: """ # inject methods can handle the empty input_tokens search.query = inject_tokens_into_string(input_tokens, search.query) search.earliest_time, search.latest_time = inject_time_tokens( input_tokens, search.earliest_time, search.latest_time) params = get_search_job_request_params( query=search.query, earliest_time=search.earliest_time, latest_time=search.latest_time, sample_ratio=search.sample_ratio, exec_mode=EXEC_MODE_NORMAL, max_time='0', # max_time=0 will allow search to run until complete sid=sid) if not params: LOGGER.error("Failed to get search job params %s", search) raise SpacebridgeApiRequestError("Missing search query.", status_code=HTTPStatus.NOT_FOUND) response = await async_splunk_client.async_get_search_data_request( auth_header=auth_header, owner=owner, app_name=app_name, data=urllib.urlencode(params)) if response.code not in {HTTPStatus.OK, HTTPStatus.CREATED}: error = await response.text() LOGGER.error( "Failed to create search job status_code=%s, error=%s, %s", response.code, error, search) raise SpacebridgeApiRequestError(format_splunk_error( response.code, error), status_code=response.code) response_json = await response.json() sid = response_json.get("sid") LOGGER.info("Created search job with sid=%s", response_json) return sid
async def fetch_visualization_data(auth_header, owner, app_name, subscription_search, input_tokens, async_splunk_client, map_post_search=None): """ This method will loop until job is complete. After job is complete it will return visualization data :param auth_header: :param owner: :param app_name: :param subscription_search: :param input_tokens: :param async_splunk_client: :param map_post_search: :return: """ post_search = None # Add post_search if search is dependent (i.e. defines a base) sid = subscription_search.sid if subscription_search.base: post_search = inject_tokens_into_string(input_tokens, subscription_search.query) # If we have a base search and a post_search_map update then we must append the two in this order if map_post_search and post_search: post_search += " " + map_post_search elif map_post_search: post_search = map_post_search # when done get data results from search visualization_data = await fetch_search_job_results_visualization_data( auth_header=auth_header, owner=owner, app_name=app_name, search_id=sid, post_search=post_search, async_splunk_client=async_splunk_client) if not visualization_data: raise SpacebridgeApiRequestError( "Unable to get visualization data for search. {}".format( subscription_search), status_code=HTTPStatus.NOT_FOUND) return visualization_data
def fetch_dashboard_data(request_context, dashboard_visualization_id=None, visualization=None, input_tokens=None, async_splunk_client=None): """ Fetch DashboardData given visualization_id :param request_context: :param dashboard_visualization_id: :param visualization: :param input_tokens: :param async_splunk_client: :return: """ dashboard_id = dashboard_visualization_id.dashboard_id visualization_id = dashboard_visualization_id.visualization_id owner, app_name, _ = dashboard_helpers.parse_dashboard_id(dashboard_id) LOGGER.info( "Start fetch_dashboard_data dashboard_id={}, visualization_id={}". format(dashboard_id, visualization_id)) # inject methods can handle the empty input_tokens visualization.search.query = inject_tokens_into_string( input_tokens, visualization.search.query) visualization.search.earliest, visualization.search.latest = inject_time_tokens( input_tokens, visualization.search.earliest, visualization.search.latest) # perform export call to retrieve visualization data without search_id visualization_data = yield fetch_visualization_data( request_context=request_context, owner=owner, app_name=app_name, visualization=visualization, async_splunk_client=async_splunk_client) LOGGER.info( "Finished fetch_dashboard_data dashboard_id={}, visualization_id={}, visualization.search.query={}" .format(dashboard_id, visualization_id, visualization.search.query)) defer.returnValue( DashboardData(dashboard_visualization_id=dashboard_visualization_id, visualization_data=visualization_data))
async def process_single_subscription_update(request_context, search, visualization_id, server_subscription_update, async_splunk_client, map_post_search=None): """ An async processor which will create a subscription data event :param request_context: :param search: :param visualization_id: :param server_subscription_update: :param async_splunk_client: :param map_post_search: :return: """ user, app_name, dashboard_name = parse_dashboard_id(search.dashboard_id) # Add post_search if search is dependent (i.e. defines a base) post_search = None sid = search.sid if search.base: input_tokens = load_input_tokens(search.input_tokens) post_search = inject_tokens_into_string(input_tokens, search.query) LOGGER.debug("Search has base, using parent sid, search_key=%s, sid=%s, post_search=%s", search.key(), sid, post_search) if not post_search: post_search = map_post_search elif map_post_search: post_search += " " + map_post_search # Query the job status job_status = await get_search_job_content(auth_header=request_context.system_auth_header, owner=user, app_name=app_name, search_id=sid, async_splunk_client=async_splunk_client) # If no job_status we don't try to send this update if job_status is not None: # call api with sid visualization_data = await fetch_search_job_results_visualization_data( owner=user, app_name=app_name, search_id=sid, post_search=post_search, auth_header=request_context.system_auth_header, async_splunk_client=async_splunk_client) # populate update if data available, if no data is available it means job is still processing or error occurred # its okay if we miss this update as it should get processing in looping update if visualization_data: subscription_update = build_subscription_update(search=search, visualization_data=visualization_data, job_status=job_status) build_server_subscription_update(subscription_update, server_subscription_update) else: LOGGER.debug("No visualization data found, sid=%s, visualization_id=%s", sid, visualization_id) else: LOGGER.debug("No search job status found, sid=%s, visualization_id=%s", sid, visualization_id)
def process_single_subscription_update(request_context, search, visualization_id, server_subscription_update, async_splunk_client, guid_generator, map_post_search=None): """ An async processor which will create a subscription data event :param request_context: :param search: :param visualization_id: :param server_subscription_update: :param async_splunk_client: :param guid_generator: :return: """ user, app_name, dashboard_name = parse_dashboard_id(search.dashboard_id) # Add post_search if search is dependent (i.e. defines a base) post_search = None sid = search.sid if search.base: input_tokens = load_input_tokens(search.input_tokens) post_search = inject_tokens_into_string(input_tokens, search.query) LOGGER.debug( "Search has base, using parent sid, search_key=%s, sid=%s, post_search=%s", search.key(), sid, post_search) if not post_search: post_search = map_post_search elif map_post_search: post_search += " " + map_post_search # Query the job status job_status = yield get_search_job_content( auth_header=request_context.system_auth_header, owner=user, app_name=app_name, search_id=sid, async_splunk_client=async_splunk_client) # If no job_status we don't try to send this update if job_status: # call api with sid visualization_data = yield fetch_search_job_results_visualization_data( owner=user, app_name=app_name, search_id=sid, post_search=post_search, auth_header=request_context.system_auth_header, async_splunk_client=async_splunk_client) # populate update if data available, if no data is available it means job is still processing or error occurred # its okay if we miss this update as it should get processing in looping update if visualization_data: # Populate the server_subscription_update LOGGER.debug("Post processing sid=%s, visualization_id=%s", sid, visualization_id) dashboard_visualization_id = DashboardVisualizationId( dashboard_id=search.dashboard_id, visualization_id=visualization_id) # Set params server_subscription_update.updateId = guid_generator() visualization_data.set_protobuf( server_subscription_update.dashboardVisualizationEvent. visualizationData) # Update update with search job state, if job_status: server_subscription_update.dashboardVisualizationEvent.dispatchState = int( job_status.dispatch_state) server_subscription_update.dashboardVisualizationEvent.doneProgress = float( job_status.done_progress) # Update with dashboard_visualization_id dashboard_visualization_id.set_protobuf( server_subscription_update.dashboardVisualizationEvent. dashboardVisualizationId) else: LOGGER.debug( "No visualization data found, sid=%s, visualization_id=%s", sid, visualization_id) else: LOGGER.debug("No search job status found, sid=%s, visualization_id=%s", sid, visualization_id)