async def fetch_dashboard_list_json(request_context, offset=0, max_results=0, app_names=[], dashboard_ids=[], async_splunk_client=None): """ Fetch the dashboard list json Splunk api /data/ui/views :param request_context: :param offset: :param max_results: :param app_names: :param dashboard_ids: :param async_splunk_client: :return: """ search_str = generate_search_str(app_names, dashboard_ids) params = { 'output_mode': 'json', 'search': search_str, 'sort_dir': 'asc', 'sort_key': 'label', 'sort_mode': 'alpha', 'offset': offset, 'count': max_results } # Don't specify owner, defaults to '-' which will retrieve dashboards from all users visible for current_user response = await async_splunk_client.async_get_dashboard_list_request( auth_header=request_context.auth_header, params=params) LOGGER.info(f'fetch_dashboard_list_json response={response.code}') if response.code != HTTPStatus.OK: response_text = await response.text() raise SpacebridgeApiRequestError( f"Failed fetch_dashboard_list_json " f"response.code={response.code}, response.text={response_text}", status_code=response.code) response_json = await response.json() return response_json
def parse_udf_kvstore_resource(data_jsn, request_context=None): """ Parse the response from KV Store for stored resources :param data_jsn: :param request_context: :return: (String, Bytes) containing the mime-type of the resource and the raw bytes of the resource respectively """ data_uri = data_jsn["dataURI"] d = data_uri.split(",") data_meta = d[0] data_payload = d[1] mime, encoding = data_meta.split(";") if encoding != 'base64': raise SpacebridgeApiRequestError( "Unexpected data encoding type. Expected base64 but got {}, {}". format("base64", request_context)) resource_bytes = base64.b64decode(data_payload) return (mime, resource_bytes)
async def _create_subscription_credentials(request_context, auth, async_kvstore_client): # create subscription and return _key response = await async_kvstore_client.async_kvstore_post_or_update_request( owner=request_context.current_user, collection=SUBSCRIPTION_CREDENTIALS_COLLECTION_NAME, data=auth.to_json(), auth_header=request_context.auth_header, key_id=SUBSCRIPTION_CREDENTIAL_GLOBAL) if response.code == HTTPStatus.OK or response.code == HTTPStatus.CREATED: LOGGER.debug("Subscription Created. subscription_id=%s", auth.subscription_id) return auth else: error = await response.text() error_message = "Failed to create Subscription credentials. status_code={}, error={}".format( response.code, error) raise SpacebridgeApiRequestError(error_message, status_code=response.code)
def check_and_raise_error(response, request_context, request_type, valid_codes=None): """ Function checks for and raises an error if necessary from a deferred response @param response: deferred object response @param request_context: @param request_type: Type of request that was made (used in error logging) """ if valid_codes is None: valid_codes = [http.OK] if response.code not in valid_codes: error = yield response.text() error_message = "Request: {} failed. status_code={}, error={}, {}"\ .format(request_type, response.code, error, request_context) raise SpacebridgeApiRequestError(error_message, status_code=response.code)
def create_role(self, request_context, name, parent_roles=None, general_capabilities=None, object_capabilities=None): """ Creates a new Splunk role with the given settings. :param request_context: The RequestContext for the request. :param name: The name of the role to create. :param parent_roles: A list of role names the new role should inherit from. :param general_capabilities: A list of capability names this role should possess. :param object_capabilities: A list of ObjectCapability that this role should possess. :return: The name of the created role. This may differ from "name" since Splunk formats role names. Thus far I have only seen Splunk make names lowercase but there may be other formatting so do not assume calling name.lower() will result in the same name returned from here. """ user_can_edit_roles = yield self._user_can_edit_roles(request_context) if not user_can_edit_roles: raise SpacebridgeARPermissionError( 'User must have edit_roles or ar_edit_roles to update AR roles.' ) create_role_response = yield self._splunk_client.async_create_role( auth_header=request_context.auth_header, name=name, imported_roles=parent_roles, capabilities=general_capabilities) if create_role_response.code not in {http.OK, http.CREATED}: message = yield create_role_response.text() raise SpacebridgeApiRequestError( 'Failed to create role="{}" message="{}" status_code={}'. format(name, message, create_role_response.code), status_code=create_role_response.code) created_role_json = yield create_role_response.json() name = created_role_json[ENTRY][0][NAME] if object_capabilities: yield self._add_object_capabilities_to_role( request_context, name, object_capabilities) defer.returnValue(name)
async def _fetch_registered_apps(auth_header, async_splunk_client): """ Fetch registered companion apps from the services/ssg/registration/companion_apps endpoint. Returns dictionary of app id to app bundle (json of app registration info) """ response = await async_splunk_client.async_fetch_companion_apps(auth_header ) if response.code != HTTPStatus.OK: raise SpacebridgeApiRequestError( "Error fetching registered companion apps with code={}".format( response.code), status_code=response.code) jsn = await response.json() companion_apps = {} for app in jsn: companion_apps[app[constants.KEY]] = app return companion_apps
def delete_all_alerts_for_device(request_context, device_id, async_kvstore_client): """ Delete all alert ids for a given device from the alert_recipient_devices collection. This is so that the alert is not fetched for that particular device in the feature. The actual body of the alert still exists in KV Store :param request_context: :param device_id: [string] id of device for which to remove the alert ids :param alert_ids: [list[string]] list of alert ids to be deleted :param async_kvstore_client: :return: """ query = {"device_id": device_id} response = yield async_kvstore_client.async_kvstore_delete_request( collection=constants.ALERTS_RECIPIENT_DEVICES_COLLECTION_NAME, auth_header=request_context.auth_header, params={QUERY: json.dumps(query)}) if response.code != http.OK: message = yield response.text() raise SpacebridgeApiRequestError( "Call to delete all alert for user failed with message={}".format( message))
def _start_phantom_registration(treq_client, phantom_metadata, phantom_registration_request): uri = '{phantom_domain}/rest/device_profile'.format( phantom_domain=phantom_metadata.domain) payload = { 'auth_code': phantom_registration_request.auth_code, 'name': phantom_registration_request.device_name, } auth = (phantom_registration_request.phantom_creds.username, phantom_registration_request.phantom_creds.password) start_registration_response = yield treq_client.post( url=uri.encode('utf-8'), json=payload, auth=auth) if start_registration_response.code != http.OK: message = yield start_registration_response.text() raise SpacebridgeApiRequestError( 'Unable to start registration with Phantom message={} status_code={}' .format(message, start_registration_response.code), status_code=start_registration_response.code) start_registration_json = yield start_registration_response.json() defer.returnValue(start_registration_json[REGISTRATION_ID_FIELD])
def delete_roles_from_capabilities_table(self, request_context, role_names): if not role_names: return # Lookup which objects are referenced by the roles we're about to delete so we can add objects back to the # public pool if possible later on. documents_for_roles = yield self._get_documents_for_roles( request_context, role_names) capabilities_from_roles = [ ObjectCapability.from_kvstore_document(doc) for doc in documents_for_roles ] # Remove all entries in the capabilities KV store collection that reference the roles we'd like to delete. delete_roles_query = { QUERY: json.dumps(({ OR_OPERATOR: [{ ROLE: role } for role in role_names] } if len(role_names) > 1 else { ROLE: next(iter(role_names)) })) } delete_role_from_kvstore = yield self._kvstore_client.async_kvstore_delete_request( collection=AR_CAPABILITIES_COLLECTION, auth_header=request_context.auth_header, params=delete_roles_query) if delete_role_from_kvstore.code != http.OK: message = yield delete_role_from_kvstore.text() raise SpacebridgeApiRequestError( 'Failed to delete roles={} from kvstore message={} status_code={}' .format(role_names, message, delete_role_from_kvstore.code), status_code=delete_role_from_kvstore.code) # See if there are any other roles that reference the same set of IDs we retrieved earlier, and if not, add # no longer referenced IDs back to the public pools. yield self._restore_object_ids_to_public_pool(request_context, capabilities_from_roles)
def _load_many(auth_header, collection, params, async_kvstore_client, mapper_fn=_noop, owner=NOBODY, app=SPACEBRIDGE_APP_NAME): """ Queries a collection for many entries for the specified params. It will marshal the result into a format you specify using mapper_fn. The default behavior will return all items as python dict. :param auth_header: A splunk auth header used to authenticate the call :param collection: The name of the collection being queried :param params: JSON-ified query, see https://docs.splunk.com/Documentation/Splunk/8.0.2/RESTREF/RESTkvstore :param async_kvstore_client: An instance of the async kvstore client :param mapper_fn: A function that can convert a dict to something else. Default to returning the dict. :param owner: The owner of the collection, defaults to nobody :param app: The app the collection is part of. Default to cloud gateway. :return: A list of mapped entries from the collection. :raises: :class:`SpacebridgeApiRequestError` if the kvstore call failed """ # Get all Searches so no input params response = yield async_kvstore_client.async_kvstore_get_request( collection=collection, params=params, owner=owner, app=app, auth_header=auth_header) if response.code != http.OK: error = yield response.text() message = "_load_many failed. status_code={}, error={}".format( response.code, error) LOGGER.error(message) raise SpacebridgeApiRequestError(message, code=Error.ERROR_API_REQUEST) response_json = yield response.json() result_list = [mapper_fn(obj) for obj in response_json] defer.returnValue(result_list)
def delete_objects(self, request_context, object_ids): """ Removes the given object IDs from the AR capabilities table. :param request_context: The RequestContext of the request. :param object_ids: The object IDs to remove. """ if not object_ids: return delete_response = yield self._kvstore_client.async_kvstore_delete_request( collection=AR_CAPABILITIES_COLLECTION, auth_header=request_context.auth_header, params={QUERY: get_query_matching_field(OBJECT_ID, object_ids)}) if delete_response.code != http.OK: message = yield delete_response.text() raise SpacebridgeApiRequestError( 'Failed to remove capabilities documents for objects={} message={} status_code={}' .format(object_ids, message, delete_response.code), status_code=delete_response.code) LOGGER.debug('Removed object_ids=%s from the capabilities table.', object_ids)
async def process_post_search(request_context, validate_map_update=None, construct_post_search_string=None, dashboard_visualization_map=None, subscription_id=None, server_subscription_response=None, async_kvstore_client=None): if validate_map_update: validate_error_list = validate_map_update(dashboard_visualization_map) if validate_error_list: error_message = ','.join(validate_error_list) raise SpacebridgeApiRequestError( error_message, status_code=HTTPStatus.BAD_REQUEST) post_search = construct_post_search_string(dashboard_visualization_map) subscription_update = SearchUpdate(post_search=post_search, key=subscription_id) await process_subscription_update(request_context, subscription_update, server_subscription_response, async_kvstore_client)
def set_phantom_metadata(request_context, kvstore_client, phantom_metadata): """ Associates a Phantom instance for use with AR with this Splunk instance. :param request_context: The RequestContext of the request. :param kvstore_client: An AsyncKvStoreClient for storing Phantom metadata :param phantom_metadata: A PhantomMetadata instance containing the information about Phantom to set. """ existing_metadata = None try: existing_metadata = yield get_phantom_metadata(request_context, kvstore_client) if phantom_metadata == existing_metadata: LOGGER.info( 'New Phantom metadata is the same as the existing one.') return except NoRegisteredPhantomInstanceException: LOGGER.debug( 'No existing phantom metadata found. No need to check whether the hostname has changed.' ) if existing_metadata: response = yield kvstore_client.async_kvstore_post_request( collection=PHANTOM_METADATA_COLLECTION, auth_header=request_context.auth_header, key_id=phantom_metadata.key, data=phantom_metadata.to_json()) else: response = yield kvstore_client.async_kvstore_post_request( collection=PHANTOM_METADATA_COLLECTION, auth_header=request_context.auth_header, data=phantom_metadata.to_json()) if response.code not in {http.OK, http.CREATED}: message = yield response.text() raise SpacebridgeApiRequestError( 'Failed to set Phantom metadata KV store entry message="{}" status_code={}' .format(message, response.code), status_code=response.code)
def _add_object_capabilities_to_role(self, request_context, role_name, capabilities_to_add): if not capabilities_to_add: return # Write the given object capabilities to the AR capabilities table and associate them with the given role effected_ids = yield self._kvstore_client.async_batch_save_request( collection=AR_CAPABILITIES_COLLECTION, auth_header=request_context.auth_header, entries=[ oc.to_kvstore_capabilities_entry(role_name) for oc in capabilities_to_add ]) LOGGER.debug('Wrote capabilities=%s for role="%s"', effected_ids, role_name) # Remove the effected objects from either the read or write public pool. keys_to_delete = [] for oc in capabilities_to_add: if oc.access_level == LEVEL_READ: keys_to_delete.append( make_ar_capabilities_entry_key(AR_PUBLIC_READ_ROLE, oc.object_type, oc.object_id)) keys_to_delete.append( make_ar_capabilities_entry_key(AR_PUBLIC_WRITE_ROLE, oc.object_type, oc.object_id)) delete_response = yield self._kvstore_client.async_kvstore_delete_request( collection=AR_CAPABILITIES_COLLECTION, auth_header=request_context.auth_header, params={QUERY: get_query_matching_keys(keys_to_delete)}) if delete_response.code != http.OK: message = yield delete_response.text() raise SpacebridgeApiRequestError( 'Failed to remove the following keys from the AR capabilities table for role="{}" keys={} message={} ' 'status_code={}'.format(role_name, keys_to_delete, message, delete_response.code), status_code=delete_response.code)
async def fetch_dashboard_meta(request_context, dashboard_id=None, async_kvstore_client=None): """ Helper to fetch dashboard_meta :param request_context: :param dashboard_id: :param async_kvstore_client: :return: """ # Only set the key_id if a dashboard_id is provided key_id = urllib.quote_plus(dashboard_id) if dashboard_id else None response = await async_kvstore_client.async_kvstore_get_request( collection=DASHBOARD_META_COLLECTION_NAME, auth_header=request_context.auth_header, owner=request_context.current_user, key_id=key_id) if response.code == HTTPStatus.OK: response_json = await response.json() if response_json and dashboard_id: dashboard_meta = UserDashboardMeta.from_json(response_json) return dashboard_meta # If the response is the whole meta collection, return a list of dashboard metas elif response_json and dashboard_id is None: dashboard_meta = {} for meta in response_json: meta_obj = UserDashboardMeta.from_json(meta) dashboard_meta[meta_obj.dashboard_id()] = meta_obj return dashboard_meta elif response.code != HTTPStatus.NOT_FOUND: error = await response.text() error_message = "Failed to fetch Dashboard Meta. status_code={}, error={}".format( response.code, error) raise SpacebridgeApiRequestError(message=error_message, status_code=response.code) # Return None no data if meta isn't available return None
def process_group_delete_request(request_context, client_single_request, server_single_response, async_kvstore_client): """ This method processes and group delete requests :param request_context: :param client_single_request: :param server_single_response: :param async_kvstore_client: :return: """ # request params group_ids = client_single_request.groupDeleteRequest.groupIds is_public = client_single_request.groupDeleteRequest.isPublic owner = constants.NOBODY if is_public else request_context.current_user query = { constants.OR_OPERATOR: [{ constants.KEY: group_id } for group_id in group_ids] } response = yield async_kvstore_client.async_kvstore_delete_request( collection=constants.GROUPS_COLLECTION_NAME, owner=owner, auth_header=request_context.auth_header, params={constants.QUERY: json.dumps(query)}) if response.code != http.OK: message = yield response.text() raise SpacebridgeApiRequestError( "Call to GROUPS DELETE failed with groupIds={}, code={}, message={}" .format(group_ids, response.code, message)) # populate delete response server_single_response.groupDeleteResponse.SetInParent() LOGGER.debug( "Call to GROUPS DELETE succeeded, groupIds={}".format(group_ids))
async def process_dashboard_visualization_map_update( request_context, validate_map_update=None, construct_post_search_string=None, dashboard_visualization_map=None, client_subscription_message=None, server_subscription_response=None, async_kvstore_client=None): # Retrieve Params client_subscription_update = client_subscription_message.clientSubscriptionUpdate subscription_id = client_subscription_update.subscriptionId # Validate subscription_id LOGGER.debug('Start process_subscription_update subscription_id=%s', subscription_id) # See if subscription_id exists in KVStore get_response = await async_kvstore_client.async_kvstore_get_request( collection=SUBSCRIPTIONS_COLLECTION_NAME, key_id=subscription_id, owner=NOBODY, auth_header=request_context.auth_header) if get_response.code != HTTPStatus.OK: error = await get_response.text() error_message = "Update failed. Subscription ID Not Found! status_code={}, error={},subscription_id={}" \ .format(get_response.code, error, subscription_id) raise SpacebridgeApiRequestError(error_message, status_code=get_response.code) await process_post_search( request_context=request_context, validate_map_update=validate_map_update, construct_post_search_string=construct_post_search_string, dashboard_visualization_map=dashboard_visualization_map, subscription_id=subscription_id, server_subscription_response=server_subscription_response, async_kvstore_client=async_kvstore_client)
def get_phantom_metadata(request_context, kvstore_client): """ Retrieves data about the Phantom instance associated with this Splunk instance for use with AR. :param request_context: The RequestContext of the request. :param kvstore_client: An AsyncKvStoreClient for looking up data from KV store. :return: a PhantomMetadata instance :raises: NoRegisteredPhantomInstanceException if there is no Phantom instance associated with this Splunk. """ phantom_metadata_response = yield kvstore_client.async_kvstore_get_request( collection=PHANTOM_METADATA_COLLECTION, auth_header=request_context.auth_header, key_id=PhantomMetadata.key) if phantom_metadata_response.code == http.NOT_FOUND: raise NoRegisteredPhantomInstanceException() if phantom_metadata_response.code != http.OK: message = yield phantom_metadata_response.text() raise SpacebridgeApiRequestError( 'Unable to fetch Phantom registration information message="{}" status_code={}' .format(message, phantom_metadata_response.code), status_code=phantom_metadata_response.code) phantom_metadata_json = yield phantom_metadata_response.json() defer.returnValue(PhantomMetadata.from_json(phantom_metadata_json))
def fetch_dashboard_app_list(request_context, async_kvstore_client=None, async_splunk_client=None): """ Helper to fetch dashboard_app_list from user_meta :param request_context: :param async_kvstore_client: :return: """ if async_kvstore_client and async_splunk_client: response = yield async_kvstore_client.async_kvstore_get_request( collection=USER_META_COLLECTION_NAME, auth_header=request_context.auth_header, owner=request_context.current_user, key_id=DASHBOARD_APP_LIST) if response.code == http.OK: response_json = yield response.json() if response_json: app_names = json.loads(response_json[APP_NAMES]) # make sure apps are not disabled/invisible (fetch app names filters these out) valid_app_names = yield fetch_app_names( request_context=request_context, async_splunk_client=async_splunk_client) valid_app_names = [app.app_name for app in valid_app_names] defer.returnValue( [app for app in app_names if app in valid_app_names]) elif response.code != http.NOT_FOUND: error = yield response.text() error_message = "Unable to find dashboard_app_list in user_meta collection. status_code={}, error={}"\ .format(response.code, error) raise SpacebridgeApiRequestError(error_message) # Return None no data if meta isn't available defer.returnValue(None)
async def fetch_search(auth_header, search_key=None, async_kvstore_client=None, none_if_not_found=False): """ Fetch a object from kvstore collection [searches] with search _key, searches stored in nobody namespace :param auth_header: :param search_key: :param async_kvstore_client: :param none_if_not_found: :return: """ response = await async_kvstore_client.async_kvstore_get_request( collection=SEARCHES_COLLECTION_NAME, key_id=search_key, auth_header=auth_header) if response.code == HTTPStatus.OK: response_json = await response.json() if not response_json: # No Search found with search_key return None # Create SubscriptionSearch from json search = SubscriptionSearch.from_json(response_json) return search # Special case here to return None if not found if response.code == HTTPStatus.NOT_FOUND and none_if_not_found: return None # Return error in case of unsuccessful response error = await response.text() error_message = "Failed to fetch searches. status_code=%s, error=%s, search_key=%s, auth_header={}".format( response.code, error, search_key, auth_header) raise SpacebridgeApiRequestError(error_message, status_code=response.code)
def _get_documents_for_object_ids(self, request_context, object_ids): if not object_ids: defer.returnValue([]) capabilities_for_ids = yield self._kvstore_client.async_kvstore_get_request( collection=AR_CAPABILITIES_COLLECTION, auth_header=request_context.auth_header, params={ QUERY: json.dumps({ OR_OPERATOR: [{ OBJECT_ID: object_id } for object_id in object_ids] }) }) if capabilities_for_ids.code != http.OK: message = yield capabilities_for_ids.text() raise SpacebridgeApiRequestError( 'Failed to lookup object_ids={} to add back to the public access pool ' 'message="{}" status_code={}'.format( object_ids, message, capabilities_for_ids.code), status_code=capabilities_for_ids.code) capabilities_json = yield capabilities_for_ids.json() defer.returnValue(capabilities_json)
def validate_dashboard_search(request_context, dashboard_id=None, type_id=None, search_type=SearchType.VISUALIZATION, input_tokens=None, async_kvstore_client=None, async_splunk_client=None): """ Validation method to validate a dashboard_id and visualization_id. Will except a SpacebridgeApiRequestError if issues are detected and return a dashboard_description if valid :param request_context: :param dashboard_id: :param type_id: :param search_type: :param input_tokens: :param async_kvstore_client: :param async_splunk_client: :return: """ # Validate params if not dashboard_id or not type_id: error_message = "Invalid Request Params dashboard_id={}, search_type_id={}, search_type={}" \ .format(dashboard_id, type_id, search_type) raise SpacebridgeApiRequestError(error_message) # fetch dashboard body dashboard_description = yield fetch_dashboard_description( request_context=request_context, dashboard_id=dashboard_id, async_splunk_client=async_splunk_client, async_kvstore_client=async_kvstore_client) search = None if search_type == SearchType.VISUALIZATION: visualization = dashboard_description.get_visualization(type_id) if not visualization: error_message = "Dashboard visualization not found. dashboard_id={}, visualization_id={}" \ .format(dashboard_id, type_id) raise SpacebridgeApiRequestError(error_message) search = visualization.search elif search_type == SearchType.INPUT: input_token = dashboard_description.get_input_token_by_query_id(type_id) if not input_token: error_message = "Input Search not found. dashboard_id={}, query_id={}".format(dashboard_id, type_id) raise SpacebridgeApiRequestError(error_message) search = input_token.input_type.dynamic_options.search elif search_type == SearchType.DATA_SOURCE: datasources = [d for d in dashboard_description.definition.udf_data_sources if d.name == type_id] if len(datasources) != 1: raise SpacebridgeApiRequestError( "Unexpected number of matching datasources in dashboard. Expected 1 but found {}" .format(len(datasources))) # validate depends if search and not search.are_render_tokens_defined(input_tokens): error_message = "Search is waiting for input. depends={}, rejects={}, dashboard_id={}, type_id={}, search_type={}" \ .format(search.depends, search.rejects, dashboard_id, type_id, search_type) raise SpacebridgeApiRequestError(error_message) defer.returnValue(dashboard_description)
async def lazy_load_subscription_search(request_context, input_tokens, search_key, search_type, search_type_id, search_defn, async_kvstore_client, async_splunk_client, shard_id, spawn_search_job, dashboard_defn=None, dashboard_id=None, default_input_tokens=None, visualization_type=None, trellis_enabled=False, trellis_split_by=None): """ :param trellis_enabled: :param request_context: :param input_tokens: :param search_key: :param search_type: :param search_type_id: :param search_defn: :param async_kvstore_client: :param async_splunk_client: :param shard_id: :param spawn_search_job: :param dashboard_defn: :param dashboard_id: :param default_input_tokens: :param visualization_type: :param trellis_split_by: :return: """ if default_input_tokens is None: default_input_tokens = {} if type != SearchType.SAVED_SEARCH: # Set Default input_token values set_default_token_values(input_tokens, default_input_tokens) parent_id = None search_query = search_defn.query sid = None if search_defn.base: parent_search_defn = dashboard_defn.find_base_search(search_defn.base) parent_search_key = generate_search_hash( dashboard_id, parent_search_defn.id, input_tokens, request_context.current_user, refresh_interval=refresh_to_seconds(parent_search_defn.refresh)) LOGGER.debug("Base search exists, base=%s", parent_search_defn) parent = await lazy_load_subscription_search( request_context=request_context, input_tokens=input_tokens, search_key=parent_search_key, search_type=SearchType.ROOT.value, search_type_id=parent_search_key, search_defn=parent_search_defn, async_kvstore_client=async_kvstore_client, async_splunk_client=async_splunk_client, shard_id=request_context.shard_id, spawn_search_job=spawn_search_job, dashboard_defn=parent_search_defn, dashboard_id=dashboard_id, default_input_tokens=default_input_tokens, visualization_type=visualization_type, trellis_enabled=trellis_enabled, trellis_split_by=trellis_split_by) sid = parent.sid parent_id = parent.key() # i.e. no parent search if not sid: sid = search_key subscription_search = build_subscription_search( request_context, dashboard_id, search_defn, search_query, input_tokens, parent_id, search_key, search_type, search_type_id, shard_id, sid, visualization_type=visualization_type, trellis_enabled=trellis_enabled, trellis_split_by=trellis_split_by) params = {} if visualization_type is not None and VisualizationType( visualization_type ) == VisualizationType.DASHBOARD_VISUALIZATION_EVENT: params[constants.COUNT] = '200' job_status = await get_search_job_content(request_context.auth_header, subscription_search.owner, SPACEBRIDGE_APP_NAME, subscription_search.sid, async_splunk_client, params=params) if not job_status: LOGGER.debug("Start search job, sid=%s", sid) await spawn_search_job(subscription_search, input_tokens, sid) http_result = await create_search(request_context, subscription_search, async_kvstore_client) if http_result not in [HTTPStatus.CREATED, HTTPStatus.CONFLICT]: raise SpacebridgeApiRequestError( message='Failed to create subscription search', status_code=http_result) return subscription_search
async def update_search_job_status(auth_header, owner, app_name, search, sid, async_splunk_client, async_kvstore_client): """ Helper method to update search job status on a search collection object :param auth_header: :param owner: :param app_name: :param search: :param sid: :param async_splunk_client: :param async_kvstore_client: :return: """ # Loops and calls itself until job has results loop_response = await deferred_loop( poll_interval_seconds=DEFAULT_JOB_RESULTS_POLL_INTERVAL, timeout_seconds=DEFAULT_JOB_RESULTS_TIMEOUT, deferred_function=get_running_search_job_content, # params to deferred_function auth_header=auth_header, owner=owner, app_name=app_name, sid=sid, async_splunk_client=async_splunk_client) # search_content check if None if loop_response is None or loop_response.response is None: await delete_search_job(auth_header=auth_header, owner=owner, app_name=app_name, sid=sid, async_splunk_client=async_splunk_client) raise SpacebridgeApiRequestError( "Search job timed out processing. sid=%s" % sid, status_code=HTTPStatus.REQUEST_TIMEOUT) # set valid search_job_content from loop_response search_job_content = loop_response.response LOGGER.info("Search Job Processed: %s", search_job_content) # Check if search job Failed if search_job_content.is_failed(): error_message = search_job_content.get_first_error_message( 'Search job Failed. sid=%s' % sid) raise SpacebridgeApiRequestError( error_message, status_code=HTTPStatus.FAILED_DEPENDENCY) if search_job_content.is_done: # only update the next_update_time if the refresh_interval_seconds has a value search.next_update_time = get_expiration_timestamp_str(ttl_seconds=search.refresh_interval_seconds) \ if search.refresh_interval_seconds else "" # update metadata on search search.sid = sid search.dispatch_state = search_job_content.dispatch_state search.done_progress = search_job_content.done_progress # Update Search Collection response = await async_kvstore_client.async_kvstore_post_request( collection=SEARCHES_COLLECTION_NAME, data=jsonpickle.encode( search, unpicklable=False), # Don't write py/object field key_id=search.key(), auth_header=auth_header) if response.code != HTTPStatus.OK: error = await response.text() LOGGER.error( "Unable to update sid on kvstore search. status_code=%s, error=%s, %s", response.code, error, search) raise SpacebridgeApiRequestError(format_splunk_error( response.code, error), status_code=response.code) return search
def fetch_subscriptions(auth_header, async_kvstore_client, user_list=None, subscription_type=None, subscription_id=None, device_ids=None): """ Fetch drone mode subscription objects from kvstore collection [subscription] # It specifically does the following: # 1. Returns all drone mode subscriptions # 2. Returns all drone mode subcriptions of a certain type (ipad, tv) # 3. Returns all subscriptions given a list of ids :param auth_header: auth header used to authenticate to kvstore :param owner: collection owner :param async_kvstore_client: kvstore client used to make request :param subscription_type: Type of subscription to fetch (ipad, tv) :param subscription_id: subscription_id to fetch :param device_ids: device ids to fetch (must also provide subscription type) :return: """ if subscription_type and subscription_id: raise SpacebridgeApiRequestError( 'You can only provide one of subscription type ' + 'or subscription id to fetch subscriptions.', status_code=http.BAD_REQUEST) if device_ids and not subscription_type: raise SpacebridgeApiRequestError( 'You must provide a subscription type to filter by ' + 'device ids.', status_code=http.BAD_REQUEST) if device_ids and subscription_type: params = { constants.QUERY: json.dumps({ constants.AND_OPERATOR: [ build_containedin_clause(constants.USER, user_list), build_containedin_clause(constants.DEVICE_ID, device_ids), { constants.SUBSCRIPTION_TYPE: subscription_type } ] }) } elif subscription_type: params = { constants.QUERY: json.dumps({constants.SUBSCRIPTION_TYPE: subscription_type}) } elif user_list: params = { constants.QUERY: json.dumps({ constants.AND_OPERATOR: [ build_containedin_clause(constants.USER, user_list), build_containedin_clause( constants.SUBSCRIPTION_TYPE, {constants.DRONE_MODE_IPAD, constants.DRONE_MODE_TV}) ] }) } else: params = { constants.QUERY: json.dumps( build_containedin_clause( constants.SUBSCRIPTION_TYPE, {constants.DRONE_MODE_IPAD, constants.DRONE_MODE_TV})) } response = yield async_kvstore_client.async_kvstore_get_request( collection=constants.SUBSCRIPTIONS_COLLECTION_NAME, params=params, owner=constants.NOBODY, key_id=subscription_id, auth_header=auth_header) subscriptions = [] if response.code == http.OK: response_json = yield response.json() if isinstance(response_json, list): subscriptions = [ Subscription.from_json(subscription) for subscription in response_json ] else: if response_json: subscriptions.append(Subscription.from_json(response_json)) else: error = yield response.text() LOGGER.error("Unable to fetch_subscriptions. status_code=%s, error=%s", response.code, error) LOGGER.debug('subscriptions in fetch_subscriptions=%s', subscriptions) defer.returnValue(subscriptions)
def validate_devices(device_ids, request_context, async_kvstore_client): """ Function to determine if device ids are valid (subscribed to drone mode and in the registered_devices table) :param device_ids: the set of device ids to check :param request_context: request context used to make kvstore requests :param async_kvstore_client: async client used to make kvstore requests """ # Check to see if devices are in registered_devices_table tv_list = yield get_registered_tvs(request_context.auth_header, request_context.current_user, async_kvstore_client, device_ids=device_ids) tv_list_ids = { registered_tv[constants.DEVICE_ID] for registered_tv in tv_list } errors = [] LOGGER.debug('device_ids=%s, tv_list_ids=%s', device_ids, tv_list_ids) registered_set_difference = set() subscription_set_difference = set() if tv_list_ids != device_ids: registered_set_difference = device_ids - tv_list_ids errors.append( 'The tvs with ids={} are no longer registered to this instance'. format(list(registered_set_difference))) if not tv_list_ids: raise SpacebridgeApiRequestError(errors[0], status_code=http.NOT_FOUND) tv_subscriptions = yield fetch_subscriptions( request_context.auth_header, async_kvstore_client, user_list=[request_context.current_user], subscription_type=constants.DRONE_MODE_TV, device_ids=device_ids) # Check to see if subscriptions are active subscription_ids = { subscription.device_id for subscription in tv_subscriptions } LOGGER.debug('device_ids=%s, subscription_ids=%s', device_ids, subscription_ids) if subscription_ids != device_ids: subscription_set_difference = device_ids - subscription_ids errors.append( 'The tvs with ids={} are no longer subscribed to drone mode'. format(list(subscription_set_difference))) if not subscription_ids: raise SpacebridgeApiRequestError(' and '.join(errors), status_code=http.BAD_REQUEST) # if all tvs are invalid, but for different reasons if registered_set_difference.union( subscription_set_difference) == device_ids: raise SpacebridgeApiRequestError( 'The tvs with ids={} are all invalid: these ids={} are no longer registered to the instance and ' 'these={} are no longer subscribed to drone mode'.format( list(device_ids), list(registered_set_difference), list(subscription_set_difference)), status_code=http.BAD_REQUEST) defer.returnValue(errors)
async def fetch_visualization_data(request_context, owner=None, app_name=None, visualization=None, async_splunk_client=None): """ Perform async call to fetch visualization search data NOTE: This function doesn't support saved searches (i.e. 'ref' attribute) :param request_context: :param owner: :param app_name: :param visualization: :param async_splunk_client: :return: """ LOGGER.info( "Start fetch_visualization_data owner={}, app_name={}, visualization.id={}, visualization.search.query={}" .format(owner, app_name, visualization.id, visualization.search.query)) # validate search if visualization.search is None: raise SpacebridgeApiRequestError( "Search is Empty for visualization_id={}".format(visualization.id), status_code=HTTPStatus.NO_CONTENT) # populate search params search = visualization.search params = get_search_job_request_params( query=search.query, earliest_time=search.earliest, latest_time=search.latest, sample_ratio=search.sample_ratio, exec_mode=constants.EXEC_MODE_ONESHOT) if not params: raise SpacebridgeApiRequestError( "Search Query is Empty for visualization_id={}".format( visualization.id), status_code=HTTPStatus.BAD_REQUEST) response = await async_splunk_client.async_get_search_data_request( owner=owner, app_name=app_name, auth_header=request_context.auth_header, data=urllib.urlencode(params)) if response.code != HTTPStatus.OK: response_text = await response.text() raise SpacebridgeApiRequestError( "Failed to get search data search_query={}, response.code={}, response.text={}" .format(params['search'], response.code, response_text), status_code=response.code) response_json = await response.json() visualization_data = parse.to_visualization_data(response_json) # Log warning message if fields, columns are empty # If fields and columns are empty we should log messages if available if visualization_data.is_empty_data( ) and 'messages' in response_json and response_json['messages']: for message in response_json['messages']: msg_type = message['type'] if 'type' in message else "no_type" text = message['text'] if 'text' in message else "no_text" LOGGER.info( "Search Data Message search_query={}, type={}, text={}".format( params['search'], msg_type, text)) return visualization_data
async def fetch_dashboard_description(request_context, dashboard_id=None, show_refresh=True, async_splunk_client=None, async_kvstore_client=None): """ Method will make async http call to get single dashboard and return a DashboardDescription object :param request_context: :param dashboard_id: :param show_refresh: show refresh params, default True :param async_splunk_client: :param async_kvstore_client: :return: """ params = {'output_mode': 'json'} owner, app_name, dashboard_name = dashboard_helpers.parse_dashboard_id( dashboard_id) if not owner or owner == '-': owner = request_context.current_user LOGGER.info("No owner given in dashboard_id=%s taking owner=%s", dashboard_id, owner) response = await async_splunk_client.async_get_dashboard_request( owner=owner, app_name=app_name, auth_header=request_context.auth_header, params=params, dashboard_name=dashboard_name) LOGGER.info('fetch_dashboard_description response={}'.format( response.code)) if response.code != HTTPStatus.OK: response_text = await response.text() raise SpacebridgeApiRequestError( "Failed fetch_dashboard_description request dashboard_id={}, response.code={}, response.text={}" .format(dashboard_id, response.code, response_text), status_code=response.code) response_json = await response.json() entry_json_list = response_json.get('entry') if not entry_json_list: # log result in the event the dashboardId is not valid raise SpacebridgeApiRequestError( "No Entries found for dashboard_id={}".format(dashboard_id), status_code=HTTPStatus.NOT_FOUND) dashboard = await parse.to_dashboard_description( entry_json_list[0], request_context=request_context, async_splunk_client=async_splunk_client, show_refresh=show_refresh) if async_kvstore_client is not None: dashboard_meta = await fetch_dashboard_meta( request_context=request_context, dashboard_id=dashboard_id, async_kvstore_client=async_kvstore_client) if dashboard_meta: dashboard.is_favorite = dashboard_meta.is_favorite return dashboard
def update_asset(request_context, asset_data, kvstore, permissions): """ Updates a single already existing asset. :param request_context: the RequestContext of the caller :param asset_data: the asset to update :param kvstore: the AsyncKvStoreClient to query KV store with :param permissions: the AsyncArPermissionsClient to validate permissions with :return: the ID of the updated asset """ if not asset_data.asset_id: raise MissingAssetIDException( 'Must provide an asset ID when updating an asset.') existing_asset_response = yield kvstore.async_kvstore_get_request( collection=ASSETS_COLLECTION_NAME, auth_header=request_context.auth_header, key_id=asset_data.asset_id) if existing_asset_response.code != http.OK: message = yield existing_asset_response.text() raise SpacebridgeApiRequestError( 'Failed to load asset_id={} message={} status_code={}'.format( asset_data.asset_id, message, existing_asset_response.code), status_code=existing_asset_response.code) existing_asset = yield existing_asset_response.json() existing_asset = AssetData.from_json(existing_asset) # The asset group changed. Users must have asset_group_manage to do this. if asset_data.asset_group != existing_asset.asset_group: message = 'User username={} needs asset_group_manage to update the asset group for asset_id={}'.format( request_context.current_user, asset_data.asset_id) yield permissions.check_permission(request_context, Capabilities.ASSET_GROUP_MANAGE, message=message) if asset_data.asset_group: yield permissions.check_object_permission( request_context, Capabilities.ASSET_GROUP_WRITE, asset_data.asset_group, ) else: yield permissions.check_object_permission(request_context, Capabilities.ASSET_WRITE, asset_data.asset_id) update_response = yield kvstore.async_kvstore_post_request( collection=ASSETS_COLLECTION_NAME, auth_header=request_context.auth_header, key_id=asset_data.asset_id, data=asset_data.to_json()) if update_response.code == http.NOT_FOUND: raise NoSuchAssetException(asset_id=asset_data.asset_id) if update_response.code != http.OK: message = yield update_response.text() raise SpacebridgeApiRequestError( 'Failed to update asset_id={} with payload={} message={} status_code={}' .format(asset_data.asset_id, asset_data.to_json(), message, update_response.code), status_code=update_response.code) defer.returnValue(asset_data.asset_id)
def process_subscribe_drone_mode_ipad(request_context, client_subscription_message, server_subscription_response, async_client_factory): """ Process subscribe to drone mode ipad events for a device id :param request_context: Used to authenticate kvstore requests :param client_single_request: client request object protobuf :param single_server_response: server response object protobuf :param async_client factory: factory class used to generate kvstore and spacebridge clients """ LOGGER.debug('start of subscribe drone mode ipad') async_kvstore_client = async_client_factory.kvstore_client() LOGGER.debug('request_context=%s', request_context) # base64 encode device id device_id = request_context.device_id urlsafe_b64encoded_device_id = b64_to_urlsafe_b64(device_id) query = { constants.QUERY: json.dumps({ constants.DEVICE_ID: device_id, }), constants.LIMIT: 1 } # fetch device from registered_devices_table to get device name response = yield async_kvstore_client.async_kvstore_get_request( collection=constants.REGISTERED_DEVICES_COLLECTION_NAME, params=query, owner=request_context.current_user, auth_header=request_context.auth_header) yield check_and_raise_error(response, request_context, 'Fetch registered devices') device = yield response.json() if device: device = device[0] else: raise SpacebridgeApiRequestError('Invalid device id={}'.format(device_id), status_code=http.BAD_REQUEST) device_name = device[constants.DEVICE_NAME] # write to ipads collection ipad_json = {constants.DEVICE_NAME: device_name} response = yield async_kvstore_client.async_kvstore_post_or_update_request( constants.DRONE_MODE_IPADS_COLLECTION_NAME, json.dumps(ipad_json), request_context.system_auth_header, key_id=urlsafe_b64encoded_device_id, owner=request_context.current_user) # construct parameters for subscription creation / updating ttl_seconds = client_subscription_message.clientSubscribeRequest.ttlSeconds expiration_time = get_expiration_timestamp_str(ttl_seconds=ttl_seconds) now = get_current_timestamp_str() params = { constants.TTL_SECONDS: str(ttl_seconds), constants.DEVICE_ID: device_id, constants.LAST_UPDATE_TIME: now, constants.EXPIRED_TIME: expiration_time, constants.USER: request_context.current_user, constants.SUBSCRIPTION_TYPE: constants.DRONE_MODE_IPAD } # delete existing subscriptions query = { constants.QUERY: json.dumps({ constants.DEVICE_ID: device_id, constants.USER: request_context.current_user, constants.SUBSCRIPTION_TYPE: constants.DRONE_MODE_IPAD }) } delete_response = yield async_kvstore_client.async_kvstore_delete_request( collection=constants.SUBSCRIPTIONS_COLLECTION_NAME, params=query, owner=constants.NOBODY, auth_header=request_context.auth_header) yield check_and_raise_error(delete_response, request_context, 'Delete existing ipad subscriptions') # create subscription subscription_id = yield write_drone_mode_subscription(request_context, params, async_client_factory) server_subscription_response.subscriptionId = subscription_id server_subscription_response.serverSubscribeResponse.droneModeiPadSubscribe.SetInParent() LOGGER.debug('Successfully wrote drone mode ipad subscription with id=%s', subscription_id)