async def fetch_dashboard_list_json(request_context,
offset=0,
max_results=0,
app_names=[],
dashboard_ids=[],
async_splunk_client=None):
"""
Fetch the dashboard list json Splunk api /data/ui/views
:param request_context:
:param offset:
:param max_results:
:param app_names:
:param dashboard_ids:
:param async_splunk_client:
:return:
"""
search_str = generate_search_str(app_names, dashboard_ids)
params = {
'output_mode': 'json',
'search': search_str,
'sort_dir': 'asc',
'sort_key': 'label',
'sort_mode': 'alpha',
'offset': offset,
'count': max_results
}
# Don't specify owner, defaults to '-' which will retrieve dashboards from all users visible for current_user
response = await async_splunk_client.async_get_dashboard_list_request(
auth_header=request_context.auth_header, params=params)
LOGGER.info(f'fetch_dashboard_list_json response={response.code}')
if response.code != HTTPStatus.OK:
response_text = await response.text()
raise SpacebridgeApiRequestError(
f"Failed fetch_dashboard_list_json "
f"response.code={response.code}, response.text={response_text}",
status_code=response.code)
response_json = await response.json()
return response_json
def parse_udf_kvstore_resource(data_jsn, request_context=None):
"""
Parse the response from KV Store for stored resources
:param data_jsn:
:param request_context:
:return: (String, Bytes) containing the mime-type of the resource and the raw bytes of the resource respectively
"""
data_uri = data_jsn["dataURI"]
d = data_uri.split(",")
data_meta = d[0]
data_payload = d[1]
mime, encoding = data_meta.split(";")
if encoding != 'base64':
raise SpacebridgeApiRequestError(
"Unexpected data encoding type. Expected base64 but got {}, {}".
format("base64", request_context))
resource_bytes = base64.b64decode(data_payload)
return (mime, resource_bytes)
async def _create_subscription_credentials(request_context, auth,
async_kvstore_client):
# create subscription and return _key
response = await async_kvstore_client.async_kvstore_post_or_update_request(
owner=request_context.current_user,
collection=SUBSCRIPTION_CREDENTIALS_COLLECTION_NAME,
data=auth.to_json(),
auth_header=request_context.auth_header,
key_id=SUBSCRIPTION_CREDENTIAL_GLOBAL)
if response.code == HTTPStatus.OK or response.code == HTTPStatus.CREATED:
LOGGER.debug("Subscription Created. subscription_id=%s",
auth.subscription_id)
return auth
else:
error = await response.text()
error_message = "Failed to create Subscription credentials. status_code={}, error={}".format(
response.code, error)
raise SpacebridgeApiRequestError(error_message,
status_code=response.code)
def check_and_raise_error(response,
request_context,
request_type,
valid_codes=None):
"""
Function checks for and raises an error if necessary
from a deferred response
@param response: deferred object response
@param request_context:
@param request_type: Type of request that was made (used in error logging)
"""
if valid_codes is None:
valid_codes = [http.OK]
if response.code not in valid_codes:
error = yield response.text()
error_message = "Request: {} failed. status_code={}, error={}, {}"\
.format(request_type, response.code, error, request_context)
raise SpacebridgeApiRequestError(error_message,
status_code=response.code)
def create_role(self,
request_context,
name,
parent_roles=None,
general_capabilities=None,
object_capabilities=None):
"""
Creates a new Splunk role with the given settings.
:param request_context: The RequestContext for the request.
:param name: The name of the role to create.
:param parent_roles: A list of role names the new role should inherit from.
:param general_capabilities: A list of capability names this role should possess.
:param object_capabilities: A list of ObjectCapability that this role should possess.
:return: The name of the created role. This may differ from "name" since Splunk formats role names. Thus far I
have only seen Splunk make names lowercase but there may be other formatting so do not assume calling
name.lower() will result in the same name returned from here.
"""
user_can_edit_roles = yield self._user_can_edit_roles(request_context)
if not user_can_edit_roles:
raise SpacebridgeARPermissionError(
'User must have edit_roles or ar_edit_roles to update AR roles.'
)
create_role_response = yield self._splunk_client.async_create_role(
auth_header=request_context.auth_header,
name=name,
imported_roles=parent_roles,
capabilities=general_capabilities)
if create_role_response.code not in {http.OK, http.CREATED}:
message = yield create_role_response.text()
raise SpacebridgeApiRequestError(
'Failed to create role="{}" message="{}" status_code={}'.
format(name, message, create_role_response.code),
status_code=create_role_response.code)
created_role_json = yield create_role_response.json()
name = created_role_json[ENTRY][0][NAME]
if object_capabilities:
yield self._add_object_capabilities_to_role(
request_context, name, object_capabilities)
defer.returnValue(name)
async def _fetch_registered_apps(auth_header, async_splunk_client):
"""
Fetch registered companion apps from the services/ssg/registration/companion_apps endpoint.
Returns dictionary of app id to app bundle (json of app registration info)
"""
response = await async_splunk_client.async_fetch_companion_apps(auth_header
)
if response.code != HTTPStatus.OK:
raise SpacebridgeApiRequestError(
"Error fetching registered companion apps with code={}".format(
response.code),
status_code=response.code)
jsn = await response.json()
companion_apps = {}
for app in jsn:
companion_apps[app[constants.KEY]] = app
return companion_apps
def delete_all_alerts_for_device(request_context, device_id,
async_kvstore_client):
"""
Delete all alert ids for a given device from the alert_recipient_devices collection. This is so that the
alert is not fetched for that particular device in the feature. The actual body of the alert still exists in KV Store
:param request_context:
:param device_id: [string] id of device for which to remove the alert ids
:param alert_ids: [list[string]] list of alert ids to be deleted
:param async_kvstore_client:
:return:
"""
query = {"device_id": device_id}
response = yield async_kvstore_client.async_kvstore_delete_request(
collection=constants.ALERTS_RECIPIENT_DEVICES_COLLECTION_NAME,
auth_header=request_context.auth_header,
params={QUERY: json.dumps(query)})
if response.code != http.OK:
message = yield response.text()
raise SpacebridgeApiRequestError(
"Call to delete all alert for user failed with message={}".format(
message))
def _start_phantom_registration(treq_client, phantom_metadata,
phantom_registration_request):
uri = '{phantom_domain}/rest/device_profile'.format(
phantom_domain=phantom_metadata.domain)
payload = {
'auth_code': phantom_registration_request.auth_code,
'name': phantom_registration_request.device_name,
}
auth = (phantom_registration_request.phantom_creds.username,
phantom_registration_request.phantom_creds.password)
start_registration_response = yield treq_client.post(
url=uri.encode('utf-8'), json=payload, auth=auth)
if start_registration_response.code != http.OK:
message = yield start_registration_response.text()
raise SpacebridgeApiRequestError(
'Unable to start registration with Phantom message={} status_code={}'
.format(message, start_registration_response.code),
status_code=start_registration_response.code)
start_registration_json = yield start_registration_response.json()
defer.returnValue(start_registration_json[REGISTRATION_ID_FIELD])
def delete_roles_from_capabilities_table(self, request_context,
role_names):
if not role_names:
return
# Lookup which objects are referenced by the roles we're about to delete so we can add objects back to the
# public pool if possible later on.
documents_for_roles = yield self._get_documents_for_roles(
request_context, role_names)
capabilities_from_roles = [
ObjectCapability.from_kvstore_document(doc)
for doc in documents_for_roles
]
# Remove all entries in the capabilities KV store collection that reference the roles we'd like to delete.
delete_roles_query = {
QUERY:
json.dumps(({
OR_OPERATOR: [{
ROLE: role
} for role in role_names]
} if len(role_names) > 1 else {
ROLE: next(iter(role_names))
}))
}
delete_role_from_kvstore = yield self._kvstore_client.async_kvstore_delete_request(
collection=AR_CAPABILITIES_COLLECTION,
auth_header=request_context.auth_header,
params=delete_roles_query)
if delete_role_from_kvstore.code != http.OK:
message = yield delete_role_from_kvstore.text()
raise SpacebridgeApiRequestError(
'Failed to delete roles={} from kvstore message={} status_code={}'
.format(role_names, message, delete_role_from_kvstore.code),
status_code=delete_role_from_kvstore.code)
# See if there are any other roles that reference the same set of IDs we retrieved earlier, and if not, add
# no longer referenced IDs back to the public pools.
yield self._restore_object_ids_to_public_pool(request_context,
capabilities_from_roles)
def _load_many(auth_header,
collection,
params,
async_kvstore_client,
mapper_fn=_noop,
owner=NOBODY,
app=SPACEBRIDGE_APP_NAME):
"""
Queries a collection for many entries for the specified params. It will
marshal the result into a format you specify using mapper_fn. The default
behavior will return all items as python dict.
:param auth_header: A splunk auth header used to authenticate the call
:param collection: The name of the collection being queried
:param params: JSON-ified query, see https://docs.splunk.com/Documentation/Splunk/8.0.2/RESTREF/RESTkvstore
:param async_kvstore_client: An instance of the async kvstore client
:param mapper_fn: A function that can convert a dict to something else. Default to returning the dict.
:param owner: The owner of the collection, defaults to nobody
:param app: The app the collection is part of. Default to cloud gateway.
:return: A list of mapped entries from the collection.
:raises: :class:`SpacebridgeApiRequestError` if the kvstore call failed
"""
# Get all Searches so no input params
response = yield async_kvstore_client.async_kvstore_get_request(
collection=collection,
params=params,
owner=owner,
app=app,
auth_header=auth_header)
if response.code != http.OK:
error = yield response.text()
message = "_load_many failed. status_code={}, error={}".format(
response.code, error)
LOGGER.error(message)
raise SpacebridgeApiRequestError(message, code=Error.ERROR_API_REQUEST)
response_json = yield response.json()
result_list = [mapper_fn(obj) for obj in response_json]
defer.returnValue(result_list)
def delete_objects(self, request_context, object_ids):
"""
Removes the given object IDs from the AR capabilities table.
:param request_context: The RequestContext of the request.
:param object_ids: The object IDs to remove.
"""
if not object_ids:
return
delete_response = yield self._kvstore_client.async_kvstore_delete_request(
collection=AR_CAPABILITIES_COLLECTION,
auth_header=request_context.auth_header,
params={QUERY: get_query_matching_field(OBJECT_ID, object_ids)})
if delete_response.code != http.OK:
message = yield delete_response.text()
raise SpacebridgeApiRequestError(
'Failed to remove capabilities documents for objects={} message={} status_code={}'
.format(object_ids, message, delete_response.code),
status_code=delete_response.code)
LOGGER.debug('Removed object_ids=%s from the capabilities table.',
object_ids)
async def process_post_search(request_context,
validate_map_update=None,
construct_post_search_string=None,
dashboard_visualization_map=None,
subscription_id=None,
server_subscription_response=None,
async_kvstore_client=None):
if validate_map_update:
validate_error_list = validate_map_update(dashboard_visualization_map)
if validate_error_list:
error_message = ','.join(validate_error_list)
raise SpacebridgeApiRequestError(
error_message, status_code=HTTPStatus.BAD_REQUEST)
post_search = construct_post_search_string(dashboard_visualization_map)
subscription_update = SearchUpdate(post_search=post_search,
key=subscription_id)
await process_subscription_update(request_context, subscription_update,
server_subscription_response,
async_kvstore_client)
def set_phantom_metadata(request_context, kvstore_client, phantom_metadata):
"""
Associates a Phantom instance for use with AR with this Splunk instance.
:param request_context: The RequestContext of the request.
:param kvstore_client: An AsyncKvStoreClient for storing Phantom metadata
:param phantom_metadata: A PhantomMetadata instance containing the information about Phantom to set.
"""
existing_metadata = None
try:
existing_metadata = yield get_phantom_metadata(request_context,
kvstore_client)
if phantom_metadata == existing_metadata:
LOGGER.info(
'New Phantom metadata is the same as the existing one.')
return
except NoRegisteredPhantomInstanceException:
LOGGER.debug(
'No existing phantom metadata found. No need to check whether the hostname has changed.'
)
if existing_metadata:
response = yield kvstore_client.async_kvstore_post_request(
collection=PHANTOM_METADATA_COLLECTION,
auth_header=request_context.auth_header,
key_id=phantom_metadata.key,
data=phantom_metadata.to_json())
else:
response = yield kvstore_client.async_kvstore_post_request(
collection=PHANTOM_METADATA_COLLECTION,
auth_header=request_context.auth_header,
data=phantom_metadata.to_json())
if response.code not in {http.OK, http.CREATED}:
message = yield response.text()
raise SpacebridgeApiRequestError(
'Failed to set Phantom metadata KV store entry message="{}" status_code={}'
.format(message, response.code),
status_code=response.code)
def _add_object_capabilities_to_role(self, request_context, role_name,
capabilities_to_add):
if not capabilities_to_add:
return
# Write the given object capabilities to the AR capabilities table and associate them with the given role
effected_ids = yield self._kvstore_client.async_batch_save_request(
collection=AR_CAPABILITIES_COLLECTION,
auth_header=request_context.auth_header,
entries=[
oc.to_kvstore_capabilities_entry(role_name)
for oc in capabilities_to_add
])
LOGGER.debug('Wrote capabilities=%s for role="%s"', effected_ids,
role_name)
# Remove the effected objects from either the read or write public pool.
keys_to_delete = []
for oc in capabilities_to_add:
if oc.access_level == LEVEL_READ:
keys_to_delete.append(
make_ar_capabilities_entry_key(AR_PUBLIC_READ_ROLE,
oc.object_type,
oc.object_id))
keys_to_delete.append(
make_ar_capabilities_entry_key(AR_PUBLIC_WRITE_ROLE,
oc.object_type, oc.object_id))
delete_response = yield self._kvstore_client.async_kvstore_delete_request(
collection=AR_CAPABILITIES_COLLECTION,
auth_header=request_context.auth_header,
params={QUERY: get_query_matching_keys(keys_to_delete)})
if delete_response.code != http.OK:
message = yield delete_response.text()
raise SpacebridgeApiRequestError(
'Failed to remove the following keys from the AR capabilities table for role="{}" keys={} message={} '
'status_code={}'.format(role_name, keys_to_delete, message,
delete_response.code),
status_code=delete_response.code)
async def fetch_dashboard_meta(request_context,
dashboard_id=None,
async_kvstore_client=None):
"""
Helper to fetch dashboard_meta
:param request_context:
:param dashboard_id:
:param async_kvstore_client:
:return:
"""
# Only set the key_id if a dashboard_id is provided
key_id = urllib.quote_plus(dashboard_id) if dashboard_id else None
response = await async_kvstore_client.async_kvstore_get_request(
collection=DASHBOARD_META_COLLECTION_NAME,
auth_header=request_context.auth_header,
owner=request_context.current_user,
key_id=key_id)
if response.code == HTTPStatus.OK:
response_json = await response.json()
if response_json and dashboard_id:
dashboard_meta = UserDashboardMeta.from_json(response_json)
return dashboard_meta
# If the response is the whole meta collection, return a list of dashboard metas
elif response_json and dashboard_id is None:
dashboard_meta = {}
for meta in response_json:
meta_obj = UserDashboardMeta.from_json(meta)
dashboard_meta[meta_obj.dashboard_id()] = meta_obj
return dashboard_meta
elif response.code != HTTPStatus.NOT_FOUND:
error = await response.text()
error_message = "Failed to fetch Dashboard Meta. status_code={}, error={}".format(
response.code, error)
raise SpacebridgeApiRequestError(message=error_message,
status_code=response.code)
# Return None no data if meta isn't available
return None
def process_group_delete_request(request_context, client_single_request,
server_single_response, async_kvstore_client):
"""
This method processes and group delete requests
:param request_context:
:param client_single_request:
:param server_single_response:
:param async_kvstore_client:
:return:
"""
# request params
group_ids = client_single_request.groupDeleteRequest.groupIds
is_public = client_single_request.groupDeleteRequest.isPublic
owner = constants.NOBODY if is_public else request_context.current_user
query = {
constants.OR_OPERATOR: [{
constants.KEY: group_id
} for group_id in group_ids]
}
response = yield async_kvstore_client.async_kvstore_delete_request(
collection=constants.GROUPS_COLLECTION_NAME,
owner=owner,
auth_header=request_context.auth_header,
params={constants.QUERY: json.dumps(query)})
if response.code != http.OK:
message = yield response.text()
raise SpacebridgeApiRequestError(
"Call to GROUPS DELETE failed with groupIds={}, code={}, message={}"
.format(group_ids, response.code, message))
# populate delete response
server_single_response.groupDeleteResponse.SetInParent()
LOGGER.debug(
"Call to GROUPS DELETE succeeded, groupIds={}".format(group_ids))
async def process_dashboard_visualization_map_update(
request_context,
validate_map_update=None,
construct_post_search_string=None,
dashboard_visualization_map=None,
client_subscription_message=None,
server_subscription_response=None,
async_kvstore_client=None):
# Retrieve Params
client_subscription_update = client_subscription_message.clientSubscriptionUpdate
subscription_id = client_subscription_update.subscriptionId
# Validate subscription_id
LOGGER.debug('Start process_subscription_update subscription_id=%s',
subscription_id)
# See if subscription_id exists in KVStore
get_response = await async_kvstore_client.async_kvstore_get_request(
collection=SUBSCRIPTIONS_COLLECTION_NAME,
key_id=subscription_id,
owner=NOBODY,
auth_header=request_context.auth_header)
if get_response.code != HTTPStatus.OK:
error = await get_response.text()
error_message = "Update failed. Subscription ID Not Found! status_code={}, error={},subscription_id={}" \
.format(get_response.code, error, subscription_id)
raise SpacebridgeApiRequestError(error_message,
status_code=get_response.code)
await process_post_search(
request_context=request_context,
validate_map_update=validate_map_update,
construct_post_search_string=construct_post_search_string,
dashboard_visualization_map=dashboard_visualization_map,
subscription_id=subscription_id,
server_subscription_response=server_subscription_response,
async_kvstore_client=async_kvstore_client)
def get_phantom_metadata(request_context, kvstore_client):
"""
Retrieves data about the Phantom instance associated with this Splunk instance for use with AR.
:param request_context: The RequestContext of the request.
:param kvstore_client: An AsyncKvStoreClient for looking up data from KV store.
:return: a PhantomMetadata instance
:raises: NoRegisteredPhantomInstanceException if there is no Phantom instance associated with this Splunk.
"""
phantom_metadata_response = yield kvstore_client.async_kvstore_get_request(
collection=PHANTOM_METADATA_COLLECTION,
auth_header=request_context.auth_header,
key_id=PhantomMetadata.key)
if phantom_metadata_response.code == http.NOT_FOUND:
raise NoRegisteredPhantomInstanceException()
if phantom_metadata_response.code != http.OK:
message = yield phantom_metadata_response.text()
raise SpacebridgeApiRequestError(
'Unable to fetch Phantom registration information message="{}" status_code={}'
.format(message, phantom_metadata_response.code),
status_code=phantom_metadata_response.code)
phantom_metadata_json = yield phantom_metadata_response.json()
defer.returnValue(PhantomMetadata.from_json(phantom_metadata_json))
def fetch_dashboard_app_list(request_context,
async_kvstore_client=None,
async_splunk_client=None):
"""
Helper to fetch dashboard_app_list from user_meta
:param request_context:
:param async_kvstore_client:
:return:
"""
if async_kvstore_client and async_splunk_client:
response = yield async_kvstore_client.async_kvstore_get_request(
collection=USER_META_COLLECTION_NAME,
auth_header=request_context.auth_header,
owner=request_context.current_user,
key_id=DASHBOARD_APP_LIST)
if response.code == http.OK:
response_json = yield response.json()
if response_json:
app_names = json.loads(response_json[APP_NAMES])
# make sure apps are not disabled/invisible (fetch app names filters these out)
valid_app_names = yield fetch_app_names(
request_context=request_context,
async_splunk_client=async_splunk_client)
valid_app_names = [app.app_name for app in valid_app_names]
defer.returnValue(
[app for app in app_names if app in valid_app_names])
elif response.code != http.NOT_FOUND:
error = yield response.text()
error_message = "Unable to find dashboard_app_list in user_meta collection. status_code={}, error={}"\
.format(response.code, error)
raise SpacebridgeApiRequestError(error_message)
# Return None no data if meta isn't available
defer.returnValue(None)
async def fetch_search(auth_header,
search_key=None,
async_kvstore_client=None,
none_if_not_found=False):
"""
Fetch a object from kvstore collection [searches] with search _key, searches stored in nobody namespace
:param auth_header:
:param search_key:
:param async_kvstore_client:
:param none_if_not_found:
:return:
"""
response = await async_kvstore_client.async_kvstore_get_request(
collection=SEARCHES_COLLECTION_NAME,
key_id=search_key,
auth_header=auth_header)
if response.code == HTTPStatus.OK:
response_json = await response.json()
if not response_json:
# No Search found with search_key
return None
# Create SubscriptionSearch from json
search = SubscriptionSearch.from_json(response_json)
return search
# Special case here to return None if not found
if response.code == HTTPStatus.NOT_FOUND and none_if_not_found:
return None
# Return error in case of unsuccessful response
error = await response.text()
error_message = "Failed to fetch searches. status_code=%s, error=%s, search_key=%s, auth_header={}".format(
response.code, error, search_key, auth_header)
raise SpacebridgeApiRequestError(error_message, status_code=response.code)
def _get_documents_for_object_ids(self, request_context, object_ids):
if not object_ids:
defer.returnValue([])
capabilities_for_ids = yield self._kvstore_client.async_kvstore_get_request(
collection=AR_CAPABILITIES_COLLECTION,
auth_header=request_context.auth_header,
params={
QUERY:
json.dumps({
OR_OPERATOR: [{
OBJECT_ID: object_id
} for object_id in object_ids]
})
})
if capabilities_for_ids.code != http.OK:
message = yield capabilities_for_ids.text()
raise SpacebridgeApiRequestError(
'Failed to lookup object_ids={} to add back to the public access pool '
'message="{}" status_code={}'.format(
object_ids, message, capabilities_for_ids.code),
status_code=capabilities_for_ids.code)
capabilities_json = yield capabilities_for_ids.json()
defer.returnValue(capabilities_json)
def validate_dashboard_search(request_context,
dashboard_id=None,
type_id=None,
search_type=SearchType.VISUALIZATION,
input_tokens=None,
async_kvstore_client=None,
async_splunk_client=None):
"""
Validation method to validate a dashboard_id and visualization_id. Will except a SpacebridgeApiRequestError if
issues are detected and return a dashboard_description if valid
:param request_context:
:param dashboard_id:
:param type_id:
:param search_type:
:param input_tokens:
:param async_kvstore_client:
:param async_splunk_client:
:return:
"""
# Validate params
if not dashboard_id or not type_id:
error_message = "Invalid Request Params dashboard_id={}, search_type_id={}, search_type={}" \
.format(dashboard_id, type_id, search_type)
raise SpacebridgeApiRequestError(error_message)
# fetch dashboard body
dashboard_description = yield fetch_dashboard_description(
request_context=request_context,
dashboard_id=dashboard_id,
async_splunk_client=async_splunk_client,
async_kvstore_client=async_kvstore_client)
search = None
if search_type == SearchType.VISUALIZATION:
visualization = dashboard_description.get_visualization(type_id)
if not visualization:
error_message = "Dashboard visualization not found. dashboard_id={}, visualization_id={}" \
.format(dashboard_id, type_id)
raise SpacebridgeApiRequestError(error_message)
search = visualization.search
elif search_type == SearchType.INPUT:
input_token = dashboard_description.get_input_token_by_query_id(type_id)
if not input_token:
error_message = "Input Search not found. dashboard_id={}, query_id={}".format(dashboard_id, type_id)
raise SpacebridgeApiRequestError(error_message)
search = input_token.input_type.dynamic_options.search
elif search_type == SearchType.DATA_SOURCE:
datasources = [d for d in dashboard_description.definition.udf_data_sources if d.name == type_id]
if len(datasources) != 1:
raise SpacebridgeApiRequestError(
"Unexpected number of matching datasources in dashboard. Expected 1 but found {}"
.format(len(datasources)))
# validate depends
if search and not search.are_render_tokens_defined(input_tokens):
error_message = "Search is waiting for input. depends={}, rejects={}, dashboard_id={}, type_id={}, search_type={}" \
.format(search.depends, search.rejects, dashboard_id, type_id, search_type)
raise SpacebridgeApiRequestError(error_message)
defer.returnValue(dashboard_description)
async def lazy_load_subscription_search(request_context,
input_tokens,
search_key,
search_type,
search_type_id,
search_defn,
async_kvstore_client,
async_splunk_client,
shard_id,
spawn_search_job,
dashboard_defn=None,
dashboard_id=None,
default_input_tokens=None,
visualization_type=None,
trellis_enabled=False,
trellis_split_by=None):
"""
:param trellis_enabled:
:param request_context:
:param input_tokens:
:param search_key:
:param search_type:
:param search_type_id:
:param search_defn:
:param async_kvstore_client:
:param async_splunk_client:
:param shard_id:
:param spawn_search_job:
:param dashboard_defn:
:param dashboard_id:
:param default_input_tokens:
:param visualization_type:
:param trellis_split_by:
:return:
"""
if default_input_tokens is None:
default_input_tokens = {}
if type != SearchType.SAVED_SEARCH:
# Set Default input_token values
set_default_token_values(input_tokens, default_input_tokens)
parent_id = None
search_query = search_defn.query
sid = None
if search_defn.base:
parent_search_defn = dashboard_defn.find_base_search(search_defn.base)
parent_search_key = generate_search_hash(
dashboard_id,
parent_search_defn.id,
input_tokens,
request_context.current_user,
refresh_interval=refresh_to_seconds(parent_search_defn.refresh))
LOGGER.debug("Base search exists, base=%s", parent_search_defn)
parent = await lazy_load_subscription_search(
request_context=request_context,
input_tokens=input_tokens,
search_key=parent_search_key,
search_type=SearchType.ROOT.value,
search_type_id=parent_search_key,
search_defn=parent_search_defn,
async_kvstore_client=async_kvstore_client,
async_splunk_client=async_splunk_client,
shard_id=request_context.shard_id,
spawn_search_job=spawn_search_job,
dashboard_defn=parent_search_defn,
dashboard_id=dashboard_id,
default_input_tokens=default_input_tokens,
visualization_type=visualization_type,
trellis_enabled=trellis_enabled,
trellis_split_by=trellis_split_by)
sid = parent.sid
parent_id = parent.key()
# i.e. no parent search
if not sid:
sid = search_key
subscription_search = build_subscription_search(
request_context,
dashboard_id,
search_defn,
search_query,
input_tokens,
parent_id,
search_key,
search_type,
search_type_id,
shard_id,
sid,
visualization_type=visualization_type,
trellis_enabled=trellis_enabled,
trellis_split_by=trellis_split_by)
params = {}
if visualization_type is not None and VisualizationType(
visualization_type
) == VisualizationType.DASHBOARD_VISUALIZATION_EVENT:
params[constants.COUNT] = '200'
job_status = await get_search_job_content(request_context.auth_header,
subscription_search.owner,
SPACEBRIDGE_APP_NAME,
subscription_search.sid,
async_splunk_client,
params=params)
if not job_status:
LOGGER.debug("Start search job, sid=%s", sid)
await spawn_search_job(subscription_search, input_tokens, sid)
http_result = await create_search(request_context, subscription_search,
async_kvstore_client)
if http_result not in [HTTPStatus.CREATED, HTTPStatus.CONFLICT]:
raise SpacebridgeApiRequestError(
message='Failed to create subscription search',
status_code=http_result)
return subscription_search
async def update_search_job_status(auth_header, owner, app_name, search, sid,
async_splunk_client, async_kvstore_client):
"""
Helper method to update search job status on a search collection object
:param auth_header:
:param owner:
:param app_name:
:param search:
:param sid:
:param async_splunk_client:
:param async_kvstore_client:
:return:
"""
# Loops and calls itself until job has results
loop_response = await deferred_loop(
poll_interval_seconds=DEFAULT_JOB_RESULTS_POLL_INTERVAL,
timeout_seconds=DEFAULT_JOB_RESULTS_TIMEOUT,
deferred_function=get_running_search_job_content,
# params to deferred_function
auth_header=auth_header,
owner=owner,
app_name=app_name,
sid=sid,
async_splunk_client=async_splunk_client)
# search_content check if None
if loop_response is None or loop_response.response is None:
await delete_search_job(auth_header=auth_header,
owner=owner,
app_name=app_name,
sid=sid,
async_splunk_client=async_splunk_client)
raise SpacebridgeApiRequestError(
"Search job timed out processing. sid=%s" % sid,
status_code=HTTPStatus.REQUEST_TIMEOUT)
# set valid search_job_content from loop_response
search_job_content = loop_response.response
LOGGER.info("Search Job Processed: %s", search_job_content)
# Check if search job Failed
if search_job_content.is_failed():
error_message = search_job_content.get_first_error_message(
'Search job Failed. sid=%s' % sid)
raise SpacebridgeApiRequestError(
error_message, status_code=HTTPStatus.FAILED_DEPENDENCY)
if search_job_content.is_done:
# only update the next_update_time if the refresh_interval_seconds has a value
search.next_update_time = get_expiration_timestamp_str(ttl_seconds=search.refresh_interval_seconds) \
if search.refresh_interval_seconds else ""
# update metadata on search
search.sid = sid
search.dispatch_state = search_job_content.dispatch_state
search.done_progress = search_job_content.done_progress
# Update Search Collection
response = await async_kvstore_client.async_kvstore_post_request(
collection=SEARCHES_COLLECTION_NAME,
data=jsonpickle.encode(
search, unpicklable=False), # Don't write py/object field
key_id=search.key(),
auth_header=auth_header)
if response.code != HTTPStatus.OK:
error = await response.text()
LOGGER.error(
"Unable to update sid on kvstore search. status_code=%s, error=%s, %s",
response.code, error, search)
raise SpacebridgeApiRequestError(format_splunk_error(
response.code, error),
status_code=response.code)
return search
def fetch_subscriptions(auth_header,
async_kvstore_client,
user_list=None,
subscription_type=None,
subscription_id=None,
device_ids=None):
"""
Fetch drone mode subscription objects from kvstore collection [subscription]
# It specifically does the following:
# 1. Returns all drone mode subscriptions
# 2. Returns all drone mode subcriptions of a certain type (ipad, tv)
# 3. Returns all subscriptions given a list of ids
:param auth_header: auth header used to authenticate to kvstore
:param owner: collection owner
:param async_kvstore_client: kvstore client used to make request
:param subscription_type: Type of subscription to fetch (ipad, tv)
:param subscription_id: subscription_id to fetch
:param device_ids: device ids to fetch (must also provide subscription type)
:return:
"""
if subscription_type and subscription_id:
raise SpacebridgeApiRequestError(
'You can only provide one of subscription type ' +
'or subscription id to fetch subscriptions.',
status_code=http.BAD_REQUEST)
if device_ids and not subscription_type:
raise SpacebridgeApiRequestError(
'You must provide a subscription type to filter by ' +
'device ids.',
status_code=http.BAD_REQUEST)
if device_ids and subscription_type:
params = {
constants.QUERY:
json.dumps({
constants.AND_OPERATOR: [
build_containedin_clause(constants.USER, user_list),
build_containedin_clause(constants.DEVICE_ID, device_ids),
{
constants.SUBSCRIPTION_TYPE: subscription_type
}
]
})
}
elif subscription_type:
params = {
constants.QUERY:
json.dumps({constants.SUBSCRIPTION_TYPE: subscription_type})
}
elif user_list:
params = {
constants.QUERY:
json.dumps({
constants.AND_OPERATOR: [
build_containedin_clause(constants.USER, user_list),
build_containedin_clause(
constants.SUBSCRIPTION_TYPE,
{constants.DRONE_MODE_IPAD, constants.DRONE_MODE_TV})
]
})
}
else:
params = {
constants.QUERY:
json.dumps(
build_containedin_clause(
constants.SUBSCRIPTION_TYPE,
{constants.DRONE_MODE_IPAD, constants.DRONE_MODE_TV}))
}
response = yield async_kvstore_client.async_kvstore_get_request(
collection=constants.SUBSCRIPTIONS_COLLECTION_NAME,
params=params,
owner=constants.NOBODY,
key_id=subscription_id,
auth_header=auth_header)
subscriptions = []
if response.code == http.OK:
response_json = yield response.json()
if isinstance(response_json, list):
subscriptions = [
Subscription.from_json(subscription)
for subscription in response_json
]
else:
if response_json:
subscriptions.append(Subscription.from_json(response_json))
else:
error = yield response.text()
LOGGER.error("Unable to fetch_subscriptions. status_code=%s, error=%s",
response.code, error)
LOGGER.debug('subscriptions in fetch_subscriptions=%s', subscriptions)
defer.returnValue(subscriptions)
def validate_devices(device_ids, request_context, async_kvstore_client):
"""
Function to determine if device ids are valid
(subscribed to drone mode and in the registered_devices table)
:param device_ids: the set of device ids to check
:param request_context: request context used to make kvstore requests
:param async_kvstore_client: async client used to make kvstore requests
"""
# Check to see if devices are in registered_devices_table
tv_list = yield get_registered_tvs(request_context.auth_header,
request_context.current_user,
async_kvstore_client,
device_ids=device_ids)
tv_list_ids = {
registered_tv[constants.DEVICE_ID]
for registered_tv in tv_list
}
errors = []
LOGGER.debug('device_ids=%s, tv_list_ids=%s', device_ids, tv_list_ids)
registered_set_difference = set()
subscription_set_difference = set()
if tv_list_ids != device_ids:
registered_set_difference = device_ids - tv_list_ids
errors.append(
'The tvs with ids={} are no longer registered to this instance'.
format(list(registered_set_difference)))
if not tv_list_ids:
raise SpacebridgeApiRequestError(errors[0],
status_code=http.NOT_FOUND)
tv_subscriptions = yield fetch_subscriptions(
request_context.auth_header,
async_kvstore_client,
user_list=[request_context.current_user],
subscription_type=constants.DRONE_MODE_TV,
device_ids=device_ids)
# Check to see if subscriptions are active
subscription_ids = {
subscription.device_id
for subscription in tv_subscriptions
}
LOGGER.debug('device_ids=%s, subscription_ids=%s', device_ids,
subscription_ids)
if subscription_ids != device_ids:
subscription_set_difference = device_ids - subscription_ids
errors.append(
'The tvs with ids={} are no longer subscribed to drone mode'.
format(list(subscription_set_difference)))
if not subscription_ids:
raise SpacebridgeApiRequestError(' and '.join(errors),
status_code=http.BAD_REQUEST)
# if all tvs are invalid, but for different reasons
if registered_set_difference.union(
subscription_set_difference) == device_ids:
raise SpacebridgeApiRequestError(
'The tvs with ids={} are all invalid: these ids={} are no longer registered to the instance and '
'these={} are no longer subscribed to drone mode'.format(
list(device_ids), list(registered_set_difference),
list(subscription_set_difference)),
status_code=http.BAD_REQUEST)
defer.returnValue(errors)
async def fetch_visualization_data(request_context,
owner=None,
app_name=None,
visualization=None,
async_splunk_client=None):
"""
Perform async call to fetch visualization search data
NOTE: This function doesn't support saved searches (i.e. 'ref' attribute)
:param request_context:
:param owner:
:param app_name:
:param visualization:
:param async_splunk_client:
:return:
"""
LOGGER.info(
"Start fetch_visualization_data owner={}, app_name={}, visualization.id={}, visualization.search.query={}"
.format(owner, app_name, visualization.id, visualization.search.query))
# validate search
if visualization.search is None:
raise SpacebridgeApiRequestError(
"Search is Empty for visualization_id={}".format(visualization.id),
status_code=HTTPStatus.NO_CONTENT)
# populate search params
search = visualization.search
params = get_search_job_request_params(
query=search.query,
earliest_time=search.earliest,
latest_time=search.latest,
sample_ratio=search.sample_ratio,
exec_mode=constants.EXEC_MODE_ONESHOT)
if not params:
raise SpacebridgeApiRequestError(
"Search Query is Empty for visualization_id={}".format(
visualization.id),
status_code=HTTPStatus.BAD_REQUEST)
response = await async_splunk_client.async_get_search_data_request(
owner=owner,
app_name=app_name,
auth_header=request_context.auth_header,
data=urllib.urlencode(params))
if response.code != HTTPStatus.OK:
response_text = await response.text()
raise SpacebridgeApiRequestError(
"Failed to get search data search_query={}, response.code={}, response.text={}"
.format(params['search'], response.code, response_text),
status_code=response.code)
response_json = await response.json()
visualization_data = parse.to_visualization_data(response_json)
# Log warning message if fields, columns are empty
# If fields and columns are empty we should log messages if available
if visualization_data.is_empty_data(
) and 'messages' in response_json and response_json['messages']:
for message in response_json['messages']:
msg_type = message['type'] if 'type' in message else "no_type"
text = message['text'] if 'text' in message else "no_text"
LOGGER.info(
"Search Data Message search_query={}, type={}, text={}".format(
params['search'], msg_type, text))
return visualization_data
async def fetch_dashboard_description(request_context,
dashboard_id=None,
show_refresh=True,
async_splunk_client=None,
async_kvstore_client=None):
"""
Method will make async http call to get single dashboard and return a DashboardDescription object
:param request_context:
:param dashboard_id:
:param show_refresh: show refresh params, default True
:param async_splunk_client:
:param async_kvstore_client:
:return:
"""
params = {'output_mode': 'json'}
owner, app_name, dashboard_name = dashboard_helpers.parse_dashboard_id(
dashboard_id)
if not owner or owner == '-':
owner = request_context.current_user
LOGGER.info("No owner given in dashboard_id=%s taking owner=%s",
dashboard_id, owner)
response = await async_splunk_client.async_get_dashboard_request(
owner=owner,
app_name=app_name,
auth_header=request_context.auth_header,
params=params,
dashboard_name=dashboard_name)
LOGGER.info('fetch_dashboard_description response={}'.format(
response.code))
if response.code != HTTPStatus.OK:
response_text = await response.text()
raise SpacebridgeApiRequestError(
"Failed fetch_dashboard_description request dashboard_id={}, response.code={}, response.text={}"
.format(dashboard_id, response.code, response_text),
status_code=response.code)
response_json = await response.json()
entry_json_list = response_json.get('entry')
if not entry_json_list:
# log result in the event the dashboardId is not valid
raise SpacebridgeApiRequestError(
"No Entries found for dashboard_id={}".format(dashboard_id),
status_code=HTTPStatus.NOT_FOUND)
dashboard = await parse.to_dashboard_description(
entry_json_list[0],
request_context=request_context,
async_splunk_client=async_splunk_client,
show_refresh=show_refresh)
if async_kvstore_client is not None:
dashboard_meta = await fetch_dashboard_meta(
request_context=request_context,
dashboard_id=dashboard_id,
async_kvstore_client=async_kvstore_client)
if dashboard_meta:
dashboard.is_favorite = dashboard_meta.is_favorite
return dashboard
def update_asset(request_context, asset_data, kvstore, permissions):
"""
Updates a single already existing asset.
:param request_context: the RequestContext of the caller
:param asset_data: the asset to update
:param kvstore: the AsyncKvStoreClient to query KV store with
:param permissions: the AsyncArPermissionsClient to validate permissions with
:return: the ID of the updated asset
"""
if not asset_data.asset_id:
raise MissingAssetIDException(
'Must provide an asset ID when updating an asset.')
existing_asset_response = yield kvstore.async_kvstore_get_request(
collection=ASSETS_COLLECTION_NAME,
auth_header=request_context.auth_header,
key_id=asset_data.asset_id)
if existing_asset_response.code != http.OK:
message = yield existing_asset_response.text()
raise SpacebridgeApiRequestError(
'Failed to load asset_id={} message={} status_code={}'.format(
asset_data.asset_id, message, existing_asset_response.code),
status_code=existing_asset_response.code)
existing_asset = yield existing_asset_response.json()
existing_asset = AssetData.from_json(existing_asset)
# The asset group changed. Users must have asset_group_manage to do this.
if asset_data.asset_group != existing_asset.asset_group:
message = 'User username={} needs asset_group_manage to update the asset group for asset_id={}'.format(
request_context.current_user, asset_data.asset_id)
yield permissions.check_permission(request_context,
Capabilities.ASSET_GROUP_MANAGE,
message=message)
if asset_data.asset_group:
yield permissions.check_object_permission(
request_context,
Capabilities.ASSET_GROUP_WRITE,
asset_data.asset_group,
)
else:
yield permissions.check_object_permission(request_context,
Capabilities.ASSET_WRITE,
asset_data.asset_id)
update_response = yield kvstore.async_kvstore_post_request(
collection=ASSETS_COLLECTION_NAME,
auth_header=request_context.auth_header,
key_id=asset_data.asset_id,
data=asset_data.to_json())
if update_response.code == http.NOT_FOUND:
raise NoSuchAssetException(asset_id=asset_data.asset_id)
if update_response.code != http.OK:
message = yield update_response.text()
raise SpacebridgeApiRequestError(
'Failed to update asset_id={} with payload={} message={} status_code={}'
.format(asset_data.asset_id, asset_data.to_json(), message,
update_response.code),
status_code=update_response.code)
defer.returnValue(asset_data.asset_id)
def process_subscribe_drone_mode_ipad(request_context,
client_subscription_message,
server_subscription_response,
async_client_factory):
"""
Process subscribe to drone mode ipad events for a device id
:param request_context: Used to authenticate kvstore requests
:param client_single_request: client request object protobuf
:param single_server_response: server response object protobuf
:param async_client factory: factory class used to generate kvstore and spacebridge clients
"""
LOGGER.debug('start of subscribe drone mode ipad')
async_kvstore_client = async_client_factory.kvstore_client()
LOGGER.debug('request_context=%s', request_context)
# base64 encode device id
device_id = request_context.device_id
urlsafe_b64encoded_device_id = b64_to_urlsafe_b64(device_id)
query = {
constants.QUERY: json.dumps({
constants.DEVICE_ID: device_id,
}),
constants.LIMIT: 1
}
# fetch device from registered_devices_table to get device name
response = yield async_kvstore_client.async_kvstore_get_request(
collection=constants.REGISTERED_DEVICES_COLLECTION_NAME,
params=query,
owner=request_context.current_user,
auth_header=request_context.auth_header)
yield check_and_raise_error(response, request_context, 'Fetch registered devices')
device = yield response.json()
if device:
device = device[0]
else:
raise SpacebridgeApiRequestError('Invalid device id={}'.format(device_id), status_code=http.BAD_REQUEST)
device_name = device[constants.DEVICE_NAME]
# write to ipads collection
ipad_json = {constants.DEVICE_NAME: device_name}
response = yield async_kvstore_client.async_kvstore_post_or_update_request(
constants.DRONE_MODE_IPADS_COLLECTION_NAME,
json.dumps(ipad_json),
request_context.system_auth_header,
key_id=urlsafe_b64encoded_device_id,
owner=request_context.current_user)
# construct parameters for subscription creation / updating
ttl_seconds = client_subscription_message.clientSubscribeRequest.ttlSeconds
expiration_time = get_expiration_timestamp_str(ttl_seconds=ttl_seconds)
now = get_current_timestamp_str()
params = {
constants.TTL_SECONDS: str(ttl_seconds),
constants.DEVICE_ID: device_id,
constants.LAST_UPDATE_TIME: now,
constants.EXPIRED_TIME: expiration_time,
constants.USER: request_context.current_user,
constants.SUBSCRIPTION_TYPE: constants.DRONE_MODE_IPAD
}
# delete existing subscriptions
query = {
constants.QUERY: json.dumps({
constants.DEVICE_ID: device_id,
constants.USER: request_context.current_user,
constants.SUBSCRIPTION_TYPE: constants.DRONE_MODE_IPAD
})
}
delete_response = yield async_kvstore_client.async_kvstore_delete_request(
collection=constants.SUBSCRIPTIONS_COLLECTION_NAME,
params=query,
owner=constants.NOBODY,
auth_header=request_context.auth_header)
yield check_and_raise_error(delete_response, request_context, 'Delete existing ipad subscriptions')
# create subscription
subscription_id = yield write_drone_mode_subscription(request_context,
params,
async_client_factory)
server_subscription_response.subscriptionId = subscription_id
server_subscription_response.serverSubscribeResponse.droneModeiPadSubscribe.SetInParent()
LOGGER.debug('Successfully wrote drone mode ipad subscription with id=%s', subscription_id)