def calculate(self):
kvstore_client = KVStoreCollectionAccessObject(APPLICATION_TYPES_COLLECTION_NAME,
self.session_token,
owner=NOBODY)
r, app_states = kvstore_client.get_all_items()
metrics = {}
for app_state in json.loads(app_states):
metrics[app_state["application_name"]] = app_state["application_enabled"]
return {self.METRIC_NAME: metrics}
def delete_all_devices_of_type_for_user(user, app_type, authtoken, system_authtoken):
"""
Removes all devices of a given type from a single user in the kvstore
"""
kvstore = KvStore(constants.REGISTERED_DEVICES_COLLECTION_NAME, authtoken, owner=user)
r, devices = kvstore.get_all_items()
devices = json.loads(devices)
kvstore.delete_items_by_query({'device_type': app_type})
for device in devices:
if device['device_type'] == app_type:
delete_device_from_spacebridge(device['device_id'], system_authtoken)
def get(self, request):
""" get registered companion apps """
system_authtoken = request['system_authtoken']
kvstore_client = KVStoreCollectionAccessObject(collection=self.COMPANION_APPS_COLLECTION_NAME,
session_key=system_authtoken)
r, content = kvstore_client.get_all_items()
payload = json.loads(content.decode('utf-8')) if r.status == HTTPStatus.OK else content.decode('utf-8')
return {
'payload': payload,
'status': r.status
}
def sync(self):
"""
Gets all registered users. Gets all tokens per user, sorted by expiry date. Deletes all tokens except the one
with the most recent expiration date and any that are being used as subscription credentials.
"""
all_registered_users = get_all_mobile_users(self.session_key)
try:
for user in all_registered_users:
tokens = get_all_secure_gateway_tokens(self.session_key, user)
current_time = get_current_timestamp()
index_to_delete = min(3, len(tokens))
for i in range(0, index_to_delete - 1):
if token[i]['content']['claims']['exp'] < current_time:
index_to_delete = i
break
tokens_to_delete = tokens[index_to_delete:]
kvstore_subscription_credentials = KvStore(
constants.SUBSCRIPTION_CREDENTIALS_COLLECTION_NAME,
self.session_key,
owner=user)
response, credentials = kvstore_subscription_credentials.get_all_items(
)
credentials = json.loads(credentials)
jwt_credential = [
c for c in credentials if 'session_type' in c
and c['session_type'] == constants.JWT_TOKEN_TYPE[0]
]
if jwt_credential:
subscription_token_info = calculate_token_info(
jwt_credential['session_key'])
else:
subscription_token_info = None
for token in tokens_to_delete:
# if that token does not exist in subscription credentials
if not subscription_token_info or token[
'name'] != subscription_token_info['id']:
delete_token_by_id(self.session_key, user,
token['name'])
except:
LOGGER.exception("Exception performing DeleteTokensSync")
def get(self, request):
"""
Lists one or more subscriptions based on provided json/query payload parameters, OR key provided on path
Request Parameters
users optional list of user to fetch subscriptions for
subscription_types optional list of subscription types to filter on
device_ids optional list of device_ids to filter on
You can either pass these in the json body, or pass them in like
https://localhost:8089/services/ssg/subscription/?users=user1&users=user2
Returns:
list of subscriptions found
Example:
[
{
"device_id": "3",
"expired_time": "12345678",
"last_update_time": "123456",
"shard_id": "shard",
"subscription_key": "keyy",
"subscription_type": "Splunk TV",
"ttl_seconds": "10",
"user": "******",
"version": 2,
"visualization_id": "",
"_user": "******",
"_key": "5f04e4d1ed7a6aab3e6f2e91"
}
]
"""
session_key = request[constants.SESSION][constants.AUTHTOKEN]
path = request.get(PATH_INFO)
kvstore_object = KVStore(
collection=constants.SUBSCRIPTIONS_COLLECTION_NAME,
session_key=session_key)
if path: # search using path as key
LOGGER.debug('Fetching subscription with key %s', path)
try:
response_header, response = kvstore_object.get_item_by_key(
path)
except splunk.RESTException as e:
if e.statusCode == HTTPStatus.NOT_FOUND:
return {
constants.PAYLOAD: {
'Error': 'Not found'
},
constants.STATUS: HTTPStatus.NOT_FOUND,
}
raise e
else: # use json parameters or search for all
payload = json.loads(request.get(constants.PAYLOAD, '{}'))
query_params = request.get(constants.QUERY)
users = payload.get(USERS)
subscription_types = payload.get(SUBSCRIPTION_TYPES)
device_ids = payload.get(DEVICE_IDS)
# if not in json, try to pull from query params
if not users and query_params:
users = get_list_from_query(query_params, USERS)
if not subscription_types and query_params:
subscription_types = get_list_from_query(
query_params, SUBSCRIPTION_TYPES)
if not device_ids and query_params:
device_ids = get_list_from_query(query_params, DEVICE_IDS)
query = {}
if users:
query = build_containedin_clause(constants.USER, users)
if subscription_types:
query = {
constants.AND_OPERATOR: [
build_containedin_clause(constants.SUBSCRIPTION_TYPE,
subscription_types), query
]
}
if device_ids:
query = {
constants.AND_OPERATOR: [
build_containedin_clause(constants.DEVICE_ID,
device_ids), query
]
}
if query:
LOGGER.debug('Fetching subscription(s) with query %s', query)
response_header, response = kvstore_object.get_items_by_query(
query)
else:
LOGGER.debug('Fetching all subscriptions')
response_header, response = kvstore_object.get_all_items()
payload = json.loads(response)
return {
constants.PAYLOAD: payload,
constants.STATUS: response_header.status,
}
def run(self):
"""
Attempts to sync the migration. If the kvstore is not yet available, schedules
a non-blocking retry attempt in 5 seconds
"""
LOGGER.info("Attempting Migration from Splunk Cloud Gateway to Splunk Secure Gateway")
try:
meta_collection = KvStore(META_COLLECTION_NAME, self.session_key, owner=NOBODY)
migration_info = {KEY: MIGRATION_DONE, STATUS: '0'}
meta_keys = meta_collection.get_collection_keys()
if MIGRATION_DONE in meta_keys:
migration_status = meta_collection.get_item_by_key(MIGRATION_DONE)
else:
migration_status = '0'
if migration_status.isdigit() and int(migration_status):
LOGGER.debug("Migration modular input will not run because migration from Splunk Cloud Gateway "
"is already done")
else:
# List of all collections
_, collections_content = get_all_collections(self.session_key, app_name=constants.CLOUDGATEWAY_APP_NAME)
all_collections = json.loads(collections_content)['entry']
app_collections = [x for x in all_collections if x['acl']['app'] == CLOUDGATEWAY_APP_NAME]
app_collection_names = {x['name'] for x in app_collections
if x['name'] not in collections_not_to_migrate}
# Special case this collection because it doesn't show up in the SCG /collections/config endpoint
app_collection_names.add("user_meta")
registered_users_kvstore = KvStore(constants.REGISTERED_USERS_COLLECTION_NAME, self.session_key,
app=constants.CLOUDGATEWAY_APP_NAME, owner=NOBODY)
_, users_content = registered_users_kvstore.get_collection_keys()
registered_user_records = json.loads(users_content)
all_registered_users = [registered_user_record[u'_key']
for registered_user_record in registered_user_records]
if NOBODY not in all_registered_users:
all_registered_users.append(NOBODY)
for collection in app_collection_names:
# Iterate through all users and insert data per user
for user in all_registered_users:
cloud_gateway_kvstore = KvStore(collection, self.session_key,
app=constants.CLOUDGATEWAY_APP_NAME, owner=user)
_, scg_content = cloud_gateway_kvstore.get_all_items()
scg_content = json.loads(scg_content)
if scg_content:
if collection == constants.REGISTERED_DEVICES_COLLECTION_NAME:
scg_content = [resolve_device_platform_and_type(device) for device in scg_content]
secure_gateway_kvstore = KvStore(collection, self.session_key,
app=constants.SPACEBRIDGE_APP_NAME, owner=user)
_, ssg_content = secure_gateway_kvstore.insert_multiple_items(scg_content)
# Copying passwords.conf to Splunk Secure Gateway
for key in password_keys:
try:
value = fetch_sensitive_data(self.session_key, key, app=CLOUDGATEWAY_APP_NAME)
update_or_create_sensitive_data(self.session_key, key, value)
except splunk.ResourceNotFound:
LOGGER.debug('key=%s not found in storage/passwords', key)
migration_info[STATUS] = '1'
meta_collection.insert_or_update_item_containing_key(migration_info)
except splunk.RESTException as e:
if e.statusCode == HTTPStatus.SERVICE_UNAVAILABLE:
LOGGER.info("KVStore is not yet setup. Retrying migration in 5 seconds")
time.sleep(TIMEOUT_SECONDS)
self.run()
else:
raise e
