def test_flatten_dict(self):
"""
Test the flattening of a dict via flatten_dict().
"""
d = '{ "name" : "Test", "configuration" : { "views" : [ { "name" : "some_view", "app" : "some_app" } ], "delay" : 300, "delay_readable" : "5m", "hide_chrome" : true, "invert_colors" : true }, "_user" : "nobody", "_key" : "123456789" }'
flattened_d = shortcuts.flatten_dict(json.loads(d))
self.assertEquals(flattened_d['configuration.delay'], 300)
self.assertEquals(flattened_d['configuration.views'][0]['app'], 'some_app')
self.assertEquals(flattened_d['name'], "Test")
def test_flatten_dict(self):
"""
Test the flattening of a dict via flatten_dict().
"""
d = '{ "name" : "Test", "configuration" : { "views" : [ { "name" : "some_view", "app" : "some_app" } ], "delay" : 300, "delay_readable" : "5m", "hide_chrome" : true, "invert_colors" : true }, "_user" : "nobody", "_key" : "123456789" }'
flattened_d = shortcuts.flatten_dict(json.loads(d))
self.assertEquals(flattened_d['configuration.delay'], 300)
self.assertEquals(flattened_d['configuration.views'][0]['app'],
'some_app')
self.assertEquals(flattened_d['name'], "Test")
def test_flatten_dict_specified_fields(self):
"""
Test the flattening of a dict via flatten_dict() but only convert the given fields.
"""
d = '{ "name" : "Test", "configuration" : { "views" : [ { "name" : "some_view", "app" : "some_app" } ], "delay" : 300, "delay_readable" : "5m", "hide_chrome" : true, "invert_colors" : true }, "_user" : "nobody", "_key" : "123456789" }'
flattened_d = shortcuts.flatten_dict(json.loads(d), fields=['name', 'configuration', '_user', '_key'])
self.assertEquals(flattened_d['name'], 'Test')
# Now parse the text within the configuration element and make sure it is the expected JSON
c = json.loads(flattened_d['configuration'])
self.assertEquals(c['views'][0]["name"], 'some_view')
def test_flatten_dict_specified_fields(self):
"""
Test the flattening of a dict via flatten_dict() but only convert the given fields.
"""
d = '{ "name" : "Test", "configuration" : { "views" : [ { "name" : "some_view", "app" : "some_app" } ], "delay" : 300, "delay_readable" : "5m", "hide_chrome" : true, "invert_colors" : true }, "_user" : "nobody", "_key" : "123456789" }'
flattened_d = shortcuts.flatten_dict(
json.loads(d), fields=['name', 'configuration', '_user', '_key'])
self.assertEquals(flattened_d['name'], 'Test')
# Now parse the text within the configuration element and make sure it is the expected JSON
c = json.loads(flattened_d['configuration'])
self.assertEquals(c['views'][0]["name"], 'some_view')
def get_kv_lookup(self, session_key, lookup_file, namespace="lookup_editor", owner=None):
"""
Get the contents of a KV store lookup.
"""
if owner is None:
owner = 'nobody'
lookup_contents = []
# Get the fields so that we can compose the header
# Note: this call must be done with the user context of "nobody".
response, content = splunk.rest.simpleRequest('/servicesNS/nobody/' + namespace +
'/storage/collections/config/' +
lookup_file,
sessionKey=session_key,
getargs={'output_mode': 'json'})
if response.status == 403:
raise PermissionDeniedException("You do not have permission to view this lookup")
header = json.loads(content, object_pairs_hook=OrderedDict)
fields = ['_key']
for field in header['entry'][0]['content']:
if field.startswith('field.'):
fields.append(field[6:])
# If we couldn't get fields from the collections config, try to get it from transforms.conf
if len(fields) <= 1:
fields = self.get_kv_fields_from_transform(session_key, lookup_file, namespace, owner)
# See if we got the fields from the transform. If we didn't, then just assume the _key field exists
if fields is None:
self.logger.info('Unable to get the fields list from lookup transforms', fields)
fields = ['_key']
else:
self.logger.info('Got fields list from the transform, fields=%r', fields)
# Add the fields as the first row
lookup_contents.append(fields)
# Get the contents
response, content = splunk.rest.simpleRequest('/servicesNS/' + owner + '/' + namespace +
'/storage/collections/data/' + lookup_file,
sessionKey=session_key,
getargs={'output_mode': 'json'})
if response.status == 403:
raise PermissionDeniedException("You do not have permission to view this lookup")
rows = json.loads(content, object_pairs_hook=OrderedDict)
for row in rows:
new_row = []
# Convert the JSON style format of the row and convert it down to chunk of text
flattened_row = flatten_dict(row, fields=fields)
# Add each field to the table row
for field in fields:
# If the field was found, add it
if field in flattened_row:
new_row.append(flattened_row[field])
# If the field wasn't found, add a blank string. We need to do this to make
# sure that the number of columns is consistent. We can't have fewer data
# columns than we do header columns. Otherwise, the header won't line up with
# the field since the number of columns items in the header won't match the
# number of columns in the rows.
else:
new_row.append("")
lookup_contents.append(new_row)
return lookup_contents
def get_kv_lookup(self, session_key, lookup_file, namespace="lookup_editor", owner=None):
"""
Get the contents of a KV store lookup.
"""
if owner is None:
owner = 'nobody'
lookup_contents = []
# Get the fields so that we can compose the header
# Note: this call must be done with the user context of "nobody".
response, content = splunk.rest.simpleRequest('/servicesNS/nobody/' + namespace +
'/storage/collections/config/' +
lookup_file,
sessionKey=session_key,
getargs={'output_mode': 'json'})
if response.status == 403:
raise PermissionDeniedException("You do not have permission to view this lookup")
header = json.loads(content)
fields = ['_key']
for field in header['entry'][0]['content']:
if field.startswith('field.'):
fields.append(field[6:])
lookup_contents.append(fields)
# Get the contents
response, content = splunk.rest.simpleRequest('/servicesNS/' + owner + '/' + namespace +
'/storage/collections/data/' + lookup_file,
sessionKey=session_key,
getargs={'output_mode': 'json'})
if response.status == 403:
raise PermissionDeniedException("You do not have permission to view this lookup")
rows = json.loads(content)
for row in rows:
new_row = []
# Convert the JSON style format of the row and convert it down to chunk of text
flattened_row = flatten_dict(row, fields=fields)
# Add each field to the table row
for field in fields:
# If the field was found, add it
if field in flattened_row:
new_row.append(flattened_row[field])
# If the field wasn't found, add a blank string. We need to do this to make
# sure that the number of columns is consistent. We can't have fewer data
# columns than we do header columns. Otherwise, the header won't line up with
# the field since the number of columns items in the header won't match the
# number of columns in the rows.
else:
new_row.append("")
lookup_contents.append(new_row)
return lookup_contents
def get_kv_lookup(self,
session_key,
lookup_file,
namespace="lookup_editor",
owner=None):
"""
Get the contents of a KV store lookup.
"""
if owner is None:
owner = 'nobody'
lookup_contents = []
# Get the fields so that we can compose the header
# Note: this call must be done with the user context of "nobody".
response, content = simpleRequest('/servicesNS/nobody/' + namespace +
'/storage/collections/config/' +
lookup_file,
sessionKey=session_key,
getargs={'output_mode': 'json'})
if response.status == 403:
raise PermissionDeniedException(
"You do not have permission to view this lookup")
header = json.loads(content, object_pairs_hook=OrderedDict)
fields = ['_key']
for field in header['entry'][0]['content']:
if field.startswith('field.'):
fields.append(field[6:])
# If we couldn't get fields from the collections config, try to get it from transforms.conf
if len(fields) <= 1:
fields = self.get_kv_fields_from_transform(session_key,
lookup_file, namespace,
owner)
# See if we got the fields from the transform. If we didn't, then just assume the _key field exists
if fields is None:
self.logger.info(
'Unable to get the fields list from lookup transforms',
fields)
fields = ['_key']
else:
self.logger.info(
'Got fields list from the transform, fields=%r', fields)
# Add the fields as the first row
lookup_contents.append(fields)
# Get the contents
response, content = simpleRequest(
'/servicesNS/' + owner + '/' + namespace +
'/storage/collections/data/' + lookup_file,
sessionKey=session_key,
getargs={'output_mode': 'json'})
if response.status == 403:
raise PermissionDeniedException(
"You do not have permission to view this lookup")
rows = json.loads(content, object_pairs_hook=OrderedDict)
for row in rows:
new_row = []
# Convert the JSON style format of the row and convert it down to chunk of text
flattened_row = flatten_dict(row, fields=fields)
# Add each field to the table row
for field in fields:
# If the field was found, add it
if field in flattened_row:
new_row.append(flattened_row[field])
# If the field wasn't found, add a blank string. We need to do this to make
# sure that the number of columns is consistent. We can't have fewer data
# columns than we do header columns. Otherwise, the header won't line up with
# the field since the number of columns items in the header won't match the
# number of columns in the rows.
else:
new_row.append("")
lookup_contents.append(new_row)
return lookup_contents