def events_over_time(self, event_name = "", time_range = TimeRange.MONTH, property = ""):
query = 'search index=%s application=%s event="%s" | timechart span=%s count by %s | fields - _span*' % (
self.index, self.application_name, (event_name or "*"),
time_range,
(PROPERTY_PREFIX + property) if property else "event",
)
job = self.splunk.jobs.create(query, exec_mode="blocking")
over_time = {}
reader = results.ResultsReader(job.results())
for result in reader:
if isinstance(result, dict):
# Get the time for this entry
time = result["_time"]
del result["_time"]
# The rest is in the form of [event/property]:count
# pairs, so we decode those
for key,count in six.iteritems(result):
# Ignore internal ResultsReader properties
if key.startswith("$"):
continue
entry = over_time.get(key, [])
entry.append({
"count": int(count or 0),
"time": time,
})
over_time[key] = entry
return over_time
def stream_events(self, inputs, ew):
"""This function handles all the action: splunk calls this modular input
without arguments, streams XML describing the inputs to stdin, and waits
for XML on stdout describing events.
If you set use_single_instance to True on the scheme in get_scheme, it
will pass all the instances of this input to a single instance of this
script.
:param inputs: an InputDefinition object
:param ew: an EventWriter object
"""
# Go through each input for this modular input
for input_name, input_item in six.iteritems(inputs.inputs):
# Get the values, cast them as floats
minimum = float(input_item["min"])
maximum = float(input_item["max"])
# Create an Event object, and set its data fields
event = Event()
event.stanza = input_name
event.data = "number=\"%s\"" % str(random.uniform(minimum, maximum))
# Tell the EventWriter to write this event
ew.write_event(event)
def list(self, opts):
"""List all confs or if a conf is given, all the stanzas in it."""
argv = opts.args
count = len(argv)
# unflagged arguments are conf, stanza, key. In this order
# but all are optional
cpres = True if count > 0 else False
spres = True if count > 1 else False
kpres = True if count > 2 else False
if not cpres:
# List out the available confs
for conf in self.service.confs:
print(conf.name)
else:
# Print out detail on the requested conf
# check for optional stanza, or key requested (or all)
name = argv[0]
conf = self.service.confs[name]
for stanza in conf:
if (spres and argv[1] == stanza.name) or not spres:
print("[%s]" % stanza.name)
for key, value in six.iteritems(stanza.content):
if (kpres and argv[2] == key) or not kpres:
print("%s = %s" % (key, value))
print()
def write_metadata(self, configuration):
self._ensure_validity()
metadata = chain(six.iteritems(configuration), (('inspector', self._inspector if self._inspector else None),))
self._write_chunk(metadata, '')
self._ofile.write('\n')
self._clear()
def main(argv):
stdin_wrapper = Reader(sys.stdin)
buf, settings = read_input(stdin_wrapper, has_header = True)
events = csv.DictReader(buf)
hashtags = OrderedDict()
for event in events:
# For each event,
text = event["text"]
hash_regex = re.compile(r'\s+(#[0-9a-zA-Z+_]+)', re.IGNORECASE)
for hashtag_match in hash_regex.finditer(text):
hashtag = hashtag_match.group(0).strip().lower()
hashtag_count = 0
if hashtag in hashtags:
hashtag_count = hashtags[hashtag]
hashtags[hashtag] = hashtag_count + 1
num_hashtags = sum(hashtags.values())
from decimal import Decimal
results = []
for k, v in six.iteritems(hashtags):
results.insert(0, {
"hashtag": k,
"count": v,
"percentage": (Decimal(v) / Decimal(num_hashtags))
})
# And output it to the next stage of the pipeline
output_results(results)
def test_record_writer_with_recordings(self):
cls = self.__class__
method = cls.test_record_writer_with_recordings
base_path = os.path.join(self._recordings_path, '.'.join((cls.__name__, method.__name__)))
for input_file in iglob(base_path + '*.input.gz'):
with gzip.open(input_file, 'rb') as ifile:
test_data = pickle.load(ifile)
writer = RecordWriterV2(StringIO(), maxresultrows=10) # small for the purposes of this unit test
write_record = writer.write_record
fieldnames = test_data['fieldnames']
for values in test_data['values']:
record = OrderedDict(izip(fieldnames, values))
try:
write_record(record)
except Exception as error:
self.fail(error)
for message_type, message_text in test_data['messages']:
writer.write_message(message_type, '{}', message_text)
for name, metric in six.iteritems(test_data['metrics']):
writer.write_metric(name, metric)
writer.flush(finished=True)
# Read expected data
expected_path = os.path.splitext(os.path.splitext(input_file)[0])[0] + '.output'
with io.open(expected_path, 'rb') as ifile:
expected = ifile.read()
expected = self._load_chunks(StringIO(expected))
# Read observed data
ifile = writer._ofile
ifile.seek(0)
observed = self._load_chunks(ifile)
# Write observed data (as an aid to diagnostics)
observed_path = expected_path + '.observed'
observed_value = ifile.getvalue()
with io.open(observed_path, 'wb') as ifile:
ifile.write(observed_value)
self._compare_chunks(observed, expected)
return
def _copy_package_data(self):
for directory, path_list in six.iteritems(self.distribution.package_data):
target = os.path.join(self.build_dir, directory)
if not os.path.isdir(target):
os.makedirs(target)
for path in path_list:
for source in glob(path):
if os.path.isfile(source):
shutil.copy(source, target)
pass
pass
pass
return
def test_confs(self):
confs = self.app_service.confs
conf_name = testlib.tmpname()
self.assertRaises(KeyError, confs.__getitem__, conf_name)
self.assertFalse(conf_name in confs)
conf = confs.create(conf_name)
self.assertTrue(conf_name in confs)
self.assertEqual(conf.name, conf_name)
# New conf should be empty
stanzas = conf.list()
self.assertEqual(len(stanzas), 0)
# Creating a stanza works
count = len(conf)
stanza_name = testlib.tmpname()
stanza = conf.create(stanza_name)
self.assertEqual(len(conf), count+1)
self.assertTrue(stanza_name in conf)
# New stanzas are empty
self.assertEqual(len(stanza), 0)
# Update works
key = testlib.tmpname()
val = testlib.tmpname()
stanza.update(**{key: val})
self.assertEventuallyTrue(lambda: stanza.refresh() and len(stanza) == 1, pause_time=0.2)
self.assertEqual(len(stanza), 1)
self.assertTrue(key in stanza)
values = {testlib.tmpname(): testlib.tmpname(),
testlib.tmpname(): testlib.tmpname()}
stanza.submit(values)
stanza.refresh()
for key, value in six.iteritems(values):
self.assertTrue(key in stanza)
self.assertEqual(value, stanza[key])
count = len(conf)
conf.delete(stanza_name)
self.assertFalse(stanza_name in conf)
self.assertEqual(len(conf), count-1)
# Can't actually delete configuration files directly, at least
# not in current versions of Splunk.
self.assertRaises(client.IllegalOperationException, confs.delete, conf_name)
self.assertTrue(conf_name in confs)
def encode(props):
encoded = " "
for k,v in six.iteritems(props):
# We disallow dictionaries - it doesn't quite make sense.
assert(not isinstance(v, dict))
# We do not allow lists
assert(not isinstance(v, list))
# This is a hack to escape quotes
if isinstance(v, str):
v = v.replace('"', "'")
encoded += ('%s%s="%s" ' % (PROPERTY_PREFIX, k, v))
return encoded
def _map(metadata_map):
metadata = {}
for name, value in six.iteritems(metadata_map):
if isinstance(value, dict):
value = _map(value)
else:
transform, extract = value
if extract is None:
value = None
else:
value = extract(source)
if not (value is None or transform is None):
value = transform(value)
metadata[name] = value
return ObjectView(metadata)
def _object_hook(dictionary):
object_view = ObjectView(dictionary)
stack = deque()
stack.append((None, None, dictionary))
while len(stack):
instance, member_name, dictionary = stack.popleft()
for name, value in six.iteritems(dictionary):
if isinstance(value, dict):
stack.append((dictionary, name, value))
if instance is not None:
instance[member_name] = ObjectView(dictionary)
return object_view
def dslice(value, *args):
"""Returns a 'slice' of the given dictionary value containing only the
requested keys. The keys can be requested in a variety of ways, as an
arg list of keys, as a list of keys, or as a dict whose key(s) represent
the source keys and whose corresponding values represent the resulting
key(s) (enabling key rename), or any combination of the above."""
result = {}
for arg in args:
if isinstance(arg, dict):
for k, v in six.iteritems(arg):
if k in value:
result[v] = value[k]
elif isinstance(arg, list):
for k in arg:
if k in value:
result[k] = value[k]
else:
if arg in value:
result[arg] = value[arg]
return result
def load_elem(element, nametable=None):
name = localname(element.tag)
attrs = load_attrs(element)
value = load_value(element, nametable)
if attrs is None: return name, value
if value is None: return name, attrs
# If value is simple, merge into attrs dict using special key
if isinstance(value, six.string_types):
attrs["$text"] = value
return name, attrs
# Both attrs & value are complex, so merge the two dicts, resolving collisions.
collision_keys = []
for key, val in six.iteritems(attrs):
if key in value and key in collision_keys:
value[key].append(val)
elif key in value and key not in collision_keys:
value[key] = [value[key], val]
collision_keys.append(key)
else:
value[key] = val
return name, value
def __getitem__(self, key):
if key in self:
return dict.__getitem__(self, key)
key += self.sep
result = record()
for k,v in six.iteritems(self):
if not k.startswith(key):
continue
suffix = k[len(key):]
if '.' in suffix:
ks = suffix.split(self.sep)
z = result
for x in ks[:-1]:
if x not in z:
z[x] = record()
z = z[x]
z[ks[-1]] = v
else:
result[suffix] = v
if len(result) == 0:
raise KeyError("No key or prefix: %s" % key)
return result
def properties(self, event_name):
query = 'search index=%s application=%s event="%s" | stats dc(%s*) as *' % (
self.index, self.application_name, event_name, PROPERTY_PREFIX
)
job = self.splunk.jobs.create(query, exec_mode="blocking")
properties = []
reader = results.ResultsReader(job.results())
for result in reader:
if not isinstance(result, dict):
continue
for field, count in six.iteritems(result):
# Ignore internal ResultsReader properties
if field.startswith("$"):
continue
properties.append({
"name": field,
"count": int(count or 0)
})
return properties
def flatten(value, prefix=None):
"""Takes an arbitrary JSON(ish) object and 'flattens' it into a dict
with values consisting of either simple types or lists of simple
types."""
def issimple(value): # foldr(True, or, value)?
for item in value:
if isinstance(item, dict) or isinstance(item, list):
return False
return True
if isinstance(value, six.text_type):
return value.encode("utf8")
if isinstance(value, list):
if issimple(value): return value
offset = 0
result = {}
prefix = "%d" if prefix is None else "%s_%%d" % prefix
for item in value:
k = prefix % offset
v = flatten(item, k)
if not isinstance(v, dict): v = {k:v}
result.update(v)
offset += 1
return result
if isinstance(value, dict):
result = {}
prefix = "%s" if prefix is None else "%s_%%s" % prefix
for k, v in six.iteritems(value):
k = prefix % str(k)
v = flatten(v, k)
if not isinstance(v, dict): v = {k:v}
result.update(v)
return result
return value
def _process_protocol_v1(self, argv, ifile, ofile):
debug = environment.splunklib_logger.debug
class_name = self.__class__.__name__
debug('%s.process started under protocol_version=1', class_name)
self._record_writer = RecordWriterV1(ofile)
# noinspection PyBroadException
try:
if argv[1] == '__GETINFO__':
debug('Writing configuration settings')
ifile = self._prepare_protocol_v1(argv, ifile, ofile)
self._record_writer.write_record(dict(
(n, ','.join(v) if isinstance(v, (list, tuple)) else v) for n, v in six.iteritems(self._configuration)))
self.finish()
elif argv[1] == '__EXECUTE__':
debug('Executing')
ifile = self._prepare_protocol_v1(argv, ifile, ofile)
self._records = self._records_protocol_v1
self._metadata.action = 'execute'
self._execute(ifile, None)
else:
message = (
'Command {0} appears to be statically configured for search command protocol version 1 and static '
'configuration is unsupported by splunklib.searchcommands. Please ensure that '
'default/commands.conf contains this stanza:\n'
'[{0}]\n'
'filename = {1}\n'
'enableheader = true\n'
'outputheader = true\n'
'requires_srinfo = true\n'
'supports_getinfo = true\n'
'supports_multivalues = true\n'
'supports_rawargs = true'.format(self.name, os.path.basename(argv[0])))
raise RuntimeError(message)
except (SyntaxError, ValueError) as error:
self.write_error(six.text_type(error))
self.flush()
exit(0)
except SystemExit:
self.flush()
raise
except:
self._report_unexpected_error()
self.flush()
exit(1)
debug('%s.process finished under protocol_version=1', class_name)
def load_attrs(element):
if not hasattrs(element): return None
attrs = record()
for key, value in six.iteritems(element.attrib):
attrs[key] = value
return attrs
def __str__(self):
""" Converts the value of this instance to its string representation.
The value of this ConfigurationSettings instance is represented as a string of comma-separated
:code:`name=value` pairs. Items with values of :const:`None` are filtered from the list.
:return: String representation of this instance
"""
#text = ', '.join(imap(lambda (name, value): name + '=' + json_encode_string(unicode(value)), self.iteritems()))
text = ', '.join(['{}={}'.format(name, json_encode_string(six.text_type(value))) for (name, value) in six.iteritems(self)])
return text
def test_namespace(self):
tests = [({}, {
'sharing': None,
'owner': None,
'app': None
}), ({
'owner': "Bob"
}, {
'sharing': None,
'owner': "Bob",
'app': None
}),
({
'app': "search"
}, {
'sharing': None,
'owner': None,
'app': "search"
}),
({
'owner': "Bob",
'app': "search"
}, {
'sharing': None,
'owner': "Bob",
'app': "search"
}),
({
'sharing': "user",
'owner': "*****@*****.**"
}, {
'sharing': "user",
'owner': "*****@*****.**",
'app': None
}),
({
'sharing': "user"
}, {
'sharing': "user",
'owner': None,
'app': None
}),
({
'sharing': "user",
'owner': "Bob"
}, {
'sharing': "user",
'owner': "Bob",
'app': None
}),
({
'sharing': "user",
'app': "search"
}, {
'sharing': "user",
'owner': None,
'app': "search"
}),
({
'sharing': "user",
'owner': "Bob",
'app': "search"
}, {
'sharing': "user",
'owner': "Bob",
'app': "search"
}),
({
'sharing': "app"
}, {
'sharing': "app",
'owner': "nobody",
'app': None
}),
({
'sharing': "app",
'owner': "Bob"
}, {
'sharing': "app",
'owner': "nobody",
'app': None
}),
({
'sharing': "app",
'app': "search"
}, {
'sharing': "app",
'owner': "nobody",
'app': "search"
}),
({
'sharing': "app",
'owner': "Bob",
'app': "search"
}, {
'sharing': "app",
'owner': "nobody",
'app': "search"
}),
({
'sharing': "global"
}, {
'sharing': "global",
'owner': "nobody",
'app': None
}),
({
'sharing': "global",
'owner': "Bob"
}, {
'sharing': "global",
'owner': "nobody",
'app': None
}),
({
'sharing': "global",
'app': "search"
}, {
'sharing': "global",
'owner': "nobody",
'app': "search"
}),
({
'sharing': "global",
'owner': "Bob",
'app': "search"
}, {
'sharing': "global",
'owner': "nobody",
'app': "search"
}),
({
'sharing': "system"
}, {
'sharing': "system",
'owner': "nobody",
'app': "system"
}),
({
'sharing': "system",
'owner': "Bob"
}, {
'sharing': "system",
'owner': "nobody",
'app': "system"
}),
({
'sharing': "system",
'app': "search"
}, {
'sharing': "system",
'owner': "nobody",
'app': "system"
}),
({
'sharing': "system",
'owner': "Bob",
'app': "search"
}, {
'sharing': "system",
'owner': "nobody",
'app': "system"
}),
({
'sharing': 'user',
'owner': '-',
'app': '-'
}, {
'sharing': 'user',
'owner': '-',
'app': '-'
})]
for kwargs, expected in tests:
namespace = binding.namespace(**kwargs)
for k, v in six.iteritems(expected):
self.assertEqual(namespace[k], v)
def fix_up(cls, values):
is_configuration_setting = lambda attribute: isinstance(attribute, ConfigurationSetting)
definitions = getmembers(cls, is_configuration_setting)
i = 0
for name, setting in definitions:
if setting._name is None:
setting._name = name = six.text_type(name)
else:
name = setting._name
validate, specification = setting._get_specification()
backing_field_name = '_' + name
if setting.fget is None and setting.fset is None and setting.fdel is None:
value = setting._value
if setting._readonly or value is not None:
validate(specification, name, value)
def fget(bfn, value):
return lambda this: getattr(this, bfn, value)
setting = setting.getter(fget(backing_field_name, value))
if not setting._readonly:
def fset(bfn, validate, specification, name):
return lambda this, value: setattr(this, bfn, validate(specification, name, value))
setting = setting.setter(fset(backing_field_name, validate, specification, name))
setattr(cls, name, setting)
def is_supported_by_protocol(supporting_protocols):
def is_supported_by_protocol(version):
return version in supporting_protocols
return is_supported_by_protocol
del setting._name, setting._value, setting._readonly
setting.is_supported_by_protocol = is_supported_by_protocol(specification.supporting_protocols)
setting.supporting_protocols = specification.supporting_protocols
setting.backing_field_name = backing_field_name
definitions[i] = setting
setting.name = name
i += 1
try:
value = values[name]
except KeyError:
continue
if setting.fset is None:
raise ValueError('The value of configuration setting {} is fixed'.format(name))
setattr(cls, backing_field_name, validate(specification, name, value))
del values[name]
if len(values) > 0:
settings = sorted(list(six.iteritems(values)))
settings = imap(lambda n_v: '{}={}'.format(n_v[0], repr(n_v[1])), settings)
raise AttributeError('Inapplicable configuration settings: ' + ', '.join(settings))
cls.configuration_setting_definitions = definitions
def __str__(self):
""" Converts the value of this instance to its string representation.
The value of this ConfigurationSettings instance is represented as a string of comma-separated
:code:`name=value` pairs. Items with values of :const:`None` are filtered from the list.
:return: String representation of this instance
"""
#text = ', '.join(imap(lambda (name, value): name + '=' + json_encode_string(unicode(value)), self.iteritems()))
text = ', '.join(['{}={}'.format(name, json_encode_string(six.text_type(value))) for (name, value) in six.iteritems(self)])
return text
def test_input_header(self):
# No items
input_header = InputHeader()
with closing(StringIO('\r\n')) as input_file:
input_header.read(input_file)
self.assertEqual(len(input_header), 0)
# One unnamed single-line item (same as no items)
input_header = InputHeader()
with closing(
StringIO('this%20is%20an%20unnamed%20single-line%20item\n\n')
) as input_file:
input_header.read(input_file)
self.assertEqual(len(input_header), 0)
input_header = InputHeader()
with closing(
StringIO('this%20is%20an%20unnamed\nmulti-\nline%20item\n\n')
) as input_file:
input_header.read(input_file)
self.assertEqual(len(input_header), 0)
# One named single-line item
input_header = InputHeader()
with closing(StringIO(
'Foo:this%20is%20a%20single-line%20item\n\n')) as input_file:
input_header.read(input_file)
self.assertEqual(len(input_header), 1)
self.assertEqual(input_header['Foo'], 'this is a single-line item')
input_header = InputHeader()
with closing(StringIO(
'Bar:this is a\nmulti-\nline item\n\n')) as input_file:
input_header.read(input_file)
self.assertEqual(len(input_header), 1)
self.assertEqual(input_header['Bar'], 'this is a\nmulti-\nline item')
# The infoPath item (which is the path to a file that we open for reads)
input_header = InputHeader()
with closing(StringIO('infoPath:non-existent.csv\n\n')) as input_file:
input_header.read(input_file)
self.assertEqual(len(input_header), 1)
self.assertEqual(input_header['infoPath'], 'non-existent.csv')
# Set of named items
collection = {
'word_list': 'hello\nworld\n!',
'word_1': 'hello',
'word_2': 'world',
'word_3': '!',
'sentence': 'hello world!'
}
input_header = InputHeader()
text = reduce(
lambda value, item: value + '{}:{}\n'.format(item[0], item[1]),
six.iteritems(collection), '') + '\n'
with closing(StringIO(text)) as input_file:
input_header.read(input_file)
self.assertDictEqual(input_header, collection)
# Set of named items with an unnamed item at the beginning (the only place that an unnamed item can appear)
with closing(StringIO('unnamed item\n' + text)) as input_file:
input_header.read(input_file)
self.assertDictEqual(input_header, collection)
# Test iterators, indirectly through items, keys, and values
self.assertEqual(sorted(input_header.items()),
sorted(collection.items()))
self.assertEqual(sorted(input_header.keys()),
sorted(collection.keys()))
self.assertEqual(sorted(input_header.values()),
sorted(collection.values()))
return
def __str__(self):
return '\n'.join(
[name + ':' + value for name, value in six.iteritems(self)])
def load_attrs(element):
if not hasattrs(element): return None
attrs = record()
for key, value in six.iteritems(element.attrib):
attrs[key] = value
return attrs
def fix_up(cls, values):
is_configuration_setting = lambda attribute: isinstance(
attribute, ConfigurationSetting)
definitions = getmembers(cls, is_configuration_setting)
i = 0
for name, setting in definitions:
if setting._name is None:
setting._name = name = six.text_type(name)
else:
name = setting._name
validate, specification = setting._get_specification()
backing_field_name = '_' + name
if setting.fget is None and setting.fset is None and setting.fdel is None:
value = setting._value
if setting._readonly or value is not None:
validate(specification, name, value)
def fget(bfn, value):
return lambda this: getattr(this, bfn, value)
setting = setting.getter(fget(backing_field_name, value))
if not setting._readonly:
def fset(bfn, validate, specification, name):
return lambda this, value: setattr(
this, bfn, validate(specification, name, value))
setting = setting.setter(
fset(backing_field_name, validate, specification,
name))
setattr(cls, name, setting)
def is_supported_by_protocol(supporting_protocols):
def is_supported_by_protocol(version):
return version in supporting_protocols
return is_supported_by_protocol
del setting._name, setting._value, setting._readonly
setting.is_supported_by_protocol = is_supported_by_protocol(
specification.supporting_protocols)
setting.supporting_protocols = specification.supporting_protocols
setting.backing_field_name = backing_field_name
definitions[i] = setting
setting.name = name
i += 1
try:
value = values[name]
except KeyError:
continue
if setting.fset is None:
raise ValueError(
'The value of configuration setting {} is fixed'.format(
name))
setattr(cls, backing_field_name,
validate(specification, name, value))
del values[name]
if len(values) > 0:
settings = sorted(list(six.iteritems(values)))
settings = imap(lambda n_v: '{}={}'.format(n_v[0], repr(n_v[1])),
settings)
raise AttributeError('Inapplicable configuration settings: ' +
', '.join(settings))
cls.configuration_setting_definitions = definitions
def test_record_writer_with_random_data(self, save_recording=False):
# Confirmed: [minint, maxint) covers the full range of values that xrange allows
# RecordWriter writes apps in units of maxresultrows records. Default: 50,0000.
# Partial results are written when the record count reaches maxresultrows.
writer = RecordWriterV2(
BytesIO(),
maxresultrows=10) # small for the purposes of this unit test
test_data = OrderedDict()
fieldnames = [
'_serial', '_time', 'random_bytes', 'random_dict',
'random_integers', 'random_unicode'
]
test_data['fieldnames'] = fieldnames
test_data['values'] = []
write_record = writer.write_record
for serial_number in range(0, 31):
values = [
serial_number,
time(),
random_bytes(),
random_dict(),
random_integers(),
random_unicode()
]
record = OrderedDict(izip(fieldnames, values))
#try:
write_record(record)
#except Exception as error:
# self.fail(error)
test_data['values'].append(values)
# RecordWriter accumulates inspector messages and metrics until maxresultrows are written, a partial result
# is produced or we're finished
messages = [('debug', random_unicode()), ('error', random_unicode()),
('fatal', random_unicode()), ('info', random_unicode()),
('warn', random_unicode())]
test_data['messages'] = messages
for message_type, message_text in messages:
writer.write_message(message_type, '{}', message_text)
metrics = {
'metric-1': SearchMetric(1, 2, 3, 4),
'metric-2': SearchMetric(5, 6, 7, 8)
}
test_data['metrics'] = metrics
for name, metric in six.iteritems(metrics):
writer.write_metric(name, metric)
self.assertEqual(writer._chunk_count, 3)
self.assertEqual(writer._record_count, 1)
self.assertGreater(writer._buffer.tell(), 0)
self.assertEqual(writer._total_record_count, 30)
self.assertListEqual(writer._fieldnames, fieldnames)
self.assertListEqual(writer._inspector['messages'], messages)
self.assertDictEqual(
dict(
ifilter(lambda k_v: k_v[0].startswith('metric.'),
six.iteritems(writer._inspector))),
dict(
imap(lambda k_v1: ('metric.' + k_v1[0], k_v1[1]),
six.iteritems(metrics))))
writer.flush(finished=True)
self.assertEqual(writer._chunk_count, 4)
self.assertEqual(writer._record_count, 0)
self.assertEqual(writer._buffer.tell(), 0)
self.assertEqual(writer._buffer.getvalue(), '')
self.assertEqual(writer._total_record_count, 31)
self.assertRaises(AssertionError,
writer.flush,
finished=True,
partial=True)
self.assertRaises(AssertionError, writer.flush, finished='non-boolean')
self.assertRaises(AssertionError, writer.flush, partial='non-boolean')
self.assertRaises(AssertionError, writer.flush)
self.assertRaises(RuntimeError, writer.write_record, {})
self.assertFalse(writer._ofile.closed)
self.assertIsNone(writer._fieldnames)
self.assertDictEqual(writer._inspector, OrderedDict())
# P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to write partial results by calling
# RecordWriter.flush(partial=True).
# P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to finish early by calling
# RecordWriter.flush(finish=True).
return
def test_input_header(self):
# No items
input_header = InputHeader()
with closing(StringIO('\r\n'.encode())) as input_file:
input_header.read(input_file)
self.assertEquals(len(input_header), 0)
# One unnamed single-line item (same as no items)
input_header = InputHeader()
with closing(StringIO('this%20is%20an%20unnamed%20single-line%20item\n\n'.encode())) as input_file:
input_header.read(input_file)
self.assertEquals(len(input_header), 0)
input_header = InputHeader()
with closing(StringIO('this%20is%20an%20unnamed\nmulti-\nline%20item\n\n'.encode())) as input_file:
input_header.read(input_file)
self.assertEquals(len(input_header), 0)
# One named single-line item
input_header = InputHeader()
with closing(StringIO('Foo:this%20is%20a%20single-line%20item\n\n'.encode())) as input_file:
input_header.read(input_file)
self.assertEquals(len(input_header), 1)
self.assertEquals(input_header['Foo'], 'this is a single-line item')
input_header = InputHeader()
with closing(StringIO('Bar:this is a\nmulti-\nline item\n\n'.encode())) as input_file:
input_header.read(input_file)
self.assertEquals(len(input_header), 1)
self.assertEquals(input_header['Bar'], 'this is a\nmulti-\nline item')
# The infoPath item (which is the path to a file that we open for reads)
input_header = InputHeader()
with closing(StringIO('infoPath:non-existent.csv\n\n'.encode())) as input_file:
input_header.read(input_file)
self.assertEquals(len(input_header), 1)
self.assertEqual(input_header['infoPath'], 'non-existent.csv')
# Set of named items
collection = {
'word_list': 'hello\nworld\n!',
'word_1': 'hello',
'word_2': 'world',
'word_3': '!',
'sentence': 'hello world!'}
input_header = InputHeader()
text = reduce(lambda value, item: value + '{}:{}\n'.format(item[0], item[1]), six.iteritems(collection), '') + '\n'
with closing(StringIO(text.encode())) as input_file:
input_header.read(input_file)
self.assertDictEqual(input_header, collection)
# Set of named items with an unnamed item at the beginning (the only place that an unnamed item can appear)
with closing(StringIO(('unnamed item\n' + text).encode())) as input_file:
input_header.read(input_file)
self.assertDictEqual(input_header, collection)
# Test iterators, indirectly through items, keys, and values
self.assertEqual(sorted(input_header.items()), sorted(collection.items()))
self.assertEqual(sorted(input_header.keys()), sorted(collection.keys()))
self.assertEqual(sorted(input_header.values()), sorted(collection.values()))
return
def test_streaming_command(self):
from splunklib.searchcommands import Configuration, StreamingCommand
@Configuration()
class TestCommand(StreamingCommand):
def stream(self, records):
pass
command = TestCommand()
command._protocol_version = 1
self.assertEqual(
[(name, value) for name, value in six.iteritems(command.configuration)],
[('streaming', True)])
self.assertIs(command.configuration.clear_required_fields, None)
self.assertIs(command.configuration.local, None)
self.assertIs(command.configuration.overrides_timeorder, None)
self.assertIs(command.configuration.required_fields, None)
self.assertIs(command.configuration.streaming, True)
command.configuration.clear_required_fields = True
command.configuration.local = True
command.configuration.overrides_timeorder = True
command.configuration.required_fields = ['field_1', 'field_2', 'field_3']
try:
command.configuration.streaming = False
except AttributeError:
pass
except Exception as error:
self.fail('Expected AttributeError, not {}: {}'.format(type(error).__name__, error))
else:
self.fail('Expected AttributeError')
self.assertEqual(
[(name, value) for name, value in six.iteritems(command.configuration)],
[('clear_required_fields', True), ('local', True), ('overrides_timeorder', True), ('required_fields', ['field_1', 'field_2', 'field_3']), ('streaming', True)])
command = TestCommand()
command._protocol_version = 2
self.assertEqual(
[(name, value) for name, value in six.iteritems(command.configuration)],
[('type', 'streaming')])
self.assertIs(command.configuration.distributed, True)
self.assertEqual(command.configuration.type, 'streaming')
command.configuration.distributed = False
command.configuration.required_fields = ['field_1', 'field_2', 'field_3']
try:
command.configuration.type = 'events'
except AttributeError:
pass
except Exception as error:
self.fail('Expected AttributeError, not {}: {}'.format(type(error).__name__, error))
else:
self.fail('Expected AttributeError')
self.assertEqual(
[(name, value) for name, value in six.iteritems(command.configuration)],
[('required_fields', ['field_1', 'field_2', 'field_3']), ('type', 'stateful')])
return
def __str__(self):
return '\n'.join([name + ':' + value for name, value in six.iteritems(self)])
def test_record_writer_with_random_data(self, save_recording=False):
# Confirmed: [minint, maxint) covers the full range of values that xrange allows
# RecordWriter writes apps in units of maxresultrows records. Default: 50,0000.
# Partial results are written when the record count reaches maxresultrows.
writer = RecordWriterV2(StringIO(), maxresultrows=10) # small for the purposes of this unit test
test_data = OrderedDict()
fieldnames = ['_serial', '_time', 'random_bytes', 'random_dict', 'random_integers', 'random_unicode']
test_data['fieldnames'] = fieldnames
test_data['values'] = []
write_record = writer.write_record
for serial_number in range(0, 31):
values = [serial_number, time(), random_bytes(), random_dict(), random_integers(), random_unicode()]
record = OrderedDict(izip(fieldnames, values))
#try:
write_record(record)
#except Exception as error:
# self.fail(error)
test_data['values'].append(values)
# RecordWriter accumulates inspector messages and metrics until maxresultrows are written, a partial result
# is produced or we're finished
messages = [
('debug', random_unicode()),
('error', random_unicode()),
('fatal', random_unicode()),
('info', random_unicode()),
('warn', random_unicode())]
test_data['messages'] = messages
for message_type, message_text in messages:
writer.write_message(message_type, '{}', message_text)
metrics = {
'metric-1': SearchMetric(1, 2, 3, 4),
'metric-2': SearchMetric(5, 6, 7, 8)
}
test_data['metrics'] = metrics
for name, metric in six.iteritems(metrics):
writer.write_metric(name, metric)
self.assertEqual(writer._chunk_count, 3)
self.assertEqual(writer._record_count, 1)
self.assertGreater(writer._buffer.tell(), 0)
self.assertEqual(writer._total_record_count, 30)
self.assertListEqual(writer._fieldnames, fieldnames)
self.assertListEqual(writer._inspector['messages'], messages)
self.assertDictEqual(
dict(ifilter(lambda k_v: k_v[0].startswith('metric.'), six.iteritems(writer._inspector))),
dict(imap(lambda k_v1: ('metric.' + k_v1[0], k_v1[1]), six.iteritems(metrics))))
writer.flush(finished=True)
self.assertEqual(writer._chunk_count, 4)
self.assertEqual(writer._record_count, 0)
self.assertEqual(writer._buffer.tell(), 0)
self.assertEqual(writer._buffer.getvalue(), '')
self.assertEqual(writer._total_record_count, 31)
self.assertRaises(AssertionError, writer.flush, finished=True, partial=True)
self.assertRaises(AssertionError, writer.flush, finished='non-boolean')
self.assertRaises(AssertionError, writer.flush, partial='non-boolean')
self.assertRaises(AssertionError, writer.flush)
self.assertRaises(RuntimeError, writer.write_record, {})
self.assertFalse(writer._ofile.closed)
self.assertIsNone(writer._fieldnames)
self.assertDictEqual(writer._inspector, OrderedDict())
# P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to write partial results by calling
# RecordWriter.flush(partial=True).
# P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to finish early by calling
# RecordWriter.flush(finish=True).
if save_recording:
cls = self.__class__
method = cls.test_record_writer_with_recordings
base_path = os.path.join(self._recordings_path, '.'.join((cls.__name__, method.__name__, six.text_type(time()))))
with gzip.open(base_path + '.input.gz', 'wb') as f:
pickle.dump(test_data, f)
with open(base_path + '.output', 'wb') as f:
f.write(writer._ofile.getvalue())
return
def check_content(self, entity, **kwargs):
for k, v in six.iteritems(kwargs):
self.assertEqual(entity[k], str(v))
def _process_protocol_v1(self, argv, ifile, ofile):
debug = environment.splunklib_logger.debug
class_name = self.__class__.__name__
debug('%s.process started under protocol_version=1', class_name)
self._record_writer = RecordWriterV1(ofile)
# noinspection PyBroadException
try:
if argv[1] == '__GETINFO__':
debug('Writing configuration settings')
ifile = self._prepare_protocol_v1(argv, ifile, ofile)
self._record_writer.write_record(dict(
(n, ','.join(v) if isinstance(v, (list, tuple)) else v) for n, v in six.iteritems(self._configuration)))
self.finish()
elif argv[1] == '__EXECUTE__':
debug('Executing')
ifile = self._prepare_protocol_v1(argv, ifile, ofile)
self._records = self._records_protocol_v1
self._metadata.action = 'execute'
self._execute(ifile, None)
else:
message = (
'Command {0} appears to be statically configured for search command protocol version 1 and static '
'configuration is unsupported by splunklib.searchcommands. Please ensure that '
'default/commands.conf contains this stanza:\n'
'[{0}]\n'
'filename = {1}\n'
'enableheader = true\n'
'outputheader = true\n'
'requires_srinfo = true\n'
'supports_getinfo = true\n'
'supports_multivalues = true\n'
'supports_rawargs = true'.format(self.name, os.path.basename(argv[0])))
raise RuntimeError(message)
except (SyntaxError, ValueError) as error:
self.write_error(six.text_type(error))
self.flush()
exit(0)
except SystemExit:
self.flush()
raise
except:
self._report_unexpected_error()
self.flush()
exit(1)
debug('%s.process finished under protocol_version=1', class_name)
def test_generating_command(self):
from splunklib.searchcommands import Configuration, GeneratingCommand
@Configuration()
class TestCommand(GeneratingCommand):
def generate(self):
pass
command = TestCommand()
command._protocol_version = 1
self.assertTrue(
[(name, value) for name, value in six.iteritems(command.configuration)],
[('generating', True)])
self.assertIs(command.configuration.generates_timeorder, None)
self.assertIs(command.configuration.generating, True)
self.assertIs(command.configuration.retainsevents, None)
self.assertIs(command.configuration.streaming, None)
command.configuration.generates_timeorder = True
command.configuration.local = True
command.configuration.retainsevents = True
command.configuration.streaming = True
try:
command.configuration.generating = False
except AttributeError:
pass
except Exception as error:
self.fail('Expected AttributeError, not {}: {}'.format(type(error).__name__, error))
else:
self.fail('Expected AttributeError')
self.assertEqual(
[(name, value) for name, value in six.iteritems(command.configuration)],
[('generates_timeorder', True), ('generating', True), ('local', True), ('retainsevents', True),
('streaming', True)])
command = TestCommand()
command._protocol_version = 2
self.assertEqual(
[(name, value) for name, value in six.iteritems(command.configuration)],
[('generating', True), ('type', 'stateful')])
self.assertIs(command.configuration.distributed, False)
self.assertIs(command.configuration.generating, True)
self.assertEqual(command.configuration.type, 'streaming')
command.configuration.distributed = True
try:
command.configuration.generating = False
except AttributeError:
pass
except Exception as error:
self.fail('Expected AttributeError, not {}: {}'.format(type(error).__name__, error))
else:
self.fail('Expected AttributeError')
self.assertEqual(
[(name, value) for name, value in six.iteritems(command.configuration)],
[('generating', True), ('type', 'streaming')])
return
def test_streaming_command(self):
from splunklib.searchcommands import Configuration, StreamingCommand
@Configuration()
class TestCommand(StreamingCommand):
def stream(self, records):
pass
command = TestCommand()
command._protocol_version = 1
self.assertEqual(
[(name, value)
for name, value in six.iteritems(command.configuration)],
[('streaming', True)])
self.assertIs(command.configuration.clear_required_fields, None)
self.assertIs(command.configuration.local, None)
self.assertIs(command.configuration.overrides_timeorder, None)
self.assertIs(command.configuration.required_fields, None)
self.assertIs(command.configuration.streaming, True)
command.configuration.clear_required_fields = True
command.configuration.local = True
command.configuration.overrides_timeorder = True
command.configuration.required_fields = [
'field_1', 'field_2', 'field_3'
]
try:
command.configuration.streaming = False
except AttributeError:
pass
except Exception as error:
self.fail('Expected AttributeError, not {}: {}'.format(
type(error).__name__, error))
else:
self.fail('Expected AttributeError')
self.assertEqual(
[(name, value)
for name, value in six.iteritems(command.configuration)],
[('clear_required_fields', True), ('local', True),
('overrides_timeorder', True),
('required_fields', ['field_1', 'field_2', 'field_3']),
('streaming', True)])
command = TestCommand()
command._protocol_version = 2
self.assertEqual(
[(name, value)
for name, value in six.iteritems(command.configuration)],
[('type', 'streaming')])
self.assertIs(command.configuration.distributed, True)
self.assertEqual(command.configuration.type, 'streaming')
command.configuration.distributed = False
command.configuration.required_fields = [
'field_1', 'field_2', 'field_3'
]
try:
command.configuration.type = 'events'
except AttributeError:
pass
except Exception as error:
self.fail('Expected AttributeError, not {}: {}'.format(
type(error).__name__, error))
else:
self.fail('Expected AttributeError')
self.assertEqual(
[(name, value)
for name, value in six.iteritems(command.configuration)],
[('required_fields', ['field_1', 'field_2', 'field_3']),
('type', 'stateful')])
return
def test_generating_command(self):
from splunklib.searchcommands import Configuration, GeneratingCommand
@Configuration()
class TestCommand(GeneratingCommand):
def generate(self):
pass
command = TestCommand()
command._protocol_version = 1
self.assertTrue(
[(name, value)
for name, value in six.iteritems(command.configuration)],
[('generating', True)])
self.assertIs(command.configuration.generates_timeorder, None)
self.assertIs(command.configuration.generating, True)
self.assertIs(command.configuration.retainsevents, None)
self.assertIs(command.configuration.streaming, None)
command.configuration.generates_timeorder = True
command.configuration.local = True
command.configuration.retainsevents = True
command.configuration.streaming = True
try:
command.configuration.generating = False
except AttributeError:
pass
except Exception as error:
self.fail('Expected AttributeError, not {}: {}'.format(
type(error).__name__, error))
else:
self.fail('Expected AttributeError')
self.assertEqual(
[(name, value)
for name, value in six.iteritems(command.configuration)],
[('generates_timeorder', True), ('generating', True),
('local', True), ('retainsevents', True), ('streaming', True)])
command = TestCommand()
command._protocol_version = 2
self.assertEqual(
[(name, value)
for name, value in six.iteritems(command.configuration)],
[('generating', True), ('type', 'stateful')])
self.assertIs(command.configuration.distributed, False)
self.assertIs(command.configuration.generating, True)
self.assertEqual(command.configuration.type, 'streaming')
command.configuration.distributed = True
try:
command.configuration.generating = False
except AttributeError:
pass
except Exception as error:
self.fail('Expected AttributeError, not {}: {}'.format(
type(error).__name__, error))
else:
self.fail('Expected AttributeError')
self.assertEqual(
[(name, value)
for name, value in six.iteritems(command.configuration)],
[('generating', True), ('type', 'streaming')])
return
def check_content(self, entity, **kwargs):
for k, v in six.iteritems(kwargs):
self.assertEqual(entity[k], str(v))
def test_namespace(self):
tests = [
({ },
{ 'sharing': None, 'owner': None, 'app': None }),
({ 'owner': "Bob" },
{ 'sharing': None, 'owner': "Bob", 'app': None }),
({ 'app': "search" },
{ 'sharing': None, 'owner': None, 'app': "search" }),
({ 'owner': "Bob", 'app': "search" },
{ 'sharing': None, 'owner': "Bob", 'app': "search" }),
({ 'sharing': "user", 'owner': "*****@*****.**" },
{ 'sharing': "user", 'owner': "*****@*****.**", 'app': None }),
({ 'sharing': "user" },
{ 'sharing': "user", 'owner': None, 'app': None }),
({ 'sharing': "user", 'owner': "Bob" },
{ 'sharing': "user", 'owner': "Bob", 'app': None }),
({ 'sharing': "user", 'app': "search" },
{ 'sharing': "user", 'owner': None, 'app': "search" }),
({ 'sharing': "user", 'owner': "Bob", 'app': "search" },
{ 'sharing': "user", 'owner': "Bob", 'app': "search" }),
({ 'sharing': "app" },
{ 'sharing': "app", 'owner': "nobody", 'app': None }),
({ 'sharing': "app", 'owner': "Bob" },
{ 'sharing': "app", 'owner': "nobody", 'app': None }),
({ 'sharing': "app", 'app': "search" },
{ 'sharing': "app", 'owner': "nobody", 'app': "search" }),
({ 'sharing': "app", 'owner': "Bob", 'app': "search" },
{ 'sharing': "app", 'owner': "nobody", 'app': "search" }),
({ 'sharing': "global" },
{ 'sharing': "global", 'owner': "nobody", 'app': None }),
({ 'sharing': "global", 'owner': "Bob" },
{ 'sharing': "global", 'owner': "nobody", 'app': None }),
({ 'sharing': "global", 'app': "search" },
{ 'sharing': "global", 'owner': "nobody", 'app': "search" }),
({ 'sharing': "global", 'owner': "Bob", 'app': "search" },
{ 'sharing': "global", 'owner': "nobody", 'app': "search" }),
({ 'sharing': "system" },
{ 'sharing': "system", 'owner': "nobody", 'app': "system" }),
({ 'sharing': "system", 'owner': "Bob" },
{ 'sharing': "system", 'owner': "nobody", 'app': "system" }),
({ 'sharing': "system", 'app': "search" },
{ 'sharing': "system", 'owner': "nobody", 'app': "system" }),
({ 'sharing': "system", 'owner': "Bob", 'app': "search" },
{ 'sharing': "system", 'owner': "nobody", 'app': "system" }),
({ 'sharing': 'user', 'owner': '-', 'app': '-'},
{ 'sharing': 'user', 'owner': '-', 'app': '-'})]
for kwargs, expected in tests:
namespace = binding.namespace(**kwargs)
for k, v in six.iteritems(expected):
self.assertEqual(namespace[k], v)
