def test_parse_inputdef_with_two_inputs(self):
"""Check parsing of XML that contains 2 inputs"""
found = InputDefinition.parse(data_open("data/conf_with_2_inputs.xml"))
expectedDefinition = InputDefinition()
expectedDefinition.metadata = {
"server_host": "tiny",
"server_uri": "https://127.0.0.1:8089",
"checkpoint_dir": "/some/dir",
"session_key": "123102983109283019283"
}
expectedDefinition.inputs["foobar://aaa"] = {
"param1": "value1",
"param2": "value2",
"disabled": "0",
"index": "default"
}
expectedDefinition.inputs["foobar://bbb"] = {
"param1": "value11",
"param2": "value22",
"disabled": "0",
"index": "default",
"multiValue": ["value1", "value2"],
"multiValue2": ["value3", "value4"]
}
self.assertEqual(expectedDefinition, found)
def test_parse_inputdef_with_zero_inputs(self):
"""Check parsing of XML that contains only metadata"""
found = InputDefinition.parse(data_open("data/conf_with_0_inputs.xml"))
expectedDefinition = InputDefinition()
expectedDefinition.metadata = {
"server_host": "tiny",
"server_uri": "https://127.0.0.1:8089",
"checkpoint_dir": "/some/dir",
"session_key": "123102983109283019283"
}
self.assertEqual(found, expectedDefinition)
def test_parse_inputdef_with_zero_inputs(self):
"""Check parsing of XML that contains only metadata"""
found = InputDefinition.parse(data_open("data/conf_with_0_inputs.xml"))
expectedDefinition = InputDefinition()
expectedDefinition.metadata = {
"server_host": "tiny",
"server_uri": "https://127.0.0.1:8089",
"checkpoint_dir": "/some/dir",
"session_key": "123102983109283019283"
}
self.assertEqual(found, expectedDefinition)
def get_input_definition(self):
'''Get input definition.
This method can be overwritten to get input definition from
other input instead `stdin`.
:returns: A dict object must contains `metadata` and `inputs`,
example: {
'metadata': {
'session_key': 'iCKPS0cvmpyeJk...sdaf',
'server_host': 'test-test.com',
'server_uri': 'https://127.0.0.1:8089',
'checkpoint_dir': '/tmp'
},
inputs: {
'stanza1': {'arg1': value1, 'arg2': value2},
'stanza2': {'arg1': value1, 'arg2': value2}
}
}
:rtype: ``dict``
'''
input_definition = InputDefinition.parse(sys.stdin)
return {
'metadata': input_definition.metadata,
'inputs': input_definition.inputs
}
def test_attempt_to_parse_malformed_input_definition_will_throw_exception(
self):
"""Does malformed XML cause the expected exception."""
with self.assertRaises(ValueError):
found = InputDefinition.parse(
data_open("data/conf_with_invalid_inputs.xml"))
def from_stream(cls, stream):
definition = InputDefinition.parse(stream)
metadata = definition.metadata
parts = urllib.parse.urlparse(metadata['server_uri'])
scheme = parts.scheme
host = parts.hostname
port = parts.port
token = metadata['session_key']
server_scheme = scheme
server_host = host
server_port = port
token = token
log_dir = environ.get_log_folder()
inputs = list()
for name, params in list(definition.inputs.items()):
kind, name = cls._split_stanza(name)
stanza = Stanza(kind, name, params)
inputs.append(stanza)
checkpoint_dir = metadata['checkpoint_dir']
return cls(server_scheme, server_host, server_port, token,
checkpoint_dir, log_dir, inputs)
def execute(self):
'''Modular input entry.
Usage::
>>> Class TestModularInput(ModularInput):
>>> ... .. .
>>>
>>> if __name__ == '__main__':
>>> md = TestModularInput()
>>> md.execute()
'''
if len(sys.argv) == 1:
try:
input_definition = InputDefinition.parse(sys.stdin)
self._update_metadata(input_definition.metadata)
if self.use_single_instance:
self.config_name = self.name
else:
self.config_name = input_definition.inputs.keys()[0]
self.do_run(input_definition.inputs)
logging.info('Modular input: %s exit normally.', self.name)
return 0
except Exception as e:
logging.error('Modular input: %s exit with exception: %s.',
self.name, traceback.format_exc(e))
return 1
finally:
# Stop event writer if any
if self._event_writer:
self._event_writer.close()
# Stop orphan monitor if any
if self._orphan_monitor:
self._orphan_monitor.stop()
elif str(sys.argv[1]).lower() == '--scheme':
sys.stdout.write(self._do_scheme())
sys.stdout.flush()
return 0
elif sys.argv[1].lower() == '--validate-arguments':
try:
validation_definition = ValidationDefinition.parse(sys.stdin)
self._update_metadata(validation_definition.metadata)
self.do_validation(validation_definition.parameters)
return 0
except Exception as e:
logging.error(
'Modular input: %s validate arguments with exception: %s.',
self.name, traceback.format_exc(e))
root = ET.Element('error')
ET.SubElement(root, 'message').text = str(e)
sys.stderr.write(ET.tostring(root))
sys.stderr.flush()
return 1
else:
logging.error(
'Modular input: %s run with invalid arguments: "%s".',
self.name, ' '.join(sys.argv[1:]))
return 1
def run_script(self, args, event_writer, input_stream):
"""Handles all the specifics of running a modular input
:param args: List of command line arguments passed to this script.
:param event_writer: An ``EventWriter`` object for writing events.
:param input_stream: An input stream for reading inputs.
:returns: An integer to be used as the exit value of this program.
"""
try:
if len(args) == 1:
# This script is running as an input. Input definitions will be
# passed on stdin as XML, and the script will write events on
# stdout and log entries on stderr.
self._input_definition = InputDefinition.parse(input_stream)
self.stream_events(self._input_definition, event_writer)
event_writer.close()
return 0
elif str(args[1]).lower() == "--scheme":
# Splunk has requested XML specifying the scheme for this
# modular input Return it and exit.
scheme = self.get_scheme()
if scheme is None:
event_writer.log(
EventWriter.FATAL,
"Modular input script returned a null scheme.")
return 1
else:
event_writer.write_xml_document(scheme.to_xml())
return 0
elif args[1].lower() == "--validate-arguments":
validation_definition = ValidationDefinition.parse(
input_stream)
try:
self.validate_input(validation_definition)
return 0
except Exception as e:
root = ET.Element("error")
ET.SubElement(root, "message").text = e.message
event_writer.write_xml_document(root)
import traceback
print ""
print(traceback.format_exc())
return 1
else:
err_string = "ERROR Invalid arguments to modular input script:" + ' '.join(
args)
event_writer._err.write(err_string)
except Exception as e:
err_string = EventWriter.ERROR + " : " + e.message
event_writer._err.write(err_string)
import traceback
print ""
print(traceback.format_exc())
return 1
def run_script(self, args, event_writer, input_stream):
"""Handles all the specifics of running a modular input
:param args: List of command line arguments passed to this script.
:param event_writer: An ``EventWriter`` object for writing events.
:param input_stream: An input stream for reading inputs.
:returns: An integer to be used as the exit value of this program.
"""
try:
if len(args) == 1:
# This script is running as an input. Input definitions will be
# passed on stdin as XML, and the script will write events on
# stdout and log entries on stderr.
self._input_definition = InputDefinition.parse(input_stream)
self.stream_events(self._input_definition, event_writer)
event_writer.close()
return 0
elif str(args[1]).lower() == "--scheme":
# Splunk has requested XML specifying the scheme for this
# modular input Return it and exit.
scheme = self.get_scheme()
if scheme is None:
event_writer.log(
EventWriter.FATAL,
"Modular input script returned a null scheme.")
return 1
else:
event_writer.write_xml_document(scheme.to_xml())
return 0
elif args[1].lower() == "--validate-arguments":
validation_definition = ValidationDefinition.parse(input_stream)
try:
self.validate_input(validation_definition)
return 0
except Exception as e:
root = ET.Element("error")
ET.SubElement(root, "message").text = e.message
event_writer.write_xml_document(root)
import traceback
print ""
print(traceback.format_exc())
return 1
else:
err_string = "ERROR Invalid arguments to modular input script:" + ' '.join(
args)
event_writer._err.write(err_string)
except Exception as e:
err_string = EventWriter.ERROR + " : " + e.message
event_writer._err.write(err_string)
import traceback
print ""
print(traceback.format_exc())
return 1
def test_parse_inputdef_with_two_inputs(self):
"""Check parsing of XML that contains 2 inputs"""
found = InputDefinition.parse(data_open("data/conf_with_2_inputs.xml"))
expectedDefinition = InputDefinition()
expectedDefinition.metadata = {
"server_host": "tiny",
"server_uri": "https://127.0.0.1:8089",
"checkpoint_dir": "/some/dir",
"session_key": "123102983109283019283"
}
expectedDefinition.inputs["foobar://aaa"] = {
"param1": "value1",
"param2": "value2",
"disabled": "0",
"index": "default"
}
expectedDefinition.inputs["foobar://bbb"] = {
"param1": "value11",
"param2": "value22",
"disabled": "0",
"index": "default",
"multiValue": ["value1", "value2"],
"multiValue2": ["value3", "value4"]
}
self.assertEqual(expectedDefinition, found)
def test_attempt_to_parse_malformed_input_definition_will_throw_exception(self):
"""Does malformed XML cause the expected exception."""
with self.assertRaises(ValueError):
found = InputDefinition.parse(data_open("data/conf_with_invalid_inputs.xml"))
