def test_parse_inputdef_with_two_inputs(self): """Check parsing of XML that contains 2 inputs""" found = InputDefinition.parse(data_open("data/conf_with_2_inputs.xml")) expectedDefinition = InputDefinition() expectedDefinition.metadata = { "server_host": "tiny", "server_uri": "https://127.0.0.1:8089", "checkpoint_dir": "/some/dir", "session_key": "123102983109283019283" } expectedDefinition.inputs["foobar://aaa"] = { "param1": "value1", "param2": "value2", "disabled": "0", "index": "default" } expectedDefinition.inputs["foobar://bbb"] = { "param1": "value11", "param2": "value22", "disabled": "0", "index": "default", "multiValue": ["value1", "value2"], "multiValue2": ["value3", "value4"] } self.assertEqual(expectedDefinition, found)
def test_parse_inputdef_with_zero_inputs(self): """Check parsing of XML that contains only metadata""" found = InputDefinition.parse(data_open("data/conf_with_0_inputs.xml")) expectedDefinition = InputDefinition() expectedDefinition.metadata = { "server_host": "tiny", "server_uri": "https://127.0.0.1:8089", "checkpoint_dir": "/some/dir", "session_key": "123102983109283019283" } self.assertEqual(found, expectedDefinition)
def get_input_definition(self): '''Get input definition. This method can be overwritten to get input definition from other input instead `stdin`. :returns: A dict object must contains `metadata` and `inputs`, example: { 'metadata': { 'session_key': 'iCKPS0cvmpyeJk...sdaf', 'server_host': 'test-test.com', 'server_uri': 'https://127.0.0.1:8089', 'checkpoint_dir': '/tmp' }, inputs: { 'stanza1': {'arg1': value1, 'arg2': value2}, 'stanza2': {'arg1': value1, 'arg2': value2} } } :rtype: ``dict`` ''' input_definition = InputDefinition.parse(sys.stdin) return { 'metadata': input_definition.metadata, 'inputs': input_definition.inputs }
def test_attempt_to_parse_malformed_input_definition_will_throw_exception( self): """Does malformed XML cause the expected exception.""" with self.assertRaises(ValueError): found = InputDefinition.parse( data_open("data/conf_with_invalid_inputs.xml"))
def from_stream(cls, stream): definition = InputDefinition.parse(stream) metadata = definition.metadata parts = urllib.parse.urlparse(metadata['server_uri']) scheme = parts.scheme host = parts.hostname port = parts.port token = metadata['session_key'] server_scheme = scheme server_host = host server_port = port token = token log_dir = environ.get_log_folder() inputs = list() for name, params in list(definition.inputs.items()): kind, name = cls._split_stanza(name) stanza = Stanza(kind, name, params) inputs.append(stanza) checkpoint_dir = metadata['checkpoint_dir'] return cls(server_scheme, server_host, server_port, token, checkpoint_dir, log_dir, inputs)
def execute(self): '''Modular input entry. Usage:: >>> Class TestModularInput(ModularInput): >>> ... .. . >>> >>> if __name__ == '__main__': >>> md = TestModularInput() >>> md.execute() ''' if len(sys.argv) == 1: try: input_definition = InputDefinition.parse(sys.stdin) self._update_metadata(input_definition.metadata) if self.use_single_instance: self.config_name = self.name else: self.config_name = input_definition.inputs.keys()[0] self.do_run(input_definition.inputs) logging.info('Modular input: %s exit normally.', self.name) return 0 except Exception as e: logging.error('Modular input: %s exit with exception: %s.', self.name, traceback.format_exc(e)) return 1 finally: # Stop event writer if any if self._event_writer: self._event_writer.close() # Stop orphan monitor if any if self._orphan_monitor: self._orphan_monitor.stop() elif str(sys.argv[1]).lower() == '--scheme': sys.stdout.write(self._do_scheme()) sys.stdout.flush() return 0 elif sys.argv[1].lower() == '--validate-arguments': try: validation_definition = ValidationDefinition.parse(sys.stdin) self._update_metadata(validation_definition.metadata) self.do_validation(validation_definition.parameters) return 0 except Exception as e: logging.error( 'Modular input: %s validate arguments with exception: %s.', self.name, traceback.format_exc(e)) root = ET.Element('error') ET.SubElement(root, 'message').text = str(e) sys.stderr.write(ET.tostring(root)) sys.stderr.flush() return 1 else: logging.error( 'Modular input: %s run with invalid arguments: "%s".', self.name, ' '.join(sys.argv[1:])) return 1
def run_script(self, args, event_writer, input_stream): """Handles all the specifics of running a modular input :param args: List of command line arguments passed to this script. :param event_writer: An ``EventWriter`` object for writing events. :param input_stream: An input stream for reading inputs. :returns: An integer to be used as the exit value of this program. """ try: if len(args) == 1: # This script is running as an input. Input definitions will be # passed on stdin as XML, and the script will write events on # stdout and log entries on stderr. self._input_definition = InputDefinition.parse(input_stream) self.stream_events(self._input_definition, event_writer) event_writer.close() return 0 elif str(args[1]).lower() == "--scheme": # Splunk has requested XML specifying the scheme for this # modular input Return it and exit. scheme = self.get_scheme() if scheme is None: event_writer.log( EventWriter.FATAL, "Modular input script returned a null scheme.") return 1 else: event_writer.write_xml_document(scheme.to_xml()) return 0 elif args[1].lower() == "--validate-arguments": validation_definition = ValidationDefinition.parse( input_stream) try: self.validate_input(validation_definition) return 0 except Exception as e: root = ET.Element("error") ET.SubElement(root, "message").text = e.message event_writer.write_xml_document(root) import traceback print "" print(traceback.format_exc()) return 1 else: err_string = "ERROR Invalid arguments to modular input script:" + ' '.join( args) event_writer._err.write(err_string) except Exception as e: err_string = EventWriter.ERROR + " : " + e.message event_writer._err.write(err_string) import traceback print "" print(traceback.format_exc()) return 1
def run_script(self, args, event_writer, input_stream): """Handles all the specifics of running a modular input :param args: List of command line arguments passed to this script. :param event_writer: An ``EventWriter`` object for writing events. :param input_stream: An input stream for reading inputs. :returns: An integer to be used as the exit value of this program. """ try: if len(args) == 1: # This script is running as an input. Input definitions will be # passed on stdin as XML, and the script will write events on # stdout and log entries on stderr. self._input_definition = InputDefinition.parse(input_stream) self.stream_events(self._input_definition, event_writer) event_writer.close() return 0 elif str(args[1]).lower() == "--scheme": # Splunk has requested XML specifying the scheme for this # modular input Return it and exit. scheme = self.get_scheme() if scheme is None: event_writer.log( EventWriter.FATAL, "Modular input script returned a null scheme.") return 1 else: event_writer.write_xml_document(scheme.to_xml()) return 0 elif args[1].lower() == "--validate-arguments": validation_definition = ValidationDefinition.parse(input_stream) try: self.validate_input(validation_definition) return 0 except Exception as e: root = ET.Element("error") ET.SubElement(root, "message").text = e.message event_writer.write_xml_document(root) import traceback print "" print(traceback.format_exc()) return 1 else: err_string = "ERROR Invalid arguments to modular input script:" + ' '.join( args) event_writer._err.write(err_string) except Exception as e: err_string = EventWriter.ERROR + " : " + e.message event_writer._err.write(err_string) import traceback print "" print(traceback.format_exc()) return 1
def test_attempt_to_parse_malformed_input_definition_will_throw_exception(self): """Does malformed XML cause the expected exception.""" with self.assertRaises(ValueError): found = InputDefinition.parse(data_open("data/conf_with_invalid_inputs.xml"))