def parse_mapreduce_yaml(contents): """Parses mapreduce.yaml file contents. Args: contents: mapreduce.yaml file contents. Returns: MapReduceYaml object with all the data from original file. Raises: errors.BadYamlError: when contents is not a valid mapreduce.yaml file. """ try: builder = yaml_object.ObjectBuilder(MapReduceYaml) handler = yaml_builder.BuilderHandler(builder) listener = yaml_listener.EventListener(handler) listener.Parse(contents) mr_info = handler.GetResults() except (ValueError, yaml_errors.EventError) as e: raise errors.BadYamlError(e) if len(mr_info) < 1: raise errors.BadYamlError("No configs found in mapreduce.yaml") if len(mr_info) > 1: raise errors.MultipleDocumentsInMrYaml("Found %d YAML documents" % len(mr_info)) jobs = mr_info[0] job_names = set(j.name for j in jobs.mapreduce) if len(jobs.mapreduce) != len(job_names): raise errors.BadYamlError( "Overlapping mapreduce names; names must be unique") return jobs
def LoadSingleAppInfo(app_info): """Load a single AppInfo object where one and only one is expected. Args: app_info: A file-like object or string. If it is a string, parse it as a configuration file. If it is a file-like object, read in data and parse. Returns: An instance of AppInfoExternal as loaded from a YAML file. Raises: ValueError: if a specified service is not valid. EmptyConfigurationFile: when there are no documents in YAML file. MultipleConfigurationFile: when there is more than one document in YAML file. """ builder = yaml_object.ObjectBuilder(AppInfoExternal) handler = yaml_builder.BuilderHandler(builder) listener = yaml_listener.EventListener(handler) listener.Parse(app_info) app_infos = handler.GetResults() if len(app_infos) < 1: raise appinfo_errors.EmptyConfigurationFile() if len(app_infos) > 1: raise appinfo_errors.MultipleConfigurationFile() appyaml = app_infos[0] ValidateHandlers(appyaml.handlers) if appyaml.builtins: BuiltinHandler.Validate(appyaml.builtins) return appyaml
def load_config(stream, config_globals): """Load a configuration file and generate importer and exporter classes. Args: stream: Stream containing config YAML. config_globals: Dict to use to reference globals for code in the config. Returns: BulkloaderEntry Raises: InvalidConfiguration: If the config is invalid. """ builder = yaml_object.ObjectBuilder(BulkloaderEntry) handler = yaml_builder.BuilderHandler(builder) listener = yaml_listener.EventListener(handler) global _global_temp_globals _global_temp_globals = config_globals try: listener.Parse(stream) finally: _global_temp_globals = None bulkloader_infos = handler.GetResults() if len(bulkloader_infos) < 1: raise bulkloader_errors.InvalidConfiguration( 'No configuration specified.') if len(bulkloader_infos) > 1: raise bulkloader_errors.InvalidConfiguration( 'Multiple sections in configuration.') bulkloader_info = bulkloader_infos[0] if not bulkloader_info.transformers: raise bulkloader_errors.InvalidConfiguration( 'No transformers specified.') return bulkloader_info
def testForwardsEventsInOrderReceived(self): """Check to make sure listener forwards events in order received.""" events = [ yaml.events.StreamStartEvent(), yaml.events.StreamEndEvent(), yaml.events.DocumentStartEvent(), yaml.events.DocumentEndEvent(), yaml.events.AliasEvent(None), yaml.events.ScalarEvent(None, None, (True, False), None), yaml.events.SequenceStartEvent(None, None, None), yaml.events.SequenceEndEvent(), yaml.events.MappingStartEvent(None, None, None), yaml.events.MappingEndEvent(), yaml.events.SequenceEndEvent(), yaml.events.DocumentEndEvent(), yaml.events.MappingStartEvent(None, None, None), ] FAKE_LOADER = object() event_generator = [(event, FAKE_LOADER) for event in events] handler = yaml_test_util.FakeEventHandler(self, FAKE_LOADER) listener = yaml_listener.EventListener(handler) listener._HandleEvents(event_generator) self.assertEqual(events, handler.events)
def LoadSingleDos(dos_info, open_fn=None): """Load a dos.yaml file or string and return a DosInfoExternal object. Args: dos_info: The contents of a dos.yaml file as a string, or an open file object. open_fn: Function for opening files. Unused. Returns: A DosInfoExternal instance which represents the contents of the parsed yaml file. Raises: MalformedDosConfiguration: The yaml file contains multiple blacklist sections. yaml_errors.EventError: An error occured while parsing the yaml file. """ builder = yaml_object.ObjectBuilder(DosInfoExternal) handler = yaml_builder.BuilderHandler(builder) listener = yaml_listener.EventListener(handler) listener.Parse(dos_info) parsed_yaml = handler.GetResults() if not parsed_yaml: return DosInfoExternal() if len(parsed_yaml) > 1: raise MalformedDosConfiguration('Multiple blacklist: sections ' 'in configuration.') return parsed_yaml[0]
def LoadPagespeedEntry(pagespeed_entry, open_fn=None): """Load a yaml file or string and return a PagespeedEntry. Args: pagespeed_entry: The contents of a pagespeed entry from a yaml file as a string, or an open file object. open_fn: Function for opening files. Unused. Returns: A PagespeedEntry instance which represents the contents of the parsed yaml. Raises: yaml_errors.EventError: An error occured while parsing the yaml. MalformedPagespeedConfiguration: The configuration is parseable but invalid. """ builder = yaml_object.ObjectBuilder(PagespeedEntry) handler = yaml_builder.BuilderHandler(builder) listener = yaml_listener.EventListener(handler) listener.Parse(pagespeed_entry) parsed_yaml = handler.GetResults() if not parsed_yaml: return PagespeedEntry() if len(parsed_yaml) > 1: raise MalformedPagespeedConfiguration( 'Multiple configuration sections in the yaml') return parsed_yaml[0]
def LoadSingleClientDeployInfo(client_deploy_info): """Returns a ClientDeployInfoExternal from a deploy_info.yaml file or string. Args: client_deploy_info: The contents of a client_deploy_info.yaml file or string, or an open file object. Returns: A ClientDeployInfoExternal instance which represents the contents of the parsed yaml. Raises: EmptyYaml: when there are no documents in yaml. MultipleClientDeployInfo: when there are multiple documents in yaml. yaml_errors.EventError: when an error occurs while parsing the yaml. """ builder = yaml_object.ObjectBuilder(ClientDeployInfoExternal) handler = yaml_builder.BuilderHandler(builder) listener = yaml_listener.EventListener(handler) listener.Parse(client_deploy_info) parsed_yaml = handler.GetResults() if not parsed_yaml: raise EmptyYaml() if len(parsed_yaml) > 1: raise MultipleClientDeployInfo() return parsed_yaml[0]
def LoadBackendInfo(backend_info, open_fn=None): """Parses a BackendInfoExternal object from a string. Args: backend_info: a backends stanza (list of backends) as a string open_fn: Function for opening files. Unused. Returns: A BackendInfoExternal object. """ builder = yaml_object.ObjectBuilder(BackendInfoExternal) handler = yaml_builder.BuilderHandler(builder) listener = yaml_listener.EventListener(handler) listener.Parse(backend_info) backend_info = handler.GetResults() if len(backend_info) < 1: return BackendInfoExternal(backends=[]) if len(backend_info) > 1: raise BadConfig("Only one 'backends' clause is allowed.") info = backend_info[0] if not info.backends: # The 'backends' clause was empty. return BackendInfoExternal(backends=[]) for backend in info.backends: backend.Init() return info
def LoadSingleDispatch(dispatch_info, open_fn=None): """Load a dispatch.yaml file or string and return a DispatchInfoExternal. Args: dispatch_info: The contents of a dispatch.yaml file as a string, or an open file object. open_fn: Function for opening files. Unused here, needed to provide a polymorphic API used by appcfg.py yaml parsing. Returns: A DispatchInfoExternal instance which represents the contents of the parsed yaml file. Raises: MalformedDispatchConfigurationError: The yaml file contains multiple dispatch sections. yaml_errors.EventError: An error occured while parsing the yaml file. """ builder = yaml_object.ObjectBuilder(DispatchInfoExternal) handler = yaml_builder.BuilderHandler(builder) listener = yaml_listener.EventListener(handler) listener.Parse(dispatch_info) parsed_yaml = handler.GetResults() if not parsed_yaml: return DispatchInfoExternal() if len(parsed_yaml) > 1: raise MalformedDispatchConfigurationError( 'Multiple dispatch: sections ' 'in configuration.') return parsed_yaml[0]
def LoadSingleDos(dos_info): """Load a dos.yaml file or string and return a DosInfoExternal object. Args: dos_info: The contents of a dos.yaml file, as a string. Returns: A DosInfoExternal instance which represents the contents of the parsed yaml file. Raises: MalformedDosConfiguration if the yaml file contains multiple blacklist sections. yaml_errors.EventError if any errors occured while parsing the yaml file. """ builder = yaml_object.ObjectBuilder(DosInfoExternal) handler = yaml_builder.BuilderHandler(builder) listener = yaml_listener.EventListener(handler) listener.Parse(dos_info) dos_info = handler.GetResults() if len(dos_info) < 1: return DosInfoExternal() if len(dos_info) > 1: raise MalformedDosConfiguration('Multiple blacklist: sections ' 'in configuration.') return dos_info[0]
def LoadSinglePagespeed(pagespeed_info): """Load a pagespeed.yaml file or string and return a PagespeedInfoExternal. Args: pagespeed_info: The contents of a pagespeed.yaml file as a string, or an open file object. Returns: A PagespeedInfoExternal instance which represents the contents of the parsed yaml file. Raises: yaml_errors.EventError: An error occured while parsing the yaml file. MalformedPagespeedConfiguration: The configuration is parseable but invalid. """ builder = yaml_object.ObjectBuilder(PagespeedInfoExternal) handler = yaml_builder.BuilderHandler(builder) listener = yaml_listener.EventListener(handler) listener.Parse(pagespeed_info) parsed_yaml = handler.GetResults() if not parsed_yaml: return PagespeedInfoExternal() if len(parsed_yaml) > 1: raise MalformedPagespeedConfiguration( 'Multiple configuration sections in pagespeed.yaml') return parsed_yaml[0]
def testUnexpectedEventClass(self): """Check when parser sends illegal event type.""" listener = yaml_listener.EventListener( yaml_test_util.FakeEventHandler(self)) self.failUnlessRaises(yaml_errors.IllegalEvent, listener.HandleEvent, object())
def LoadSingleCron(cron_info, open_fn=None): """Load a cron.yaml file or string and return a CronInfoExternal object.""" builder = yaml_object.ObjectBuilder(CronInfoExternal) handler = yaml_builder.BuilderHandler(builder) listener = yaml_listener.EventListener(handler) listener.Parse(cron_info) cron_info = handler.GetResults() if len(cron_info) < 1: raise MalformedCronfigurationFile('Empty cron configuration.') if len(cron_info) > 1: raise MalformedCronfigurationFile('Multiple cron sections ' 'in configuration.') return cron_info[0]
def testUsesSafeParsingByDefault(self): """Tests that yaml parsing does not by default allow unsafe parsing.""" class SafeFakeEventHandler(yaml_test_util.FakeEventHandler): def HandleEvent(self, event, loader, event_class): self.test.assertTrue(isinstance(loader, yaml.loader.SafeLoader)) self.test.assertFalse(isinstance(loader, yaml.loader.Loader)) super(SafeFakeEventHandler, self).HandleEvent(event, loader, event_class) document = ('yaml_var: value') handler = SafeFakeEventHandler(self) listener = yaml_listener.EventListener(handler) listener.Parse(document)
def LoadSingleDispatch(dispatch_info, open_fn=None): """Load a dispatch.yaml file or string and return a DispatchInfoExternal. Args: dispatch_info: The contents of a dispatch.yaml file as a string, or an open file object. open_fn: Function for opening files. Unused here, needed to provide a polymorphic API used by appcfg.py yaml parsing. Returns: A DispatchInfoExternal instance which represents the contents of the parsed yaml file. Raises: MalformedDispatchConfigurationError: The yaml file contains multiple dispatch sections or is missing a required value. yaml_errors.EventError: An error occured while parsing the yaml file. """ builder = yaml_object.ObjectBuilder(DispatchInfoExternal) handler = yaml_builder.BuilderHandler(builder) listener = yaml_listener.EventListener(handler) listener.Parse(dispatch_info) parsed_yaml = handler.GetResults() if not parsed_yaml: return DispatchInfoExternal() if len(parsed_yaml) > 1: raise MalformedDispatchConfigurationError( 'Multiple dispatch: sections ' 'in configuration.') # The validation framework doesn't allow validating multiple fields at once, # so we have to check here. dispatch_info_external = parsed_yaml[0] for dispatch in getattr(dispatch_info_external, DISPATCH) or []: if dispatch.module and dispatch.service: raise MalformedDispatchConfigurationError( 'Both module: and service: in dispatch entry. Please use only one.' ) if not (dispatch.module or dispatch.service): raise MalformedDispatchConfigurationError( "Missing required value 'service'.") # The important part is that just 'module' is sent to the API for now; # eventually, we'll switch this to just check service. dispatch.module = dispatch.module or dispatch.service dispatch.service = None return dispatch_info_external
def testParseYaml(self): """Test events generated by PyYAML parser correctly forwarded.""" document = ('item1: string1\n' 'item2:\n' ' - listitem1\n' ' - listitem2\n' 'item3: &item3\n' ' mapkey1: mapvalue1\n' ' mapkey2: mapvalue2\n' 'item4: *item3\n') handler = yaml_test_util.FakeEventHandler(self) listener = yaml_listener.EventListener(handler) listener.Parse(document) expected_event_list = [ (yaml.events.StreamStartEvent,), (yaml.events.DocumentStartEvent,), (yaml.events.MappingStartEvent,), (yaml.events.ScalarEvent, 'item1'), (yaml.events.ScalarEvent, 'string1'), (yaml.events.ScalarEvent, 'item2'), (yaml.events.SequenceStartEvent,), (yaml.events.ScalarEvent, 'listitem1'), (yaml.events.ScalarEvent, 'listitem2'), (yaml.events.SequenceEndEvent,), (yaml.events.ScalarEvent, 'item3'), (yaml.events.MappingStartEvent,), (yaml.events.ScalarEvent, 'mapkey1'), (yaml.events.ScalarEvent, 'mapvalue1'), (yaml.events.ScalarEvent, 'mapkey2'), (yaml.events.ScalarEvent, 'mapvalue2'), (yaml.events.MappingEndEvent,), (yaml.events.ScalarEvent, 'item4'), (yaml.events.AliasEvent,), (yaml.events.MappingEndEvent,), (yaml.events.DocumentEndEvent,), (yaml.events.StreamEndEvent,), ] self.assertEqual(len(expected_event_list), len(handler.events)) for expected_event, event in zip(expected_event_list, handler.events): self._AssertEventClass(event, *expected_event)
def LoadBackendEntry(backend_entry): """Parses a BackendEntry object from a string. Args: backend_entry: a backend entry, as a string Returns: A BackendEntry object. """ builder = yaml_object.ObjectBuilder(BackendEntry) handler = yaml_builder.BuilderHandler(builder) listener = yaml_listener.EventListener(handler) listener.Parse(backend_entry) entries = handler.GetResults() if len(entries) < 1: raise BadConfig('Empty backend configuration.') if len(entries) > 1: raise BadConfig('Multiple backend entries were found in configuration.') return entries[0].Init()
def parse_mapreduce_yaml(contents): """Parses mapreduce.yaml file contents. Args: contents: mapreduce.yaml file contents. Returns: MapReduceYaml object with all the data from original file. Raises: errors.BadYamlError: when contents is not a valid mapreduce.yaml file. """ try: builder = yaml_object.ObjectBuilder(MapReduceYaml) handler = yaml_builder.BuilderHandler(builder) listener = yaml_listener.EventListener(handler) listener.Parse(contents) mr_info = handler.GetResults() except (ValueError, yaml_errors.EventError), e: raise errors.BadYamlError(e)
def LoadSingleQueue(queue_info): """Load a queue.yaml file or string and return a QueueInfoExternal object. Args: queue_info: the contents of a queue.yaml file, as a string. Returns: A QueueInfoExternal object. """ builder = yaml_object.ObjectBuilder(QueueInfoExternal) handler = yaml_builder.BuilderHandler(builder) listener = yaml_listener.EventListener(handler) listener.Parse(queue_info) queue_info = handler.GetResults() if len(queue_info) < 1: raise MalformedQueueConfiguration('Empty queue configuration.') if len(queue_info) > 1: raise MalformedQueueConfiguration('Multiple queue: sections ' 'in configuration.') return queue_info[0]
def parse_fetcher_policy_yaml(contents): """Parses fetcher_policy.yaml file contents. Args: contents: fetcher_policy.yaml file contents. Returns: FetcherPolicyYaml object with all the data from original file. Raises: errors.BadYamlError: when contents is not a valid fetcher_policy.yaml file. """ try: builder = yaml_object.ObjectBuilder(FetcherPolicyYaml) handler = yaml_builder.BuilderHandler(builder) listener = yaml_listener.EventListener(handler) listener.Parse(contents) fp_info = handler.GetResults() except (ValueError, yaml_errors.EventError), e: raise errors.BadYamlError(e)
def LoadSingleQueue(queue_info, open_fn=None): """Loads a `queue.yaml` file/string and returns a `QueueInfoExternal` object. Args: queue_info: The contents of a `queue.yaml` file, as a string. open_fn: Function for opening files. Unused. Returns: A `QueueInfoExternal` object. """ builder = yaml_object.ObjectBuilder(QueueInfoExternal) handler = yaml_builder.BuilderHandler(builder) listener = yaml_listener.EventListener(handler) listener.Parse(queue_info) queue_info = handler.GetResults() if len(queue_info) < 1: raise MalformedQueueConfiguration('Empty queue configuration.') if len(queue_info) > 1: raise MalformedQueueConfiguration('Multiple queue: sections ' 'in configuration.') return queue_info[0]
def BuildObjects(default_class, stream, loader=yaml.loader.SafeLoader): """Build objects from stream. Handles the basic case of loading all the objects from a stream. Args: default_class: Class that is instantiated upon the detection of a new document. An instance of this class will act as the document itself. stream: String document or open file object to process as per the yaml.parse method. Any object that implements a 'read()' method which returns a string document will work with the YAML parser. loader_class: Used for dependency injection. Returns: List of default_class instances parsed from the stream. """ builder = ObjectBuilder(default_class) handler = yaml_builder.BuilderHandler(builder) listener = yaml_listener.EventListener(handler) listener.Parse(stream, loader) return handler.GetResults()
def testEventHandlerError(self): """Test handling of EventHandlerError.""" class BlowUpEventHandler(yaml_test_util.FakeEventHandler): def Scalar(self, event, loader): if event.value == 'kaboom': raise ValueError('I blew up') super(BlowUpEventHandler, self).Scalar(event, loader) document = ('item1:\n' '- item2: alpha\n' ' item3: kaboom\n' ' item4: you will never see me.\n') expected_event_list = [ (yaml.events.StreamStartEvent,), (yaml.events.DocumentStartEvent,), (yaml.events.MappingStartEvent,), (yaml.events.ScalarEvent, 'item1'), (yaml.events.SequenceStartEvent,), (yaml.events.MappingStartEvent,), (yaml.events.ScalarEvent, 'item2'), (yaml.events.ScalarEvent, 'alpha'), (yaml.events.ScalarEvent, 'item3'), ] handler = BlowUpEventHandler(self) listener = yaml_listener.EventListener(handler) with self.assertRaises(yaml_errors.EventError) as e: listener.Parse(document) self.fail('No event-listener error raised') self.assertTrue(str(e.exception).startswith('I blew up')) self.assertIsInstance(e.exception.cause, ValueError) self.assertIsInstance(e.exception.event, yaml.events.ScalarEvent) self.assertEqual(u'kaboom', e.exception.event.value) self.assertEqual(len(expected_event_list), len(handler.events)) for expected_event, event in zip(expected_event_list, handler.events): self._AssertEventClass(event, *expected_event)
def testYAMLErrorScanner(self): """Test handling of parser error.""" document = 'item1:\x09bad\n' expected_event_list = [ (yaml.events.StreamStartEvent,), (yaml.events.DocumentStartEvent,), (yaml.events.MappingStartEvent,), (yaml.events.ScalarEvent, 'item1'), ] handler = yaml_test_util.FakeEventHandler(self) listener = yaml_listener.EventListener(handler) with self.assertRaises(yaml_errors.EventListenerYAMLError) as e: listener.Parse(document) self.fail('No parser error raised') self.assertTrue( str(e.exception).startswith('while scanning for the next token')) self.assertIsInstance(e.exception.cause, yaml.scanner.ScannerError) self.assertEqual(len(expected_event_list), len(handler.events)) for expected_event, event in zip(expected_event_list, handler.events): self._AssertEventClass(event, *expected_event)
def LoadSingleDispatch(dispatch_info, open_fn=None): """Load a dispatch.yaml file or string and return a DispatchInfoExternal. Args: dispatch_info: The contents of a dispatch.yaml file as a string, or an open file object. open_fn: Function for opening files. Unused here, needed to provide a polymorphic API used by appcfg.py yaml parsing. Returns: A DispatchInfoExternal instance which represents the contents of the parsed yaml file. Raises: MalformedDispatchConfigurationError: The yaml file contains multiple dispatch sections or is missing a required value. yaml_errors.EventError: An error occured while parsing the yaml file. """ builder = yaml_object.ObjectBuilder(DispatchInfoExternal) handler = yaml_builder.BuilderHandler(builder) listener = yaml_listener.EventListener(handler) listener.Parse(dispatch_info) parsed_yaml = handler.GetResults() if not parsed_yaml: return DispatchInfoExternal() if len(parsed_yaml) > 1: raise MalformedDispatchConfigurationError( 'Multiple dispatch: sections ' 'in configuration.') # The validation framework doesn't allow validating multiple fields at once, # so we have to check here. dispatch_info_external = parsed_yaml[0] dispatch_info_external.CheckInitialized() return dispatch_info_external
def testYAMLErrorParser(self): """Test handling of YAML error from Parser.""" document = ('item1: string1\n' '- item2\n') expected_event_list = [ (yaml.events.StreamStartEvent,), (yaml.events.DocumentStartEvent,), (yaml.events.MappingStartEvent,), (yaml.events.ScalarEvent, 'item1'), (yaml.events.ScalarEvent, 'string1'), ] handler = yaml_test_util.FakeEventHandler(self) listener = yaml_listener.EventListener(handler) with self.assertRaises(yaml_errors.EventListenerYAMLError) as e: listener.Parse(document) self.fail('No parser error raised') self.assertTrue(str(e.exception).startswith('while parsing a block')) self.assertIsInstance(e.exception.cause, yaml.parser.ParserError) self.assertEqual(len(expected_event_list), len(handler.events)) for expected_event, event in zip(expected_event_list, handler.events): self._AssertEventClass(event, *expected_event)
def LoadAppInclude(app_include): """Load a single AppInclude object where one and only one is expected. Args: app_include: A file-like object or string. If it is a string, parse it as a configuration file. If it is a file-like object, read in data and parse. Returns: An instance of AppInclude as loaded from a YAML file. Raises: EmptyConfigurationFile: when there are no documents in YAML file. MultipleConfigurationFile: when there is more than one document in YAML file. """ builder = yaml_object.ObjectBuilder(AppInclude) handler = yaml_builder.BuilderHandler(builder) listener = yaml_listener.EventListener(handler) listener.Parse(app_include) includes = handler.GetResults() if len(includes) < 1: raise appinfo_errors.EmptyConfigurationFile() if len(includes) > 1: raise appinfo_errors.MultipleConfigurationFile() includeyaml = includes[0] if includeyaml.handlers: for handler in includeyaml.handlers: handler.FixSecureDefaults() handler.WarnReservedURLs() if includeyaml.builtins: BuiltinHandler.Validate(includeyaml.builtins) return includeyaml
def setUp(self): """Set up parsing frame work.""" self.builder = yaml_object.ObjectBuilder(PetStore) self.handler = yaml_builder.BuilderHandler(self.builder) self.listener = yaml_listener.EventListener(self.handler)