def parse_end_statement(self, tokens: abc.Generator) -> None: """Parses the tokens for an End Statement. <End-Statement> ::= "END" ( <WSC>* | [<Statement-Delimiter>] ) """ try: end = next(tokens) if not end.is_end_statement(): tokens.send(end) raise ValueError( "Expecting an End Statement, like " f'"{self.grammar.end_statements}" but found ' f'"{end}"' ) try: t = next(tokens) if t.is_WSC(): # maybe process comment return else: tokens.send(t) return except LexerError: pass except StopIteration: pass return
def parse_statement_delimiter(tokens: abc.Generator) -> bool: """Parses the tokens for a Statement Delimiter. *tokens* is expected to be a *generator iterator* which provides ``pvl.token`` objects. <Statement-Delimiter> ::= <WSC>* (<white-space-character> | <comment> | ';' | <EOF>) Although the above structure comes from Figure 2-4 of the Blue Book, the <white-space-character> and <comment> elements are redundant with the presence of [WSC]* so it can be simplified to: <Statement-Delimiter> ::= <WSC>* [ ';' | <EOF> ] Typically written [<Statement-Delimiter>]. """ for t in tokens: if t.is_WSC(): # If there's a comment, could parse here. pass elif t.is_delimiter(): return True else: tokens.send(t) # Put the next token back into the generator return False
def parse_module(self, tokens: abc.Generator): """Parses the tokens for a PVL Module. <PVL-Module-Contents> ::= ( <Assignment-Statement> | <WSC>* | <Aggregation-Block> )* [<End-Statement>] """ m = self.modcls() parsing = True while parsing: # print(f'top of while parsing: {m}') parsing = False for p in ( self.parse_aggregation_block, self.parse_assignment_statement, self.parse_end_statement, ): try: self.parse_WSC_until(None, tokens) # t = next(tokens) # print(f'next token: {t}, {t.pos}') # tokens.send(t) parsed = p(tokens) # print(f'parsed: {parsed}') if parsed is None: # because parse_end_statement returned return m else: m.append(*parsed) parsing = True except LexerError: raise except ValueError: pass try: (m, keep_parsing) = self.parse_module_post_hook(m, tokens) if keep_parsing: parsing = True else: return m except Exception: pass # print(f'got to bottom: {m}') t = next(tokens) tokens.throw( ValueError, "Expecting an Aggregation Block, an Assignment " "Statement, or an End Statement, but found " f'"{t}" ', )
def bar(successor: abc.Generator = None): while True: msg = yield msg = '%s > %s in %s' % (msg, msg.split()[0], 'bar') if successor is not None: msg = successor.send(msg) yield msg
def parse_value(self, tokens: abc.Generator): """Parses PVL Values. <Value> ::= (<Simple-Value> | <Set> | <Sequence>) [<WSC>* <Units Expression>] Returns the decoded <Value> as an appropriate Python object. """ value = None try: t = next(tokens) value = self.decoder.decode_simple_value(t) except ValueError: tokens.send(t) for p in ( self.parse_set, self.parse_sequence, self.parse_value_post_hook, ): try: value = p(tokens) break except LexerError: # A LexerError is a subclass of ValueError, but # if we get a LexerError, that's a problem and # we need to raise it, and not let it pass. raise except ValueError: # Getting a ValueError is a normal conseqence of # one of the parsing strategies not working, # this pass allows us to go to the next one. pass else: tokens.throw( ValueError, "Was expecting a Simple Value, or the " "beginning of a Set or Sequence, but " f'found: "{t}"', ) # print(f'in parse_value, value is: {value}') self.parse_WSC_until(None, tokens) try: return self.parse_units(value, tokens) except (ValueError, StopIteration): return value
def parse_WSC_until(token: str, tokens: abc.Generator) -> bool: """Consumes objects from *tokens*, if the object's *.is_WSC()* function returns *True*, it will continue until *token* is encountered and will return *True*. If it encounters an object that does not meet these conditions, it will 'return' that object to *tokens* and will return *False*. *tokens* is expected to be a *generator iterator* which provides ``pvl.token`` objects. """ for t in tokens: if t == token: return True elif t.is_WSC(): # If there's a comment, could parse here. pass else: tokens.send(t) return False
def parse_assignment_statement(self, tokens: abc.Generator) -> tuple: """Parses the tokens for an Assignment Statement. The returned two-tuple contains the Parameter Name in the first element, and the Value in the second. <Assignment-Statement> ::= <Parameter-Name> <WSC>* '=' <WSC>* <Value> [<Statement-Delimiter>] """ try: t = next(tokens) if t.is_parameter_name(): parameter_name = str(t) else: tokens.send(t) raise ValueError( "Expecting a Parameter Name, but " f'found: "{t}"' ) except StopIteration: raise ValueError( "Ran out of tokens before starting to parse " "an Assignment-Statement." ) self.parse_around_equals(tokens) try: # print(f'parameter name: {parameter_name}') value = self.parse_value(tokens) except StopIteration: raise ParseError( "Ran out of tokens to parse after the equals " "sign in an Assignment-Statement: " f'"{parameter_name} =".', t, ) self.parse_statement_delimiter(tokens) return parameter_name, value
def foo(successor: abc.Generator = None): """ notice we use yield in both the traditional generator sense and also in the coroutine sense. """ while True: msg = yield # coroutine feature msg = '%s > %s in %s' % (msg, msg.split()[0], 'foo') if successor is not None: msg = successor.send(msg) yield msg # generator feature
def _parse_set_seq(self, delimiters, tokens: abc.Generator) -> list: """The internal parsing of PVL Sets and Sequences are very similar, and this function provides that shared logic. *delimiters* are a two-tuple containing the start and end characters for the PVL Set or Sequence. """ t = next(tokens) if t != delimiters[0]: tokens.send(t) raise ValueError(f'Expecting a begin delimiter "{delimiters[0]}" ' f'but found: "{t}"') set_seq = list() # Initial WSC and/or empty if self.parse_WSC_until(delimiters[1], tokens): return set_seq # First item: set_seq.append(self.parse_value(tokens)) if self.parse_WSC_until(delimiters[1], tokens): return set_seq # Remaining items, if any for t in tokens: # print(f'in loop, t: {t}, set_seq: {set_seq}') if t == ',': self.parse_WSC_until(None, tokens) # consume WSC after ',' set_seq.append(self.parse_value(tokens)) if self.parse_WSC_until(delimiters[1], tokens): return set_seq else: tokens.send(t) tokens.throw(ValueError, 'While parsing, expected a comma (,)' f'but found: "{t}"')
def parse_value_post_hook(self, tokens: abc.Generator): """Overrides the parent function to allow for more permissive parsing. If the next token is a reserved word or delimiter, then it is returned to the *tokens* and an EmptyValueAtLine object is returned as the value. """ t = next(tokens) # print(f't: {t}') truecase_reserved = [ x.casefold() for x in self.grammar.reserved_keywords ] trucase_delim = [x.casefold() for x in self.grammar.delimiters] if t.casefold() in (truecase_reserved + trucase_delim): # print(f'kw: {kw}') # if kw.casefold() == t.casefold(): # print('match') tokens.send(t) return self._empty_value(t.pos) else: raise ValueError
def parse_module_post_hook( self, module: MutableMappingSequence, tokens: abc.Generator ): """Overrides the parent function to allow for more permissive parsing. If an Assignment-Statement is blank, then the value will be assigned an EmptyValueAtLine object. """ # It enables this by checking to see if the next thing is an # '=' which means there was an empty assigment at the previous # equals sign, and then unwinding the stack to give the # previous assignment the EmptyValueAtLine() object and trying # to continue parsing. # print('in hook') try: t = next(tokens) if t == "=" and len(module) != 0: (last_k, last_v) = module[-1] last_token = Token( last_v, grammar=self.grammar, decoder=self.decoder ) if last_token.is_parameter_name(): # Fix the previous entry module.pop() module.append(last_k, self._empty_value(t.pos)) # Now use last_token as the parameter name # for the next assignment, and we must # reproduce the last part of parse-assignment: try: # print(f'parameter name: {last_token}') self.parse_WSC_until(None, tokens) value = self.parse_value(tokens) self.parse_statement_delimiter(tokens) module.append(str(last_token), value) except StopIteration: module.append( str(last_token), self._empty_value(t.pos + 1) ) return module, False # return through parse_module() else: tokens.send(t) else: # The next token isn't an equals sign or the module is # empty, so we want return the token and signal # parse_module() that it should ignore us. tokens.send(t) raise Exception # Peeking at the next token gives us the opportunity to # see if we're at the end of tokens, which we want to handle. t = next(tokens) tokens.send(t) return module, True # keep parsing except StopIteration: # If we're out of tokens, that's okay. return module, False # return through parse_module()
def parse_around_equals(self, tokens: abc.Generator) -> None: """Parses white space and comments on either side of an equals sign. *tokens* is expected to be a *generator iterator* which provides ``pvl.token`` objects. This is shared functionality for Begin Aggregation Statements and Assignment Statements. It basically covers parsing anything that has a syntax diagram like this: <WSC>* '=' <WSC>* """ if not self.parse_WSC_until("=", tokens): try: t = next(tokens) tokens.send(t) raise ValueError(f'Expecting "=", got: {t}') except StopIteration: raise ParseError('Expecting "=", but ran out of tokens.') self.parse_WSC_until(None, tokens) return
def parse_end_aggregation( self, begin_agg: str, block_name: str, tokens: abc.Generator ) -> None: """Parses the tokens for an End Aggregation Statement. <End-Aggregation-Statement-block> ::= <End-Aggegation-Statement> [<WSC>* '=' <WSC>* <Block-Name>] [<Statement-Delimiter>] Where <Block-Name> ::= <Parameter-Name> """ end_agg = next(tokens) # Need to do a little song and dance to case-independently # match the keys: for k in self.grammar.aggregation_keywords.keys(): if k.casefold() == begin_agg.casefold(): truecase_begin = k break if ( end_agg.casefold() != self.grammar.aggregation_keywords[truecase_begin].casefold() ): tokens.send(end_agg) raise ValueError( "Expecting an End-Aggegation-Statement that " "matched the Begin-Aggregation_Statement, " f'"{begin_agg}" but found: {end_agg}' ) try: self.parse_around_equals(tokens) except (ParseError, ValueError): # No equals statement, which is fine. self.parse_statement_delimiter(tokens) return None t = next(tokens) if t != block_name: tokens.send(t) tokens.throw( ValueError, f'Expecting a Block-Name after "{end_agg} =" ' f'that matches "{block_name}", but found: ' f'"{t}"', ) self.parse_statement_delimiter(tokens) return None
def parse_units(self, value, tokens: abc.Generator) -> str: """Parses PVL Units Expression. <Units-Expression> ::= "<" [<white-space>] <Units-Value> [<white-space>] ">" and <Units-Value> ::= <units-character> [ [ <units-character> | <white-space> ]* <units-character> ] Returns the *value* and the <Units-Value> as a ``Units()`` object. """ t = next(tokens) if not t.startswith(self.grammar.units_delimiters[0]): tokens.send(t) raise ValueError( "Was expecting the start units delimiter, " + '"{}" '.format(self.grammar.units_delimiters[0]) + f'but found "{t}"' ) if not t.endswith(self.grammar.units_delimiters[1]): tokens.send(t) raise ValueError( "Was expecting the end units delimiter, " + '"{}" '.format(self.grammar.units_delimiters[1]) + f'at the end, but found "{t}"' ) delim_strip = t.strip("".join(self.grammar.units_delimiters)) units_value = delim_strip.strip("".join(self.grammar.whitespace)) for d in self.grammar.units_delimiters: if d in units_value: tokens.throw( ValueError, "Was expecting a units character, but found a " f'unit delimiter, "{d}" instead.', ) return self.decoder.decode_quantity(value, units_value)
def parse_begin_aggregation_statement( self, tokens: abc.Generator ) -> tuple: """Parses the tokens for a Begin Aggregation Statement, and returns the name Block Name as a ``str``. <Begin-Aggregation-Statement-block> ::= <Begin-Aggegation-Statement> <WSC>* '=' <WSC>* <Block-Name> [<Statement-Delimiter>] Where <Block-Name> ::= <Parameter-Name> """ try: begin = next(tokens) if not begin.is_begin_aggregation(): tokens.send(begin) raise ValueError( "Expecting a Begin-Aggegation-Statement, but " f"found: {begin}" ) except StopIteration: raise ValueError( "Ran out of tokens before starting to parse " "a Begin-Aggegation-Statement." ) try: self.parse_around_equals(tokens) except ValueError: tokens.throw( ValueError, f'Expecting an equals sign after "{begin}" ' ) block_name = next(tokens) if not block_name.is_parameter_name(): tokens.throw( ValueError, f'Expecting a Block-Name after "{begin} =" ' f'but found: "{block_name}"', ) self.parse_statement_delimiter(tokens) return begin, str(block_name)
def throw(self, *args, **kwargs): return Generator.throw(self, *args, **kwargs)
def throw(self, *args, **kwargs): # noqa: D102 return Generator.throw(self, *args, **kwargs)
def throw(self, exception_type, value=None, trace_back=None): return Generator.throw(self, exception_type, value, trace_back)
def throw(self, *args, **kwargs): return Generator.throw(self, *args, **kwargs)