def tokenization(self) -> TokenSequence:
file_specs = collection.intersperse_list(
TokenSequence.new_line(),
[fs.tokenization() for fs in self._files])
return TokenSequence.concat([
TokenSequence.singleton(self.delimiter__begin),
TokenSequence.preceded_by_optional_new_line_if_non_empty(
TokenSequence.concat(file_specs), ),
TokenSequence.optional_new_line(),
TokenSequence.singleton(self.delimiter__end),
])
def tokenization(self) -> TokenSequence:
return TokenSequence.concat([
TokenSequence.singleton(reserved_words.PAREN_BEGIN),
TokenSequence.optional_new_line(),
self._element.tokenization(),
self._end_paren()
])
def tokenization(self) -> TokenSequence:
return TokenSequence.concat([
TokenSequence.singleton(
instruction_arguments.ASSIGNMENT_OPERATOR,
),
self.value.tokenization(),
])
def tokenization(self) -> TokenSequence:
return TokenSequence.concat([
TokenSequence.singleton(file_matcher.PROGRAM_MATCHER_NAME),
TokenSequence.optional_new_line(),
abstract_syntaxes.OptionallyOnNewLine(self._path_argument_position).tokenization(),
self._program.tokenization(),
])
def tokenization(self) -> TokenSequence:
return TokenSequence.concat([
TokenSequence.singleton(self._file_type),
TokenSequence.optional_new_line(),
self._file_name.tokenization(),
self._contents.tokenization(),
])
def _variant_tokenization(self) -> TokenSequence:
return TokenSequence.concat([
self.var_name.tokenization(),
TokenSequence.optional_new_line(),
TokenSequence.singleton(defs.ASSIGNMENT_IDENTIFIER),
TokenSequence.optional_new_line(),
self.value.tokenization(),
])
def tokenization(self) -> TokenSequence:
return TokenSequence.concat([
self._path.tokenization(),
TokenSequence.optional_new_line(),
TokenSequence.singleton(reserved_words.COLON),
TokenSequence.optional_new_line(),
self._matcher.tokenization(),
])
def symbol_reference_followed_by_superfluous_string_on_same_line(
symbol_name: str = 'STRING_MATCHER_SYMBOL_NAME', ) -> StringMatcherAbsStx:
return CustomStringMatcherAbsStx(
TokenSequence.concat([
symbol_tok_seq.SymbolReferenceAsEitherPlainNameOrReferenceSyntax(
symbol_name),
TokenSequence.singleton('superfluous')
]))
def _file_matcher_tokens(self) -> TokenSequence:
if not self._file_matcher:
return TokenSequence.empty()
else:
return TokenSequence.concat([
TokenSequence.singleton(':'),
TokenSequence.optional_new_line(),
self._file_matcher.tokenization(),
])
def test_fail_when_missing_end_end_paren(self):
# ARRANGE #
valid_string = str_abs_stx.StringLiteralAbsStx('contents')
missing_end_paren = CustomAbsStx(
TokenSequence.concat([
TokenSequence.singleton('('),
valid_string.tokenization(),
]))
# ACT & ASSERT #
parse_check.checker().check_invalid_syntax__abs_stx(
self, OptionallyOnNewLine(missing_end_paren))
def test_fail_when_missing_end_end_paren(self):
# ARRANGE #
valid_program = ProgramOfSymbolReferenceAbsStx('PROGRAM_SYMBOL')
missing_end_paren = CustomAbsStx(
TokenSequence.concat([
TokenSequence.singleton('('),
valid_program.tokenization(),
])
)
# ACT & ASSERT #
PARSE_CHECKER.check_invalid_syntax__abs_stx(
self,
OptionallyOnNewLine(missing_end_paren)
)
def tokenization(self) -> TokenSequence:
return TokenSequence.concat([
TokenSequence.singleton(names.REPLACE_TRANSFORMER_NAME),
TokenSequence.optional_new_line(),
token_sequences.FollowedByOptionalNewLineIfNonEmpty(
self.lines_filter.tokenization()
),
token_sequences.FollowedByOptionalNewLineIfNonEmpty(
token_sequences.OptionalOption.of_option_name(
names.PRESERVE_NEW_LINES_OPTION_NAME,
self.preserve_new_lines,
)
),
self.regex_token.tokenization(),
TokenSequence.optional_new_line(),
self.replacement_token.tokenization(),
])
def runTest(self):
# ARRANGE #
invalid_modification_types = [':', 2 * syntax.EXPLICIT_CREATE, 'text']
for invalid_modification_type in invalid_modification_types:
for file_type in abs_stx.FileType:
missing_contents_file_spec = abs_stx.FileSpecAbsStx.of_file_type(
file_type, StringLiteralAbsStx('valid_file_name'),
abs_stx.CustomContentsAbsStx(
abs_stx.ContentsAbsStx.of_modification(
TokenSequence.singleton(invalid_modification_type),
CustomAbsStx.singleton(A_VALID_SYMBOL_NAME),
), ))
literal_syntax = abs_stx.LiteralFilesSourceAbsStx(
[missing_contents_file_spec])
# ACT & ASSERT #
integration_check.PARSE_CHECKER__FULL.check_invalid_syntax__abs_stx(
self,
literal_syntax,
sub_test_identifiers={
'file_type': file_type,
'invalid_modification_type': invalid_modification_type,
})
def of_str(value: str) -> StringMatcherAbsStx:
return CustomStringMatcherAbsStx(TokenSequence.singleton(value))
def tokenization(self) -> TokenSequence:
return TokenSequence.singleton(SectionName(self._name).syntax)
def tokenization(self) -> TokenSequence:
return TokenSequence.concat([
TokenSequence.singleton(matcher_options.EQUALS_ARGUMENT),
TokenSequence.optional_new_line(),
self._expected.tokenization(),
])
def tokenization(self) -> TokenSequence:
return TokenSequence.concat([
TokenSequence.singleton(matcher_options.EMPTY_ARGUMENT),
])
def singleton(token: Token) -> AbstractSyntax:
return CustomAbstractSyntax(TokenSequence.singleton(token))
def tokenization(self) -> TokenSequence:
return TokenSequence.concat([
TokenSequence.singleton(names.RUN_PROGRAM_TRANSFORMER_NAME),
TokenSequence.optional_new_line(),
self._program.tokenization(),
])
def followed_by_str(first: AbstractSyntax,
following: str) -> AbstractSyntax:
return SequenceAbsStx([
first,
CustomAbstractSyntax(TokenSequence.singleton(following)),
])
def tokenization(self) -> TokenSequence:
return TokenSequence.singleton(self._token())
def followed_by_superfluous(first: AbstractSyntax) -> AbstractSyntax:
return SequenceAbsStx([
first,
CustomAbstractSyntax(TokenSequence.singleton('superfluous')),
])
def tokenization(self) -> TokenSequence:
return TokenSequence.concat([
TokenSequence.singleton(matcher.RUN_PROGRAM),
TokenSequence.optional_new_line(),
self._program.tokenization(),
])
def preceded_by_str(first: str,
following: AbstractSyntax) -> AbstractSyntax:
return SequenceAbsStx([
CustomAbstractSyntax(TokenSequence.singleton(first)),
following,
])
def of_option_name__str_arg(
option: OptionName,
value: str,
) -> 'OptionWMandatoryValue':
return OptionWMandatoryValue.of_option_name(
option, TokenSequence.singleton(value))
def _operator_tok_seq(self) -> TokenSequence:
return (TokenSequence.sequence(
(tokens.OPTIONAL_NEW_LINE, self._operator,
tokens.OPTIONAL_NEW_LINE))
if self._allow_elements_on_separate_lines else
TokenSequence.singleton(self._operator))
def tokenization(self) -> TokenSequence:
return TokenSequence.concat([
TokenSequence.singleton(defs.CONTINUATION_TOKEN),
TokenSequence.new_line(),
self._argument_on_next_line.tokenization(),
])
def tokenization(self) -> TokenSequence:
return TokenSequence.concat([
TokenSequence.singleton(self._name),
self._argument.tokenization(),
])
def tokenization(self) -> TokenSequence:
return TokenSequence.concat([
TokenSequence.singleton(logic.NOT_OPERATOR_NAME),
self.operand.tokenization(),
])
def tokenization(self) -> TokenSequence:
return TokenSequence.concat([
TokenSequence.singleton(logic.NOT_OPERATOR_NAME),
TokenSequence.optional_new_line(),
self._matcher.tokenization(),
])
