def test__templated_file_templated_slice_to_source_slice(
in_slice, out_slice, is_literal, file_slices, raw_slices):
"""Test TemplatedFile.templated_slice_to_source_slice."""
file = TemplatedFile(source_str="Dummy String",
sliced_file=file_slices,
raw_sliced=raw_slices)
source_slice = file.templated_slice_to_source_slice(in_slice)
literal_test = file.is_source_slice_literal(source_slice)
assert (is_literal, source_slice) == (literal_test, out_slice)
def test__templated_file_templated_slice_to_source_slice(
in_slice, out_slice, is_literal, file_slices, raw_slices):
"""Test TemplatedFile.templated_slice_to_source_slice."""
file = TemplatedFile(
source_str="Dummy String",
sliced_file=file_slices,
raw_sliced=[
rs if isinstance(rs, RawFileSlice) else RawFileSlice(*rs)
for rs in raw_slices
],
fname="test",
)
source_slice = file.templated_slice_to_source_slice(in_slice)
literal_test = file.is_source_slice_literal(source_slice)
assert (is_literal, source_slice) == (literal_test, out_slice)
def enrich_segments(
segment_buff: Tuple[BaseSegment, ...], templated_file: TemplatedFile
) -> Tuple[BaseSegment, ...]:
"""Enrich the segments using the templated file.
We use the mapping in the template to provide positions
in the source file.
"""
# Make a new buffer to hold the enriched segments.
# We need a new buffer to hold the new meta segments
# introduced.
new_segment_buff = []
# Get the templated slices to re-insert tokens for them
source_only_slices = templated_file.source_only_slices()
lexer_logger.info(
"Enriching Segments. Source-only slices: %s", source_only_slices
)
for segment in segment_buff:
templated_slice = slice(
segment.pos_marker.char_pos,
segment.pos_marker.char_pos + len(segment.raw),
)
source_slice = templated_file.templated_slice_to_source_slice(
templated_slice
)
# At this stage, templated slices will be INCLUDED in the source slice,
# so we should consider whether we've captured any. If we have then
# we need to re-evaluate whether it's a literal or not.
for source_only_slice in source_only_slices:
if source_only_slice.source_idx > source_slice.start:
break
elif source_only_slice.source_idx == source_slice.start:
lexer_logger.debug(
"Found templated section! %s, %s, %s",
source_only_slice.source_slice(),
source_only_slice.slice_type,
templated_slice.start,
)
# Adjust the source slice accordingly.
source_slice = slice(
source_only_slice.end_source_idx(), source_slice.stop
)
# Add segments as appropriate.
# If it's a block end, add a dedent.
if source_only_slice.slice_type in ("block_end", "block_mid"):
new_segment_buff.append(
Dedent.when(template_blocks_indent=True)(
pos_marker=segment.pos_marker
)
)
# Always add a placeholder
new_segment_buff.append(
TemplateSegment(
pos_marker=segment.pos_marker,
source_str=source_only_slice.raw,
block_type=source_only_slice.slice_type,
)
)
# If it's a block end, add a dedent.
if source_only_slice.slice_type in ("block_start", "block_mid"):
new_segment_buff.append(
Indent.when(template_blocks_indent=True)(
pos_marker=segment.pos_marker
)
)
source_line, source_pos = templated_file.get_line_pos_of_char_pos(
source_slice.start
)
# Recalculate is_literal
is_literal = templated_file.is_source_slice_literal(source_slice)
segment.pos_marker = EnrichedFilePositionMarker(
statement_index=segment.pos_marker.statement_index,
line_no=segment.pos_marker.line_no,
line_pos=segment.pos_marker.line_pos,
char_pos=segment.pos_marker.char_pos,
templated_slice=templated_slice,
source_slice=source_slice,
is_literal=is_literal,
source_pos_marker=FilePositionMarker(
segment.pos_marker.statement_index,
source_line,
source_pos,
source_slice.start,
),
)
new_segment_buff.append(segment)
lexer_logger.debug("Enriched Segments:")
for seg in new_segment_buff:
lexer_logger.debug(
"\tTmp: %s\tSrc: %s\tSeg: %s",
getattr(seg.pos_marker, "templated_slice", None),
getattr(seg.pos_marker, "source_slice", None),
seg,
)
return tuple(new_segment_buff)