def elements_to_segments(
self, elements: List[TemplateElement],
templated_file: TemplatedFile) -> Tuple[RawSegment, ...]:
"""Convert a tuple of lexed elements into a tuple of segments."""
# Working buffer to build up segments
segment_buffer: List[RawSegment] = []
lexer_logger.info("Elements to Segments.")
# Get the templated slices to re-insert tokens for them
source_only_slices = templated_file.source_only_slices()
lexer_logger.info("Source-only slices: %s", source_only_slices)
# Now work out source slices, and add in template placeholders.
for element in elements:
# Calculate Source Slice
source_slice = templated_file.templated_slice_to_source_slice(
element.template_slice)
# The calculated source slice will include any source only slices.
# We should consider all of them in turn to see whether we can
# insert them.
for source_only_slice in source_only_slices:
# If it's later in the source, stop looking. Any later
# ones *also* won't match.
if source_only_slice.source_idx > source_slice.start:
break
# Is there a templated section within this source slice?
# If there is then for some reason I can't quite explain,
# it will always be at the start of the section. This is
# very convenient beause it means we'll always have the
# start and end of it in a definite position. This makes
# slicing and looping much easier.
elif source_only_slice.source_idx == source_slice.start:
lexer_logger.debug(
"Found templated section! %s, %s, %s",
source_only_slice.source_slice(),
source_only_slice.slice_type,
element.template_slice.start,
)
# Calculate a slice for any placeholders
placeholder_source_slice = slice(
source_slice.start, source_only_slice.end_source_idx())
# Adjust the source slice accordingly.
source_slice = slice(source_only_slice.end_source_idx(),
source_slice.stop)
# TODO: Readjust this to remove .when once ProtoSegment is in.
# Add segments as appropriate.
# If it's a block end, add a dedent.
if source_only_slice.slice_type in ("block_end",
"block_mid"):
segment_buffer.append(
Dedent.when(template_blocks_indent=True)(
pos_marker=PositionMarker.from_point(
placeholder_source_slice.start,
element.template_slice.start,
templated_file,
)))
# Always add a placeholder
segment_buffer.append(
TemplateSegment(
pos_marker=PositionMarker(
placeholder_source_slice,
slice(
element.template_slice.start,
element.template_slice.start,
),
templated_file,
),
source_str=source_only_slice.raw,
block_type=source_only_slice.slice_type,
))
# If it's a block end, add a dedent.
if source_only_slice.slice_type in ("block_start",
"block_mid"):
segment_buffer.append(
Indent.when(template_blocks_indent=True)(
pos_marker=PositionMarker.from_point(
placeholder_source_slice.stop,
element.template_slice.start,
templated_file,
)))
# Add the actual segment
segment_buffer.append(
element.to_segment(pos_marker=PositionMarker(
source_slice,
element.template_slice,
templated_file,
), ))
# Convert to tuple before return
return tuple(segment_buffer)
def enrich_segments(
segment_buff: Tuple[BaseSegment, ...], templated_file: TemplatedFile
) -> Tuple[BaseSegment, ...]:
"""Enrich the segments using the templated file.
We use the mapping in the template to provide positions
in the source file.
"""
# Make a new buffer to hold the enriched segments.
# We need a new buffer to hold the new meta segments
# introduced.
new_segment_buff = []
# Get the templated slices to re-insert tokens for them
source_only_slices = templated_file.source_only_slices()
lexer_logger.info(
"Enriching Segments. Source-only slices: %s", source_only_slices
)
for segment in segment_buff:
templated_slice = slice(
segment.pos_marker.char_pos,
segment.pos_marker.char_pos + len(segment.raw),
)
source_slice = templated_file.templated_slice_to_source_slice(
templated_slice
)
# At this stage, templated slices will be INCLUDED in the source slice,
# so we should consider whether we've captured any. If we have then
# we need to re-evaluate whether it's a literal or not.
for source_only_slice in source_only_slices:
if source_only_slice.source_idx > source_slice.start:
break
elif source_only_slice.source_idx == source_slice.start:
lexer_logger.debug(
"Found templated section! %s, %s, %s",
source_only_slice.source_slice(),
source_only_slice.slice_type,
templated_slice.start,
)
# Adjust the source slice accordingly.
source_slice = slice(
source_only_slice.end_source_idx(), source_slice.stop
)
# Add segments as appropriate.
# If it's a block end, add a dedent.
if source_only_slice.slice_type in ("block_end", "block_mid"):
new_segment_buff.append(
Dedent.when(template_blocks_indent=True)(
pos_marker=segment.pos_marker
)
)
# Always add a placeholder
new_segment_buff.append(
TemplateSegment(
pos_marker=segment.pos_marker,
source_str=source_only_slice.raw,
block_type=source_only_slice.slice_type,
)
)
# If it's a block end, add a dedent.
if source_only_slice.slice_type in ("block_start", "block_mid"):
new_segment_buff.append(
Indent.when(template_blocks_indent=True)(
pos_marker=segment.pos_marker
)
)
source_line, source_pos = templated_file.get_line_pos_of_char_pos(
source_slice.start
)
# Recalculate is_literal
is_literal = templated_file.is_source_slice_literal(source_slice)
segment.pos_marker = EnrichedFilePositionMarker(
statement_index=segment.pos_marker.statement_index,
line_no=segment.pos_marker.line_no,
line_pos=segment.pos_marker.line_pos,
char_pos=segment.pos_marker.char_pos,
templated_slice=templated_slice,
source_slice=source_slice,
is_literal=is_literal,
source_pos_marker=FilePositionMarker(
segment.pos_marker.statement_index,
source_line,
source_pos,
source_slice.start,
),
)
new_segment_buff.append(segment)
lexer_logger.debug("Enriched Segments:")
for seg in new_segment_buff:
lexer_logger.debug(
"\tTmp: %s\tSrc: %s\tSeg: %s",
getattr(seg.pos_marker, "templated_slice", None),
getattr(seg.pos_marker, "source_slice", None),
seg,
)
return tuple(new_segment_buff)