def create_entity_localization_command_sequence(
entity_name: str,
EntityExtractorType: Type[EntityExtractor],
extract_contexts: bool = False,
DetectedEntityType: Optional[Type[SerializableEntity]] = None,
upload_func: Optional[EntityUploadCallable] = None,
colorize_options: ColorizeOptions = ColorizeOptions(),
colorize_func: Optional[ColorizeFunc] = None,
) -> List[Type[Command]]: # type: ignore
"""
Create a set of commands that can be used to locate a new type of entity. In the simplest case,
all you have to provide is and 'entity_name' to be used for naming output files, and
'entity_type' that can be used to filter which commands are being run when you the full
pipeline is run, and an 'EntityExtractorType' that locates all instances of that entity in the
TeX. This function creates the commands necessary to colorize the entities, compile the
LaTeX, raster the pages, and locate the colors in the pages. You may define additional
paramters (e.g., 'colorize_options') to fine-tune the commands.
To extract the contexts for an entity (i.e., the sentences in which the entities appear),
set 'extract_contexts' to True.
If you are trying to find the locations of a new type of entity, it is highly recommended that
you use this convenience methods instead of creating new commands yourself.
"""
commands: CommandList = []
directories.register(f"detected-{entity_name}")
commands.append(
make_detect_entities_command(entity_name, EntityExtractorType))
if extract_contexts:
directories.register(f"contexts-for-{entity_name}")
commands.append(make_extract_contexts_command(entity_name))
directories.register(f"sources-with-colorized-{entity_name}")
directories.register(f"compiled-sources-with-colorized-{entity_name}")
directories.register(f"paper-images-with-colorized-{entity_name}")
directories.register(f"diffed-images-with-colorized-{entity_name}")
directories.register(f"{entity_name}-locations")
commands.append(
make_locate_entities_command(entity_name, None, DetectedEntityType,
colorize_options, colorize_func))
if upload_func is not None:
upload_command = make_upload_entities_command(
entity_name, upload_func, DetectedEntityType=DetectedEntityType)
commands.append(upload_command)
return commands
def create_entity_localization_command_sequence(
entity_name: str,
EntityExtractorType: Type[EntityExtractor],
DetectedEntityType: Optional[Type[SerializableEntity]] = None,
upload_func: Optional[EntityUploadCallable] = None,
colorize_entity_when: Optional[ColorWhenFunc] = None,
get_color_positions: Optional[ColorPositionsFunc] = None,
) -> List[Type[Command]]: # type: ignore
"""
Create a set of commands that can be used to locate a new type of entity. In the simplest case,
all you have to provide is and 'entity_name' to be used for naming output files, and
'entity_type' that can be used to filter which commands are being run when you the full
pipeline is run, and an 'EntityExtractorType' that locates all instances of that entity in the
TeX. This function creates the commands necessary to colorize the entities, compile the
LaTeX, raster the pages, and locate the colors in the pages. You may define additional
paramters (e.g., 'colorize_entity_when') to fine-tune the commands.
If you are trying to find the locations of a new type of entity, it is highly recommended that
you use this convenience methods instead of creating new commands yourself.
"""
# Register directories for output from intermediate pipeline stages.
directories.register(f"detected-{entity_name}")
directories.register(f"sources-with-colorized-{entity_name}")
directories.register(f"compiled-sources-with-colorized-{entity_name}")
directories.register(f"paper-with-colorized-{entity_name}-images")
directories.register(f"diff-images-with-colorized-{entity_name}")
directories.register(f"hue-locations-for-{entity_name}")
commands: CommandList = [
make_detect_entities_command(entity_name, EntityExtractorType),
make_colorize_tex_command(
entity_name=entity_name,
DetectedEntityType=DetectedEntityType,
when=colorize_entity_when,
get_color_positions=get_color_positions,
),
make_compile_tex_command(entity_name),
make_raster_pages_command(entity_name),
make_diff_images_command(entity_name),
make_locate_hues_command(entity_name),
]
if upload_func is not None:
upload_command = make_upload_entities_command(
entity_name, upload_func, DetectedEntityType=DetectedEntityType
)
commands.append(upload_command)
return commands
),
make_locate_entities_command(
"symbols-with-affixes",
input_entity_name="symbols",
DetectedEntityType=SerializableSymbol,
colorize_options=ColorizeOptions(
adjust_color_positions=adjust_color_positions,
braces=True,
when=filter_symbols_with_affixes,
group=divide_symbols_into_nonoverlapping_groups,
),
),
LocateCompositeSymbols,
CollectSymbolLocations,
make_upload_entities_command("symbols",
upload_symbols,
DetectedEntityType=SerializableSymbol),
]
def make_digest(_: str, arxiv_id: ArxivId) -> EntityProcessingDigest:
"""
Custom digest creator. Count the equation tokens, instead of the 'symbols', as we can
use the default entity counters for the outputs of equation token commands.
"""
return make_default_paper_digest("equation-tokens", arxiv_id)
symbols_pipeline = EntityPipeline(
"symbols",
commands,
from .upload import upload_definitions
# Register directories for output from intermediate pipeline stages.
directories.register("embellished-sentences")
directories.register("detected-definitions")
directories.register("sources-with-colorized-definitions")
directories.register("compiled-sources-with-colorized-definitions")
directories.register("paper-images-with-colorized-definitions")
directories.register("diffed-images-with-colorized-definitions")
directories.register("definitions-locations")
upload_command = make_upload_entities_command(
"definitions",
upload_definitions,
DetectedEntityType={
"entities-definiendums.csv": Definiendum,
"entities-definitions.csv": Definition,
"entities-term-references.csv": TermReference,
},
)
commands: CommandList = [
EmbellishSentences,
DetectDefinitions,
make_locate_entities_command("definitions"),
upload_command,
]
definitions_pipeline = EntityPipeline(
"definitions",
commands,