def act_and_assert( source_markdown, expected_gfm, expected_tokens, show_debug=False, config_map=None, disable_consistency_checks=False, ): """ Act and assert on the expected behavior of parsing the source_markdown. """ # Arrange logging.getLogger().setLevel(logging.DEBUG if show_debug else logging.WARNING) ParserLogger.sync_on_next_call() tokenizer = TokenizedMarkdown() test_properties = ApplicationProperties() if config_map: test_properties.load_from_dict(config_map) extension_manager = ExtensionManager() extension_manager.initialize(None, test_properties) extension_manager.apply_configuration() tokenizer.apply_configuration(test_properties, extension_manager) transformer = TransformToGfm() # Act actual_tokens = tokenizer.transform(source_markdown, show_debug=show_debug) actual_gfm = transformer.transform(actual_tokens) # Assert assert_if_lists_different(expected_tokens, actual_tokens) assert_if_strings_different(expected_gfm, actual_gfm) if not disable_consistency_checks: assert_token_consistency(source_markdown, actual_tokens)
class PyMarkdownLint: """ Class to provide for a simple implementation of a title case algorithm. """ available_log_maps = { "CRITICAL": logging.CRITICAL, "ERROR": logging.ERROR, "WARNING": logging.WARNING, "INFO": logging.INFO, "DEBUG": logging.DEBUG, } def __init__(self) -> None: ( self.__version_number, self.__show_stack_trace, self.default_log_level, ) = (PyMarkdownLint.__get_semantic_version(), False, "CRITICAL") self.__tokenizer: Optional[TokenizedMarkdown] = None self.__properties: ApplicationProperties = ApplicationProperties() self.__plugins: PluginManager = PluginManager() self.__extensions: ExtensionManager = ExtensionManager() @staticmethod def __get_semantic_version() -> str: file_path = __file__ assert os.path.isabs(file_path) file_path = file_path.replace(os.sep, "/") last_index = file_path.rindex("/") file_path = f"{file_path[: last_index + 1]}version.py" version_meta = runpy.run_path(file_path) return str(version_meta["__version__"]) @staticmethod def log_level_type(argument: str) -> str: """ Function to help argparse limit the valid log levels. """ if argument in PyMarkdownLint.available_log_maps: return argument raise ValueError(f"Value '{argument}' is not a valid log level.") def __parse_arguments(self) -> argparse.Namespace: parser = argparse.ArgumentParser( description="Lint any found Markdown files.") parser.add_argument( "-e", "--enable-rules", dest="enable_rules", action="store", default="", help="comma separated list of rules to enable", ) parser.add_argument( "-d", "--disable-rules", dest="disable_rules", action="store", default="", help="comma separated list of rules to disable", ) parser.add_argument( "-x-scan", dest="x_test_scan_fault", action="store_true", default="", help=argparse.SUPPRESS, ) parser.add_argument( "-x-init", dest="x_test_init_fault", action="store_true", default="", help=argparse.SUPPRESS, ) parser.add_argument( "--add-plugin", dest="add_plugin", action="append", default=None, help="path to a plugin containing a new rule to apply", ) parser.add_argument( "--config", "-c", dest="configuration_file", action="store", default=None, help="path to the configuration file to use", ) parser.add_argument( "--set", "-s", dest="set_configuration", action="append", default=None, help="manually set an individual configuration property", type=ApplicationProperties.verify_manual_property_form, ) parser.add_argument( "--strict-config", dest="strict_configuration", action="store_true", default=False, help= "throw an error if configuration is bad, instead of assuming default", ) parser.add_argument( "--stack-trace", dest="show_stack_trace", action="store_true", default=False, help= "if an error occurs, print out the stack trace for debug purposes", ) parser.add_argument( "--log-level", dest="log_level", action="store", help="minimum level required to log messages", type=PyMarkdownLint.log_level_type, choices=list(PyMarkdownLint.available_log_maps.keys()), ) parser.add_argument( "--log-file", dest="log_file", action="store", help="destination file for log messages", ) subparsers = parser.add_subparsers(dest="primary_subparser") PluginManager.add_argparse_subparser(subparsers) ExtensionManager.add_argparse_subparser(subparsers) new_sub_parser = subparsers.add_parser( "scan", help="scan the Markdown files in the specified paths") new_sub_parser.add_argument( "-l", "--list-files", dest="list_files", action="store_true", default=False, help="list the markdown files found and exit", ) new_sub_parser.add_argument( "-r", "--recurse", dest="recurse_directories", action="store_true", default=False, help="recursively scan directories", ) new_sub_parser.add_argument( "paths", metavar="path", type=str, nargs="+", help="one or more paths to scan for eligible Markdown files", ) subparsers.add_parser("version", help="version of the application") parse_arguments = parser.parse_args() if not parse_arguments.primary_subparser: parser.print_help() sys.exit(2) elif parse_arguments.primary_subparser == "version": print(f"{self.__version_number}") sys.exit(0) return parse_arguments @classmethod def __is_file_eligible_to_scan(cls, path_to_test: str) -> bool: """ Determine if the presented path is one that we want to scan. """ return path_to_test.endswith(".md") def __scan_file(self, args: argparse.Namespace, next_file: str) -> None: """ Scan a given file and call the plugin manager for any significant events. """ POGGER.info("Scanning file '$'.", next_file) context = self.__plugins.starting_new_file(next_file) try: POGGER.info("Scanning file '$' token-by-token.", next_file) source_provider = (None if args.x_test_scan_fault else FileSourceProvider(next_file)) assert self.__tokenizer actual_tokens = self.__tokenizer.transform_from_provider( source_provider) if actual_tokens and actual_tokens[-1].is_pragma: pragma_token = cast(PragmaToken, actual_tokens[-1]) self.__plugins.compile_pragmas(next_file, pragma_token.pragma_lines) actual_tokens = actual_tokens[:-1] POGGER.info("Scanning file '$' tokens.", next_file) for next_token in actual_tokens: POGGER.info("Processing token: $", next_token) self.__plugins.next_token(context, next_token) POGGER.info("Scanning file '$' line-by-line.", next_file) source_provider = FileSourceProvider(next_file) line_number, next_line = 1, source_provider.get_next_line() while next_line is not None: POGGER.info("Processing line $: $", line_number, next_line) self.__plugins.next_line(context, line_number, next_line) line_number += 1 next_line = source_provider.get_next_line() POGGER.info("Completed scanning file '$'.", next_file) self.__plugins.completed_file(context, line_number) context.report_on_triggered_rules() except Exception: context.report_on_triggered_rules() raise def __process_next_path(self, next_path: str, files_to_parse: Set[str], recurse_directories: bool) -> bool: did_find_any = False POGGER.info("Determining files to scan for path '$'.", next_path) if not os.path.exists(next_path): print( f"Provided path '{next_path}' does not exist.", file=sys.stderr, ) POGGER.debug("Provided path '$' does not exist.", next_path) elif os.path.isdir(next_path): self.__process_next_path_directory(next_path, files_to_parse, recurse_directories) did_find_any = True elif self.__is_file_eligible_to_scan(next_path): POGGER.debug( "Provided path '$' is a valid file. Adding.", next_path, ) files_to_parse.add(next_path) did_find_any = True else: POGGER.debug( "Provided path '$' is not a valid file. Skipping.", next_path, ) print( f"Provided file path '{next_path}' is not a valid file. Skipping.", file=sys.stderr, ) return did_find_any def __process_next_path_directory(self, next_path: str, files_to_parse: Set[str], recurse_directories: bool) -> None: POGGER.debug("Provided path '$' is a directory. Walking directory.", next_path) normalized_next_path = next_path.replace("\\", "/") for root, _, files in os.walk(next_path): normalized_root = root.replace("\\", "/") if not recurse_directories and normalized_root != normalized_next_path: continue normalized_root = (normalized_root[:-1] if normalized_root.endswith("/") else normalized_root) for file in files: rooted_file_path = f"{normalized_root}/{file}" if self.__is_file_eligible_to_scan(rooted_file_path): files_to_parse.add(rooted_file_path) def __determine_files_to_scan( self, eligible_paths: List[str], recurse_directories: bool) -> Tuple[List[str], bool]: did_error_scanning_files = False files_to_parse: Set[str] = set() for next_path in eligible_paths: if "*" in next_path or "?" in next_path: globbed_paths = glob.glob(next_path) if not globbed_paths: print( f"Provided glob path '{next_path}' did not match any files.", file=sys.stderr, ) did_error_scanning_files = True break for next_globbed_path in globbed_paths: next_globbed_path = next_globbed_path.replace("\\", "/") self.__process_next_path(next_globbed_path, files_to_parse, recurse_directories) elif not self.__process_next_path(next_path, files_to_parse, recurse_directories): did_error_scanning_files = True break sorted_files_to_parse = sorted(files_to_parse) POGGER.info("Number of files found: $", len(sorted_files_to_parse)) return sorted_files_to_parse, did_error_scanning_files @classmethod def __handle_list_files(cls, files_to_scan: List[str]) -> int: if files_to_scan: print(ParserHelper.newline_character.join(files_to_scan)) return 0 print("No matching files found.", file=sys.stderr) return 1 # pylint: disable=broad-except def __apply_configuration_to_plugins(self) -> None: try: self.__plugins.apply_configuration(self.__properties) except Exception as this_exception: formatted_error = ( f"{type(this_exception).__name__} encountered while configuring plugins:\n" + f"{this_exception}") self.__handle_error(formatted_error, this_exception) # pylint: enable=broad-except def __initialize_parser(self, args: argparse.Namespace) -> None: resource_path = "fredo" if args.x_test_init_fault else None try: self.__tokenizer = TokenizedMarkdown(resource_path) self.__tokenizer.apply_configuration(self.__properties, self.__extensions) except BadTokenizationError as this_exception: formatted_error = ( f"{type(this_exception).__name__} encountered while initializing tokenizer:\n" + f"{this_exception}") self.__handle_error(formatted_error, this_exception) def __initialize_plugin_manager(self, args: argparse.Namespace, plugin_dir: str) -> None: """ Make sure all plugins are ready before being initialized. """ try: self.__plugins.initialize( plugin_dir, args.add_plugin, args.enable_rules, args.disable_rules, self.__properties, self.__show_stack_trace, ) except BadPluginError as this_exception: formatted_error = ( f"BadPluginError encountered while loading plugins:\n{this_exception}" ) self.__handle_error(formatted_error, this_exception) def __handle_error(self, formatted_error: str, thrown_error: Exception) -> None: show_error = self.__show_stack_trace or not isinstance( thrown_error, ValueError) LOGGER.warning(formatted_error, exc_info=show_error) print(f"\n\n{formatted_error}", file=sys.stderr) if self.__show_stack_trace: traceback.print_exc(file=sys.stderr) sys.exit(1) def __handle_scan_error(self, next_file: str, this_exception: Exception) -> None: formatted_error = f"{type(this_exception).__name__} encountered while scanning '{next_file}':\n{this_exception}" self.__handle_error(formatted_error, this_exception) def __set_initial_state(self, args: argparse.Namespace) -> None: self.__show_stack_trace = args.show_stack_trace base_logger = logging.getLogger() base_logger.setLevel( logging.DEBUG if self.__show_stack_trace else logging.WARNING) if args.configuration_file: LOGGER.debug("Loading configuration file: %s", args.configuration_file) ApplicationPropertiesJsonLoader.load_and_set( self.__properties, args.configuration_file, self.__handle_error) if args.set_configuration: self.__properties.set_manual_property(args.set_configuration) def __initialize_strict_mode(self, args: argparse.Namespace) -> None: effective_strict_configuration = args.strict_configuration if not effective_strict_configuration: effective_strict_configuration = self.__properties.get_boolean_property( "mode.strict-config", strict_mode=True) if effective_strict_configuration: self.__properties.enable_strict_mode() def __initialize_logging( self, args: argparse.Namespace) -> Optional[logging.FileHandler]: self.__show_stack_trace = args.show_stack_trace if not self.__show_stack_trace: self.__show_stack_trace = self.__properties.get_boolean_property( "log.stack-trace") effective_log_file = (self.__properties.get_string_property("log.file") if args.log_file is None else args.log_file) new_handler = None if effective_log_file: new_handler = logging.FileHandler(effective_log_file) logging.getLogger().addHandler(new_handler) else: temp_log_level = (logging.DEBUG if self.__show_stack_trace else logging.CRITICAL) logging.basicConfig(stream=sys.stdout, level=temp_log_level) effective_log_level = args.log_level or None if effective_log_level is None: effective_log_level = self.__properties.get_string_property( "log.level", valid_value_fn=PyMarkdownLint.log_level_type) if effective_log_level is None: effective_log_level = self.default_log_level log_level_to_enact = PyMarkdownLint.available_log_maps[ effective_log_level] logging.getLogger().setLevel(log_level_to_enact) ParserLogger.sync_on_next_call() return new_handler def __initialize_plugins(self, args: argparse.Namespace) -> None: try: plugin_dir = os.path.dirname(os.path.realpath(__file__)) plugin_dir = os.path.join(plugin_dir, "plugins") self.__initialize_plugin_manager(args, plugin_dir) self.__apply_configuration_to_plugins() except ValueError as this_exception: formatted_error = ( f"{type(this_exception).__name__} encountered while initializing plugins:\n" + f"{this_exception}") self.__handle_error(formatted_error, this_exception) # pylint: disable=broad-except def __initialize_extensions(self, args: argparse.Namespace) -> None: try: self.__extensions.initialize( args, self.__properties, ) self.__extensions.apply_configuration() except ValueError as this_exception: formatted_error = ( f"Configuration error {type(this_exception).__name__} encountered " + f"while initializing extensions:\n{this_exception}") self.__handle_error(formatted_error, this_exception) except Exception as this_exception: formatted_error = ( f"Error {type(this_exception).__name__} encountered while initializing extensions:\n" + f"{this_exception}") self.__handle_error(formatted_error, this_exception) # pylint: enable=broad-except def __handle_plugins_and_extensions(self, args: argparse.Namespace) -> None: self.__initialize_plugins(args) self.__initialize_extensions(args) if args.primary_subparser == PluginManager.argparse_subparser_name(): sys.exit(self.__plugins.handle_argparse_subparser(args)) if args.primary_subparser == ExtensionManager.argparse_subparser_name( ): sys.exit(self.__extensions.handle_argparse_subparser(args)) def __handle_main_list_files(self, args: argparse.Namespace, files_to_scan: List[str]) -> None: if args.list_files: POGGER.info( "Sending list of files that would have been scanned to stdout." ) sys.exit(self.__handle_list_files(files_to_scan)) def main(self) -> None: """ Main entrance point. """ args = self.__parse_arguments() self.__set_initial_state(args) new_handler, total_error_count = None, 0 try: self.__initialize_strict_mode(args) new_handler = self.__initialize_logging(args) self.__handle_plugins_and_extensions(args) POGGER.info("Determining files to scan.") files_to_scan, did_error_scanning_files = self.__determine_files_to_scan( args.paths, args.recurse_directories) if did_error_scanning_files: total_error_count = 1 else: self.__initialize_parser(args) self.__handle_main_list_files(args, files_to_scan) for next_file in files_to_scan: try: self.__scan_file(args, next_file) except BadPluginError as this_exception: self.__handle_scan_error(next_file, this_exception) except BadTokenizationError as this_exception: self.__handle_scan_error(next_file, this_exception) except ValueError as this_exception: formatted_error = f"Configuration Error: {this_exception}" self.__handle_error(formatted_error, this_exception) finally: if new_handler: new_handler.close() if self.__plugins.number_of_scan_failures or total_error_count: sys.exit(1)