def produce_report(self): result_obj = copy.deepcopy(self.master) # Remove output data if result is simple (except if we are in debug mode) if self.report_type == "simple" and config.get("LOG_LEVEL", "") != "DEBUG": result_obj = self.filter_fields(result_obj, self.megalinter_fields) result_obj.linters = filter(lambda x: x.is_active is True, result_obj.linters) result_obj.linters = list( map( lambda x: self.filter_fields(x, self.linter_fields), result_obj.linters, )) result_obj.reporters = list( map(lambda x: self.filter_fields(x, []), result_obj.reporters)) for reporter in result_obj.reporters: setattr(reporter, "name", reporter.name) # Generate JSON from object using jsonpickle result_json = jsonpickle.encode(result_obj, unpicklable=False, max_depth=self.max_depth, indent=4) # unserialize + serialize to sort object keys result_json_obj = json.loads(result_json) result_json = json.dumps(result_json_obj, sort_keys=True, indent=4) # Write output file json_file_name = f"{self.report_folder}{os.path.sep}" + config.get( "JSON_REPORTER_FILE_NAME", "mega-linter-report.json") with open(json_file_name, "w", encoding="utf-8") as json_file: json_file.write(result_json) logging.info( f"[JSON Reporter] Generated {self.name} report: {json_file_name}" )
def initialize_logger(self): logging_level_key = config.get("LOG_LEVEL", "INFO").upper() logging_level_list = { "INFO": logging.INFO, "DEBUG": logging.DEBUG, "WARNING": logging.WARNING, "ERROR": logging.ERROR, # Previous values for v3 ascending compatibility "TRACE": logging.WARNING, "VERBOSE": logging.INFO, } logging_level = ( logging_level_list[logging_level_key] if logging_level_key in logging_level_list else logging.INFO ) log_file = ( self.report_folder + os.path.sep + config.get("LOG_FILE", "mega-linter.log") ) if not os.path.isdir(os.path.dirname(log_file)): os.makedirs(os.path.dirname(log_file), exist_ok=True) logging.basicConfig( force=True, level=logging_level, format="%(message)s", handlers=[ logging.FileHandler(log_file, "w", "utf-8"), logging.StreamHandler(sys.stdout), ], )
def test_remote_config_extends_success_2(self): remote_config = self.test_folder + "base2.mega-linter.yml" os.environ["MEGALINTER_CONFIG"] = remote_config config.init_config() self.assertEqual("(base)", config.get("FILTER_REGEX_INCLUDE")) self.assertEqual("(extension2)", config.get("FILTER_REGEX_EXCLUDE")) self.assertEqual("true", config.get("SHOW_ELAPSED_TIME"))
def linter_test_setup(params=None): config.init_config(None) if params is None: params = {} # Root to lint sub_lint_root = (params["sub_lint_root"] if "sub_lint_root" in params else f"{os.path.sep}.automation{os.path.sep}test") # Ignore report folder config.set_value("FILTER_REGEX_EXCLUDE", "\\/(report)\\/") # TAP Output deactivated by default config.set_value("OUTPUT_FORMAT", "text") config.set_value("OUTPUT_DETAIL", "detailed") # Root path of default rules root_dir = ("/tmp/lint" if os.path.isdir("/tmp/lint") else os.path.relpath( os.path.relpath(os.path.dirname(os.path.abspath(__file__))) + "/../../../..")) config.set_value("VALIDATE_ALL_CODEBASE", "true") # Root path of files to lint config.set_value( "DEFAULT_WORKSPACE", (config.get("DEFAULT_WORKSPACE") + sub_lint_root if config.exists("DEFAULT_WORKSPACE") and os.path.isdir(config.get("DEFAULT_WORKSPACE") + sub_lint_root) else root_dir + sub_lint_root), ) assert os.path.isdir( config.get("DEFAULT_WORKSPACE")), ("DEFAULT_WORKSPACE " + config.get("DEFAULT_WORKSPACE") + " is not a valid folder")
def list_files_git_diff(self): # List all updated files from git logging.info( "Listing updated files in [" + self.github_workspace + "] using git diff, then filter with:" ) repo = git.Repo(os.path.realpath(self.github_workspace)) default_branch = config.get("DEFAULT_BRANCH", "master") current_branch = config.get("GITHUB_SHA", "") if current_branch == "": current_branch = repo.active_branch.commit.hexsha try: repo.git.pull() except git.GitCommandError: try: repo.git.checkout(current_branch) repo.git.pull() except git.GitCommandError: logging.info(f"Warning: Unable to pull current branch {current_branch}") repo.git.checkout(default_branch) diff = repo.git.diff(f"{default_branch}...{current_branch}", name_only=True) repo.git.checkout(current_branch) logging.info(f"Git diff :\n{diff}") all_files = list() for diff_line in diff.splitlines(): if os.path.isfile(self.workspace + os.path.sep + diff_line): all_files += [self.workspace + os.path.sep + diff_line] return all_files
def display_header(): # Header prints logging.info(utils.format_hyphens("")) logging.info(utils.format_hyphens("Mega-Linter")) logging.info(utils.format_hyphens("")) logging.info( " - Image Creation Date: " + config.get("BUILD_DATE", "No docker image") ) logging.info( " - Image Revision: " + config.get("BUILD_REVISION", "No docker image") ) logging.info( " - Image Version: " + config.get("BUILD_VERSION", "No docker image") ) logging.info(utils.format_hyphens("")) logging.info("The Mega-Linter documentation can be found at:") logging.info(" - https://nvuillam.github.io/mega-linter") logging.info(utils.format_hyphens("")) logging.info("GITHUB_REPOSITORY: " + os.environ.get("GITHUB_REPOSITORY", "")) logging.info("GITHUB_SHA: " + os.environ.get("GITHUB_SHA", "")) logging.info("GITHUB_TOKEN: " + os.environ.get("GITHUB_TOKEN", "")) logging.info("GITHUB_RUN_ID: " + os.environ.get("GITHUB_RUN_ID", "")) logging.info("PAT: " + "set" if os.environ.get("PAT", "") != "" else "") # Display config variables for debug mode for name, value in sorted(config.get_config().items()): logging.debug("" + name + "=" + str(value)) logging.debug(utils.format_hyphens("")) logging.info("")
def __init__(self, params=None): if params is None: params = {} self.workspace = self.get_workspace() config.init_config(self.workspace) # Initialize runtime config self.github_workspace = config.get("GITHUB_WORKSPACE", self.workspace) self.report_folder = config.get( "REPORT_OUTPUT_FOLDER", config.get("OUTPUT_FOLDER", self.github_workspace + os.path.sep + "report"), ) self.initialize_logger() self.display_header() # Mega-Linter default rules location self.default_rules_location = ( "/action/lib/.automation" if os.path.isdir("/action/lib/.automation") else os.path.relpath( os.path.relpath( os.path.dirname(os.path.abspath(__file__)) + "/../TEMPLATES" ) ) ) # User-defined rules location self.linter_rules_path = self.github_workspace + os.path.sep + ".github/linters" self.validate_all_code_base = True self.filter_regex_include = None self.filter_regex_exclude = None self.cli = params["cli"] if "cli" in params else False self.default_linter_activation = True # Get enable / disable vars self.enable_descriptors = config.get_list("ENABLE", []) self.enable_linters = config.get_list("ENABLE_LINTERS", []) self.disable_descriptors = config.get_list("DISABLE", []) self.disable_linters = config.get_list("DISABLE_LINTERS", []) self.manage_default_linter_activation() self.apply_fixes = config.get_list("APPLY_FIXES", "none") self.show_elapsed_time = ( config.get("SHOW_ELAPSED_TIME", "false") == "true" or config.get("LOG_LEVEL", "DEBUG") == "DEBUG" ) # Load optional configuration self.load_config_vars() # Runtime properties self.reporters = [] self.linters = [] self.file_extensions = [] self.file_names_regex = [] self.status = "success" self.return_code = 0 self.has_updated_sources = 0 self.flavor_suggestions = None # Initialize plugins plugin_factory.initialize_plugins() # Initialize linters and gather criteria to browse files self.load_linters() self.compute_file_extensions() # Load Mega-Linter reporters self.load_reporters()
def manage_activation(self, params): # Default value is false in case ENABLE variables are used if len(params["enable_descriptors"]) > 0 or len( params["enable_linters"]) > 0: self.is_active = False # Activate or not the linter if self.name in params["enable_linters"]: self.is_active = True elif self.name in params["disable_linters"]: self.is_active = False elif (self.descriptor_id in params["disable_descriptors"] or self.name in params["disable_linters"]): self.is_active = False elif self.descriptor_id in params["enable_descriptors"]: self.is_active = True elif (config.exists("VALIDATE_" + self.name) and config.get("VALIDATE_" + self.name) == "false"): self.is_active = False elif (config.exists("VALIDATE_" + self.descriptor_id) and config.get("VALIDATE_" + self.descriptor_id) == "false"): self.is_active = False elif (config.exists("VALIDATE_" + self.name) and config.get("VALIDATE_" + self.name) == "true"): self.is_active = True elif (config.exists("VALIDATE_" + self.descriptor_id) and config.get("VALIDATE_" + self.descriptor_id) == "true"): self.is_active = True # check activation rules if self.is_active is True and len(self.activation_rules) > 0: self.is_active = utils.check_activation_rules( self.activation_rules, self)
def manage_activation(self): if config.get("GITHUB_COMMENT_REPORTER", "true") != "true": self.is_active = False elif ( config.get("POST_GITHUB_COMMENT", "true") == "true" ): # Legacy - true by default self.is_active = True
def manage_activation(self, params): if self.name in params["enable_linters"]: self.is_active = True elif self.name in params["disable_linters"]: self.is_active = False elif ( self.descriptor_id in params["disable_descriptors"] or self.name in params["disable_linters"] ): self.is_active = False elif self.descriptor_id in params["enable_descriptors"]: self.is_active = True elif ( config.exists("VALIDATE_" + self.name) and config.get("VALIDATE_" + self.name) == "false" ): self.is_active = False elif ( config.exists("VALIDATE_" + self.descriptor_id) and config.get("VALIDATE_" + self.descriptor_id) == "false" ): self.is_active = False elif ( config.exists("VALIDATE_" + self.name) and config.get("VALIDATE_" + self.name) == "true" ): self.is_active = True elif ( config.exists("VALIDATE_" + self.descriptor_id) and config.get("VALIDATE_" + self.descriptor_id) == "true" ): self.is_active = True
def list_files_git_diff(self): # List all updated files from git logging.info("Listing updated files in [" + self.github_workspace + "] using git diff.") repo = git.Repo(os.path.realpath(self.github_workspace)) # Add auth header if necessary if config.get("GIT_AUTHORIZATION_BEARER", "") != "": auth_bearer = "Authorization: Bearer " + config.get( "GIT_AUTHORIZATION_BEARER") repo.config_writer().set_value("http", "extraheader", auth_bearer).release() self.has_git_extraheader = True # Fetch base branch content default_branch = config.get("DEFAULT_BRANCH", "HEAD") default_branch_remote = f"origin/{default_branch}" if default_branch_remote not in [ref.name for ref in repo.refs]: remote_ref = ("HEAD" if default_branch == "HEAD" else f"refs/heads/{default_branch}") local_ref = f"refs/remotes/{default_branch_remote}" # Try to fetch default_branch from origin, because it isn't cached locally. repo.git.fetch("origin", f"{remote_ref}:{local_ref}") # Make git diff to list files diff = repo.git.diff(default_branch_remote, name_only=True) logging.info(f"Modified files:\n{diff}") all_files = list() for diff_line in diff.splitlines(): if os.path.isfile(self.workspace + os.path.sep + diff_line): all_files += [self.workspace + os.path.sep + diff_line] return all_files
def manage_activation(self): if config.get("EMAIL_REPORTER", "true") != "true": self.is_active = False elif config.get("EMAIL_REPORTER_EMAIL", "none") == "none": logging.info( "To receive reports as email, please set variable EMAIL_REPORTER_EMAIL" ) self.is_active = False
def linter_test_setup(params=None): for key in [ "MEGALINTER_CONFIG", "EXTENDS", "FILTER_REGEX_INCLUDE", "FILTER_REGEX_EXCLUDE", "SHOW_ELAPSED_TIME", ]: if key in os.environ: del os.environ[key] config.delete() if params is None: params = {} # Root to lint sub_lint_root = ( params["sub_lint_root"] if "sub_lint_root" in params else f"{os.path.sep}.automation{os.path.sep}test" ) # Root path of default rules root_dir = ( "/tmp/lint" if os.path.isdir("/tmp/lint") else os.path.relpath( os.path.relpath(os.path.dirname(os.path.abspath(__file__))) + "/../../../.." ) ) workspace = None config_file_path = root_dir + sub_lint_root + os.path.sep + ".mega-linter.yml" if os.path.isfile(config_file_path): workspace = root_dir + sub_lint_root elif params.get("required_config_file", False) is True: raise Exception( f"[test] There should be a .mega-linter.yml file in test folder {config_file_path}" ) config.init_config(workspace) # Ignore report folder config.set_value("FILTER_REGEX_EXCLUDE", r"\/report\/") # TAP Output deactivated by default config.set_value("OUTPUT_FORMAT", "text") config.set_value("OUTPUT_DETAIL", "detailed") config.set_value("PLUGINS", "") config.set_value("VALIDATE_ALL_CODEBASE", "true") # Root path of files to lint config.set_value( "DEFAULT_WORKSPACE", ( config.get("DEFAULT_WORKSPACE") + sub_lint_root if config.exists("DEFAULT_WORKSPACE") and os.path.isdir(config.get("DEFAULT_WORKSPACE") + sub_lint_root) else root_dir + sub_lint_root ), ) assert os.path.isdir(config.get("DEFAULT_WORKSPACE")), ( "DEFAULT_WORKSPACE " + config.get("DEFAULT_WORKSPACE") + " is not a valid folder" )
def run(self): # Collect files for each identified linter self.collect_files() # Process linters serial or parallel according to configuration active_linters = [] linters_do_fixes = False for linter in self.linters: if linter.is_active is True: active_linters += [linter] if linter.apply_fixes is True: linters_do_fixes = True # Initialize reports for reporter in self.reporters: reporter.initialize() # Display warning if selected flavors does not match all linters if flavor_factory.check_active_linters_match_flavor(active_linters) is False: active_linters = [ linter for linter in active_linters if linter.is_active is True ] if config.get("PARALLEL", "true") == "true" and len(active_linters) > 1: self.process_linters_parallel(active_linters, linters_do_fixes) else: self.process_linters_serial(active_linters, linters_do_fixes) # Update main Mega-Linter status according to results of linters run for linter in self.linters: if linter.status != "success": self.status = "error" if linter.return_code != 0: self.return_code = linter.return_code if linter.number_fixed > 0: self.has_updated_sources = 1 # Sort linters before reports production self.linters = sorted(self.linters, key=lambda l: (l.descriptor_id, l.name)) # Check if a Mega-Linter flavor can be used for this repo, except if: # - FLAVOR_SUGGESTIONS: false is defined # - VALIDATE_ALL_CODE_BASE is false, or diff failed (we don't have all the files to calculate the suggestion) if ( self.validate_all_code_base is True and config.get("FLAVOR_SUGGESTIONS", "true") == "true" ): self.flavor_suggestions = flavor_factory.get_megalinter_flavor_suggestions( active_linters ) # Generate reports for reporter in self.reporters: reporter.produce_report() # Manage return code self.check_results()
def get_workspace(self): default_workspace = config.get("DEFAULT_WORKSPACE", "") github_workspace = config.get("GITHUB_WORKSPACE", "") # Github action run without override of DEFAULT_WORKSPACE and using /tmp/lint if ( default_workspace == "" and github_workspace != "" and os.path.isdir(github_workspace + "/tmp/lint") ): logging.debug( "Context: Github action run without override of DEFAULT_WORKSPACE and using /tmp/lint" ) return github_workspace + "/tmp/lint" # Docker run without override of DEFAULT_WORKSPACE elif default_workspace != "" and os.path.isdir( "/tmp/lint" + os.path.sep + default_workspace ): logging.debug("Context: Docker run without override of DEFAULT_WORKSPACE") return default_workspace + "/tmp/lint" + os.path.sep + default_workspace # Docker run with override of DEFAULT_WORKSPACE for test cases elif default_workspace != "" and os.path.isdir(default_workspace): logging.debug( "Context: Docker run with override of DEFAULT_WORKSPACE for test cases" ) return default_workspace # Docker run test classes without override of DEFAULT_WORKSPACE elif os.path.isdir("/tmp/lint"): logging.debug( "Context: Docker run test classes without override of DEFAULT_WORKSPACE" ) return "/tmp/lint" # Github action with override of DEFAULT_WORKSPACE elif ( default_workspace != "" and github_workspace != "" and os.path.isdir(github_workspace + os.path.sep + default_workspace) ): logging.debug("Context: Github action with override of DEFAULT_WORKSPACE") return github_workspace + os.path.sep + default_workspace # Github action without override of DEFAULT_WORKSPACE and NOT using /tmp/lint elif ( default_workspace == "" and github_workspace != "" and github_workspace != "/" and os.path.isdir(github_workspace) ): logging.debug( "Context: Github action without override of DEFAULT_WORKSPACE and NOT using /tmp/lint" ) return github_workspace # Unable to identify workspace else: raise FileNotFoundError( f"Unable to find a workspace to lint \n" f"DEFAULT_WORKSPACE: {default_workspace}\n" f"GITHUB_WORKSPACE: {github_workspace}" )
def __init__(self, params=None): # Activate console output by default self.is_active = True self.report_type = "simple" if config.get("OUTPUT_DETAIL", "") == "detailed": self.report_type = "detailed" if config.get("PRINT_ALL_FILES", "") == "false": self.print_all_files = False super().__init__(params)
def manage_default_linter_activation(self): # If at least one language/linter is activated with VALIDATE_XXX , all others are deactivated by default if len(self.enable_descriptors) > 0 or len(self.enable_linters) > 0: self.default_linter_activation = False # V3 legacy variables for env_var in config.get(): if env_var.startswith("VALIDATE_") and env_var != "VALIDATE_ALL_CODEBASE": if config.get(env_var) == "true": self.default_linter_activation = False
def manage_activation(self): # Super-Linter legacy variables output_format = config.get("OUTPUT_FORMAT", "") if output_format.startswith("text"): self.is_active = True # MegaLinter vars (true by default) elif config.get("TEXT_REPORTER", "true") != "true": self.is_active = False else: self.is_active = True
def manage_activation(self): # Super-Linter legacy variables output_format = config.get("OUTPUT_FORMAT", "") if output_format.startswith("tap"): self.is_active = True if config.get("OUTPUT_DETAIL", "") == "detailed": self.report_type = "detailed" # Mega-Linter vars (false by default) elif config.get("TEXT_REPORTER", "false") == "true": self.is_active = True else: self.is_active = False
def load_linters(self): # Linters init params linter_init_params = { "master": self, "linter_rules_path": self.linter_rules_path, "default_rules_location": self.default_rules_location, "default_linter_activation": self.default_linter_activation, "enable_descriptors": self.enable_descriptors, "enable_linters": self.enable_linters, "disable_descriptors": self.disable_descriptors, "disable_linters": self.disable_linters, "workspace": self.workspace, "github_workspace": self.github_workspace, "report_folder": self.report_folder, "apply_fixes": self.apply_fixes, "show_elapsed_time": self.show_elapsed_time, } # Build linters from descriptor files # if flavor selected and no flavor suggestion, ignore linters that are not in current flavor) if (self.megalinter_flavor != "all" and config.get("FLAVOR_SUGGESTIONS", "true") != "true"): all_linters = linter_factory.list_flavor_linters( linter_init_params, self.megalinter_flavor) else: all_linters = linter_factory.list_all_linters(linter_init_params) skipped_linters = [] # Remove inactive or disabled linters for linter in all_linters: linter.master = self if linter.is_active is False or linter.disabled is True: skipped_linters += [linter.name] if linter.disabled is True: logging.warning( f"{linter.name} has been temporary disabled in MegaLinter, please use a " "previous MegaLinter version or wait for the next one !" ) continue self.linters += [linter] # Display skipped linters in log show_skipped_linters = config.get("SHOW_SKIPPED_LINTERS", "true") == "true" if len(skipped_linters) > 0 and show_skipped_linters: skipped_linters.sort() logging.info("Skipped linters: " + ", ".join(skipped_linters)) # Sort linters by language and linter_name self.linters = sorted(self.linters, key=lambda l: (l.processing_order, l.descriptor_id))
def produce_report(self): if ( config.exists("GITHUB_REPOSITORY") and config.exists("GITHUB_SHA") and config.exists("GITHUB_TOKEN") and config.exists("GITHUB_RUN_ID") ): github_repo = config.get("GITHUB_REPOSITORY") sha = config.get("GITHUB_SHA") run_id = config.get("GITHUB_RUN_ID") success_msg = "No errors were found in the linting process" error_not_blocking = "Errors were detected but are considered not blocking" error_msg = "Errors were detected, please view logs" url = f"{self.github_api_url}/repos/{github_repo}/statuses/{sha}" headers = { "accept": "application/vnd.github.v3+json", "authorization": f"Bearer {config.get('GITHUB_TOKEN')}", "content-type": "application/json", } target_url = f"https://github.com/{github_repo}/actions/runs/{run_id}" description = ( success_msg if self.master.status == "success" and self.master.return_code == 0 else error_not_blocking if self.master.status == "error" and self.master.return_code == 0 else error_msg ) if self.master.show_elapsed_time is True: description += f" ({str(round(self.master.elapsed_time_s, 2))}s)" data = { "state": "success" if self.master.return_code == 0 else "error", "target_url": target_url, "description": description, "context": f"--> Lint: {self.master.descriptor_id} with {self.master.linter_name}", } response = requests.post(url, headers=headers, json=data) if 200 <= response.status_code < 299: logging.debug( f"Successfully posted Github Status for {self.master.descriptor_id} with {self.master.linter_name}" ) else: logging.error( f"Error posting Github Status for {self.master.descriptor_id}" f"with {self.master.linter_name}: {response.status_code}" ) logging.error(f"GitHub API response: {response.text}") else: logging.debug( f"Skipped post of Github Status for {self.master.descriptor_id} with {self.master.linter_name}" )
def load_linters(self): # Linters init params linter_init_params = { "linter_rules_path": self.linter_rules_path, "default_rules_location": self.default_rules_location, "default_linter_activation": self.default_linter_activation, "enable_descriptors": self.enable_descriptors, "enable_linters": self.enable_linters, "disable_descriptors": self.disable_descriptors, "disable_linters": self.disable_linters, "workspace": self.workspace, "github_workspace": self.github_workspace, "report_folder": self.report_folder, "apply_fixes": self.apply_fixes, "show_elapsed_time": self.show_elapsed_time, } # Build linters from descriptor files all_linters = linter_factory.list_all_linters(linter_init_params) skipped_linters = [] for linter in all_linters: if linter.is_active is False: skipped_linters += [linter.name] continue self.linters += [linter] # Display skipped linters in log show_skipped_linters = config.get("SHOW_SKIPPED_LINTERS", "true") == "true" if len(skipped_linters) > 0 and show_skipped_linters: skipped_linters.sort() logging.info("Skipped linters: " + ", ".join(skipped_linters)) # Sort linters by language and linter_name self.linters = sorted( self.linters, key=lambda l: (l.processing_order, l.descriptor_id) )
def __init__(self, params=None): # Activate console output by default self.is_active = True self.report_type = "simple" if config.get("OUTPUT_DETAIL", "") == "detailed": self.report_type = "detailed" super().__init__(params)
def produce_report(self): # Doc URL lang_lower = self.master.descriptor_id.lower() linter_name_lower = self.master.linter_name.lower().replace("-", "_") doc_name = f"{lang_lower}_{linter_name_lower}" doc_url = f"https://nvuillam.github.io/mega-linter/descriptors/{doc_name}/" # Finalize lines text_report_lines = [ f"Results of {self.master.linter_name} linter (version {self.master.get_linter_version()})", f"See documentation on {doc_url}", "-----------------------------------------------", "", ] text_report_lines += self.report_items text_report_lines += self.master.complete_text_reporter_report(self) text_report_sub_folder = config.get("TEXT_REPORTER_SUB_FOLDER", "linters_logs") text_file_name = ( f"{self.report_folder}{os.path.sep}" f"{text_report_sub_folder}{os.path.sep}" f"{self.master.status.upper()}-{self.master.name}.log") if not os.path.isdir(os.path.dirname(text_file_name)): os.makedirs(os.path.dirname(text_file_name), exist_ok=True) with open(text_file_name, "w", encoding="utf-8") as text_file: text_file_content = "\n".join(text_report_lines) + "\n" text_file.write(text_file_content) logging.debug(f"Generated {self.name} report: {text_file_name}")
def manage_docker_command(self, command): if self.cli_docker_image is None: return command docker_command = ["docker", "run"] if hasattr(self, "workspace"): volume_root = config.get("MEGALINTER_VOLUME_ROOT", "") if volume_root != "": workspace_value = (volume_root + "/" + self.workspace.replace("/tmp/lint", "")) else: workspace_value = self.workspace else: workspace_value = "/tmp/lint" docker_command += map( lambda arg, w=workspace_value: arg.replace("{{WORKSPACE}}", w), self.cli_docker_args, ) docker_command += [ f"{self.cli_docker_image}:{self.cli_docker_image_version}" ] if type(command) == str: command = " ".join(docker_command) + " " + command else: command = docker_command + command # ["ls", "-A", "/tmp/lint"] return command
def test_linter_failure(linter, test_self): if linter.disabled is True: raise unittest.SkipTest("Linter has been disabled") test_folder = linter.test_folder workspace = config.get("DEFAULT_WORKSPACE") + os.path.sep + test_folder if os.path.isdir(workspace + os.path.sep + "bad"): workspace = workspace + os.path.sep + "bad" workspace = manage_copy_sources(workspace) tmp_report_folder = tempfile.gettempdir() + os.path.sep + str(uuid.uuid4()) assert os.path.isdir(workspace), f"Test folder {workspace} is not existing" if os.path.isfile(workspace + os.path.sep + "no_test_failure"): raise unittest.SkipTest( f"Skip failure test for {linter}: no_test_failure found in test folder" ) linter_name = linter.linter_name env_vars_failure = { "DEFAULT_WORKSPACE": workspace, "FILTER_REGEX_INCLUDE": r"(bad)", "OUTPUT_FORMAT": "text", "OUTPUT_DETAIL": "detailed", "REPORT_OUTPUT_FOLDER": tmp_report_folder, "LOG_LEVEL": "DEBUG", "ENABLE_LINTERS": linter.name, } if linter.lint_all_other_linters_files is not False: env_vars_failure["ENABLE_LINTERS"] += ",JAVASCRIPT_ES" env_vars_failure.update(linter.test_variables) mega_linter, output = call_mega_linter(env_vars_failure) # Check linter run test_self.assertTrue( len(mega_linter.linters) > 0, "Linters have been created and run" ) # Check console output if linter.cli_lint_mode == "file": if len(linter.file_names_regex) > 0 and len(linter.file_extensions) == 0: test_self.assertRegex( output, rf"\[{linter_name}\] .*{linter.file_names_regex[0]}.* - ERROR" ) test_self.assertNotRegex( output, rf"\[{linter_name}\] .*{linter.file_names_regex[0]}.* - SUCCESS" ) else: test_self.assertRegex(output, rf"\[{linter_name}\] .*bad.* - ERROR") test_self.assertNotRegex(output, rf"\[{linter_name}\] .*bad.* - SUCCESS") else: test_self.assertRegex( output, rf"Linted \[{linter.descriptor_id}\] files with \[{linter_name}\]: Found", ) # Check text reporter output log report_file_name = f"ERROR-{linter.name}.log" text_report_file = ( f"{tmp_report_folder}{os.path.sep}linters_logs" f"{os.path.sep}{report_file_name}" ) test_self.assertTrue( os.path.isfile(text_report_file), f"Unable to find text report {text_report_file}", ) copy_logs_for_doc(text_report_file, test_folder, report_file_name)
def produce_report(self): if self.master.cli_lint_mode == "project": return tap_report_lines = [ "TAP version 13", f"1..{str(len(self.master.files))}" ] # Convert file results in TAP for index, file_result in enumerate(self.master.files_lint_results): file_nm = utils.normalize_log_string(file_result["file"]) tap_status = "ok" if file_result["status_code"] == 0 else "not ok" file_tap_lines = [f"{tap_status} {str(index + 1)} - {file_nm}"] if (self.report_type == "detailed" and file_result["stdout"] != "" and file_result["status_code"] != 0): std_out_tap = (file_result["stdout"].rstrip(f" {os.linesep}") + os.linesep) std_out_tap = "\\n".join(std_out_tap.splitlines()) std_out_tap = std_out_tap.replace(":", " ") detailed_lines = [ " ---", f" message: {std_out_tap}", " ..." ] file_tap_lines += detailed_lines tap_report_lines += file_tap_lines # Write TAP file tap_report_sub_folder = config.get("TAP_REPORTER_SUB_FOLDER", "tap") tap_file_name = (f"{self.report_folder}{os.path.sep}" f"{tap_report_sub_folder}{os.path.sep}" f"mega-linter-{self.master.name}.tap") if not os.path.isdir(os.path.dirname(tap_file_name)): os.makedirs(os.path.dirname(tap_file_name), exist_ok=True) with open(tap_file_name, "w", encoding="utf-8") as tap_file: tap_file_content = "\n".join(tap_report_lines) + "\n" tap_file.write(tap_file_content) logging.info( f"[Tap Reporter] Generated {self.name} report: {tap_file_name}" )
def check_active_linters_match_flavor(active_linters): flavor = get_image_flavor() if flavor == "all": logging.debug('MegaLinter flavor is "all", no need to check match with linters') return True all_flavors = get_all_flavors() flavor_linters = all_flavors[flavor]["linters"] missing_linters = [] for active_linter in active_linters: if active_linter.name not in flavor_linters: missing_linters += [active_linter.name] active_linter.is_active = False if len(missing_linters) > 0: missing_linters_str = ",".join(missing_linters) logging.warning( f"MegaLinter flavor [{flavor}] does not contain linters {missing_linters_str}.\n" "As they are not available in this docker image, they will not be processed\n" "To solve this problem, please either: \n" f"- use default flavor {ML_REPO}\n" "- add ignored linters in DISABLE or DISABLE_LINTERS variables in your .mega-linter.yml config file " "located in your root directory\n" "- ignore this message by setting config variable FLAVOR_SUGGESTIONS to false" ) if config.get("FAIL_IF_MISSING_LINTER_IN_FLAVOR", "") == "true": logging.error( 'Missing linter and FAIL_IF_MISSING_LINTER_IN_FLAVOR has been set to "true": Stop run' ) sys.exit(84) return False return True
def check_results(self): print(f"::set-output name=has_updated_sources::{str(self.has_updated_sources)}") if self.status == "success": logging.info("✅ Successfully linted all files without errors") config.delete() elif self.status == "warning": logging.warning("◬ Successfully linted all files, but with ignored errors") config.delete() else: logging.error("❌ Error(s) have been found during linting") logging.warning( "To disable linters or customize their checks, you can use a .mega-linter.yml file " "at the root of your repository" ) logging.warning( "More info at https://nvuillam.github.io/mega-linter/configuration/" ) if self.cli is True: if config.get("DISABLE_ERRORS", "false") == "true": config.delete() sys.exit(0) else: config.delete() sys.exit(self.return_code) config.delete()
def load_config_vars(self): # Linter rules root path if config.exists("LINTER_RULES_PATH"): self.linter_rules_path = (self.github_workspace + os.path.sep + config.get("LINTER_RULES_PATH")) # Filtering regex (inclusion) if config.exists("FILTER_REGEX_INCLUDE"): self.filter_regex_include = config.get("FILTER_REGEX_INCLUDE") # Filtering regex (exclusion) if config.exists("FILTER_REGEX_EXCLUDE"): self.filter_regex_exclude = config.get("FILTER_REGEX_EXCLUDE") # Disable all fields validation if VALIDATE_ALL_CODEBASE is 'false' if (config.exists("VALIDATE_ALL_CODEBASE") and config.get("VALIDATE_ALL_CODEBASE") == "false"): self.validate_all_code_base = False