def __init__(self, params=None): if params is None: params = {} self.workspace = self.get_workspace() config.init_config(self.workspace) # Initialize runtime config self.github_workspace = config.get("GITHUB_WORKSPACE", self.workspace) self.report_folder = config.get( "REPORT_OUTPUT_FOLDER", config.get("OUTPUT_FOLDER", self.github_workspace + os.path.sep + "report"), ) self.initialize_logger() self.display_header() # Mega-Linter default rules location self.default_rules_location = ( "/action/lib/.automation" if os.path.isdir("/action/lib/.automation") else os.path.relpath( os.path.relpath( os.path.dirname(os.path.abspath(__file__)) + "/../TEMPLATES" ) ) ) # User-defined rules location self.linter_rules_path = self.github_workspace + os.path.sep + ".github/linters" self.validate_all_code_base = True self.filter_regex_include = None self.filter_regex_exclude = None self.cli = params["cli"] if "cli" in params else False self.default_linter_activation = True # Get enable / disable vars self.enable_descriptors = config.get_list("ENABLE", []) self.enable_linters = config.get_list("ENABLE_LINTERS", []) self.disable_descriptors = config.get_list("DISABLE", []) self.disable_linters = config.get_list("DISABLE_LINTERS", []) self.manage_default_linter_activation() self.apply_fixes = config.get_list("APPLY_FIXES", "none") self.show_elapsed_time = ( config.get("SHOW_ELAPSED_TIME", "false") == "true" or config.get("LOG_LEVEL", "DEBUG") == "DEBUG" ) # Load optional configuration self.load_config_vars() # Runtime properties self.reporters = [] self.linters = [] self.file_extensions = [] self.file_names_regex = [] self.status = "success" self.return_code = 0 self.has_updated_sources = 0 self.flavor_suggestions = None # Initialize plugins plugin_factory.initialize_plugins() # Initialize linters and gather criteria to browse files self.load_linters() self.compute_file_extensions() # Load Mega-Linter reporters self.load_reporters()
def get_excluded_directories(): default_excluded_dirs = [ "__pycache__", ".git", ".jekyll-cache", ".pytest_cache", ".rbenv", ".venv", ".terragrunt-cache", "node_modules", "report", ] excluded_dirs = config.get_list("EXCLUDED_DIRECTORIES", default_excluded_dirs) excluded_dirs += config.get_list("ADDITIONAL_EXCLUDED_DIRECTORIES", []) return set(excluded_dirs)
def run_pre_post_commands(key, log_key, mega_linter): pre_commands = config.get_list(key, None) pre_commands_results = [] if pre_commands is None: logging.debug(f"{log_key} No commands declared in user configuration") return pre_commands_results for command_info in pre_commands: pre_command_result = run_command(command_info, log_key, mega_linter) pre_commands_results += [pre_command_result] return pre_commands_results
def list_plugins(): plugins = config.get_list("PLUGINS", []) return plugins
def __init__(self, params=None, linter_config=None): self.linter_version_cache = None self.linter_help_cache = None self.processing_order = 0 # Definition fields & default values: can be overridden at custom linter class level or in YML descriptors # Ex: JAVASCRIPT self.descriptor_id = ( "Field 'descriptor_id' must be overridden at custom linter class level" ) # If you have several linters for the same language,override with a different name.Ex: JAVASCRIPT_ES self.name = None self.is_formatter = False self.linter_name = "Field 'linter_name' must be overridden at custom linter class level" # Ex: eslint # ex: https://eslint.org/ self.linter_url = ( "Field 'linter_url' must be overridden at custom linter class level" ) self.test_folder = None # Override only if different from language.lowercase() self.activation_rules = [] self.test_variables = {} # Array of strings defining file extensions. Ex: ['.js','.cjs', ''] self.file_extensions = [] # Array of file name regular expressions. Ex: [Dockerfile(-.+)?] self.file_names_regex = [] # Default name of the configuration file to use with the linter. Ex: '.eslintrc.js' self.config_file_name = None self.files_sub_directory = None self.file_contains_regex = [] self.file_names_not_ends_with = [] self.active_only_if_file_found = [] self.lint_all_files = False self.lint_all_other_linters_files = False self.cli_lint_mode = "file" self.cli_docker_image = None self.cli_docker_image_version = "latest" self.cli_docker_args = [] self.cli_executable = None self.cli_executable_fix = None self.cli_executable_version = None self.cli_executable_help = None # Default arg name for configurations to use in linter CLI call self.cli_config_arg_name = "-c" self.cli_config_extra_args = ( []) # Extra arguments to send to cli when a config file is used self.no_config_if_fix = False self.cli_lint_extra_args = [ ] # Extra arguments to send to cli everytime self.cli_lint_fix_arg_name = None # Name of the cli argument to send in case of APPLY_FIXES required by user self.cli_lint_fix_remove_args = ( []) # Arguments to remove in case fix argument is sent self.cli_lint_user_args = ( [] ) # Arguments from config, defined in <LINTER_KEY>_ARGUMENTS variable # Extra arguments to send to cli everytime, just before file argument self.cli_lint_extra_args_after = [] self.cli_lint_errors_count = None self.cli_lint_errors_regex = None # Default arg name for configurations to use in linter version call self.cli_version_arg_name = "--version" self.cli_version_extra_args = [ ] # Extra arguments to send to cli everytime self.cli_help_arg_name = "-h" self.cli_help_extra_args = [ ] # Extra arguments to send to cli everytime self.cli_help_extra_commands = [] # If linter --help does not return 0 when it is in success, override. ex: 1 self.help_command_return_code = 0 self.version_extract_regex = r"\d+(\.\d+)+" # If linter --version does not return 0 when it is in success, override. ex: 1 self.version_command_return_code = 0 self.report_folder = "" self.reporters = [] # Initialize with configuration data for key, value in linter_config.items(): self.__setattr__(key, value) # Initialize parameters if params is None: params = { "default_linter_activation": False, "enable_descriptors": [], "enable_linters": [], "disable_descriptors": [], "disable_linters": [], "post_linter_status": True, } self.is_active = params["default_linter_activation"] self.disable_errors_if_less_than = None self.disable_errors = True if self.is_formatter is True else False if self.name is None: self.name = (self.descriptor_id + "_" + self.linter_name.upper().replace("-", "_")) if self.cli_executable is None: self.cli_executable = self.linter_name if self.cli_executable_fix is None: self.cli_executable_fix = self.cli_executable if self.cli_executable_version is None: self.cli_executable_version = self.cli_executable if self.cli_executable_help is None: self.cli_executable_help = self.cli_executable if self.test_folder is None: self.test_folder = self.descriptor_id.lower() # Apply linter customization via config settings: self.file_extensions = config.get_list(self.name + "_FILE_EXTENSIONS", self.file_extensions) self.file_names_regex = config.get_list( self.name + "_FILE_NAMES_REGEX", self.file_names_regex) self.manage_activation(params) if self.is_active is True: self.show_elapsed_time = params.get("show_elapsed_time", False) # Manage apply fixes flag on linter param_apply_fixes = params.get("apply_fixes", "none") if self.cli_lint_fix_arg_name is None: self.apply_fixes = False elif param_apply_fixes == "all" or (isinstance( param_apply_fixes, bool) and param_apply_fixes is True): self.apply_fixes = True elif (param_apply_fixes != "none" and isinstance(param_apply_fixes, str) and self.name in param_apply_fixes.split(",")): self.apply_fixes = True elif (param_apply_fixes != "none" and isinstance(param_apply_fixes, list) and (self.name in param_apply_fixes or param_apply_fixes[0] == "all")): self.apply_fixes = True else: self.apply_fixes = False # Config items self.linter_rules_path = (params["linter_rules_path"] if "linter_rules_path" in params else ".") self.default_rules_location = (params["default_rules_location"] if "default_rules_location" in params else ".") self.workspace = params[ "workspace"] if "workspace" in params else "." self.github_workspace = (params["github_workspace"] if "github_workspace" in params else ".") self.config_file = None self.config_file_label = None self.config_file_error = None self.filter_regex_include = None self.filter_regex_exclude = None self.post_linter_status = (params["post_linter_status"] if "post_linter_status" in params else False) self.github_api_url = (params["github_api_url"] if "github_api_url" in params else None) self.report_types = (params["report_types"] if "report_types" in params else []) self.report_folder = (params["report_folder"] if "report_folder" in params else "") self.load_config_vars() # Manage sub-directory filter if defined if self.files_sub_directory is not None: self.files_sub_directory = config.get( f"{self.descriptor_id}_DIRECTORY", self.files_sub_directory) if not os.path.isdir(self.workspace + os.path.sep + self.files_sub_directory): self.is_active = False logging.debug( f"[Activation] {self.name} has been set inactive, as subdirectory has not been found:" f" {self.files_sub_directory}") # Some linters require a file to be existing, else they are deactivated ( ex: .editorconfig ) if len(self.active_only_if_file_found) > 0: is_found = False for file_to_check in self.active_only_if_file_found: if os.path.isfile(f"{self.workspace}/{file_to_check}"): is_found = True break if is_found is False: self.is_active = False logging.info( f"[Activation] {self.name} has been set inactive, as none of these files has been found:" f" {str(self.active_only_if_file_found)}") # Load Mega-Linter reporters self.load_reporters() # Runtime items self.files = [] self.try_fix = False self.status = "success" self.stdout = None self.return_code = 0 self.number_errors = 0 self.total_number_errors = 0 self.number_fixed = 0 self.files_lint_results = [] self.start_perf = None self.elapsed_time_s = None self.remote_config_file_to_delete = None
def produce_report(self): # Skip report if no errors has been found if (self.master.status == "success" and config.get( "EMAIL_REPORTER_SEND_SUCCESS", "false") == "true" and self.master.has_updated_sources is False): logging.info( "[Email Reporter] No mail sent, " "as the MegaLinter status is success and there are no updated source" ) return # get server and email config values smtp_host = config.get("EMAIL_REPORTER_SMTP_HOST", "smtp.gmail.com") smtp_port = config.get("EMAIL_REPORTER_SMTP_PORT", 465) recipients = config.get_list("EMAIL_REPORTER_EMAIL", []) sender = config.get("EMAIL_REPORTER_SENDER", "*****@*****.**") smtp_username = config.get("EMAIL_REPORTER_SMTP_USERNAME", sender) smtp_password = config.get("EMAIL_REPORTER_SMTP_PASSWORD", "") # Skip report if SMTP password is not set if smtp_password == "": logging.info( "[Email Reporter] No mail sent, as EMAIL_REPORTER_SMTP_PASSWORD configuration variable is missing" ) return # Create temporary zip file with content of report folder zf = tempfile.TemporaryFile(prefix="mail", suffix=".zip") zip_file = zipfile.ZipFile(zf, "w") for root, dirs, files in os.walk(self.report_folder): for file in files: file_abs_path = os.path.join(root, file) if "copy-paste/html" not in file_abs_path: zip_file.write( file_abs_path, arcname=file_abs_path.replace(self.report_folder, ""), ) zip_file.close() zf.seek(0) # Create the message the_msg = MIMEMultipart() the_msg["Subject"] = "MegaLinter report" the_msg["To"] = ", ".join(recipients) the_msg["From"] = "*****@*****.**" the_msg.preamble = "I am not using a MIME-aware mail reader.\n" msg = MIMEBase("application", "zip") msg.set_payload(zf.read()) encoders.encode_base64(msg) msg.add_header("Content-Disposition", "attachment", filename="mega-linter-reports" + ".zip") the_msg.attach(msg) the_msg = the_msg.as_string() # send the message try: server = smtplib.SMTP_SSL( smtp_host, smtp_port, ) server.ehlo() server.login( smtp_username, smtp_password, ) server.sendmail(sender, recipients, the_msg) server.quit() except smtplib.SMTPAuthenticationError as e: logging.warning( "[Email Reporter] Unable to authenticate to SMTP server: \n" + str(e) + "\n - smtp server: " + smtp_host + ":" + str(smtp_port) + "\n - smtp username: "******"\n - smtp password:"******"SET" if smtp_password != "" else "NOT SET")) return except Exception as e: logging.warning("[Email Reporter] Unable to send e-mail: \n" + str(e.__class__) + " - " + str(e) + "\n - smtp server: " + smtp_host + ":" + str(smtp_port) + "\n - smtp username: "******"\n - smtp password:"******"SET" if smtp_password != "" else "NOT SET")) return logging.info("[Email Reporter] Sent mail to " + ", ".join(recipients))
def run_pre_post_commands(key, log_key, mega_linter): pre_or_post_commands = config.get_list(key, None) return run_commands(pre_or_post_commands, log_key, mega_linter)