def produce_report(self):
result_obj = copy.deepcopy(self.master)
# Remove output data if result is simple (except if we are in debug mode)
if self.report_type == "simple" and config.get("LOG_LEVEL",
"") != "DEBUG":
result_obj = self.filter_fields(result_obj, self.megalinter_fields)
result_obj.linters = filter(lambda x: x.is_active is True,
result_obj.linters)
result_obj.linters = list(
map(
lambda x: self.filter_fields(x, self.linter_fields),
result_obj.linters,
))
result_obj.reporters = list(
map(lambda x: self.filter_fields(x, []), result_obj.reporters))
for reporter in result_obj.reporters:
setattr(reporter, "name", reporter.name)
# Generate JSON from object using jsonpickle
result_json = jsonpickle.encode(result_obj,
unpicklable=False,
max_depth=self.max_depth,
indent=4)
# unserialize + serialize to sort object keys
result_json_obj = json.loads(result_json)
result_json = json.dumps(result_json_obj, sort_keys=True, indent=4)
# Write output file
json_file_name = f"{self.report_folder}{os.path.sep}" + config.get(
"JSON_REPORTER_FILE_NAME", "mega-linter-report.json")
with open(json_file_name, "w", encoding="utf-8") as json_file:
json_file.write(result_json)
logging.info(
f"[JSON Reporter] Generated {self.name} report: {json_file_name}"
)
def initialize_logger(self):
logging_level_key = config.get("LOG_LEVEL", "INFO").upper()
logging_level_list = {
"INFO": logging.INFO,
"DEBUG": logging.DEBUG,
"WARNING": logging.WARNING,
"ERROR": logging.ERROR,
# Previous values for v3 ascending compatibility
"TRACE": logging.WARNING,
"VERBOSE": logging.INFO,
}
logging_level = (
logging_level_list[logging_level_key]
if logging_level_key in logging_level_list
else logging.INFO
)
log_file = (
self.report_folder + os.path.sep + config.get("LOG_FILE", "mega-linter.log")
)
if not os.path.isdir(os.path.dirname(log_file)):
os.makedirs(os.path.dirname(log_file), exist_ok=True)
logging.basicConfig(
force=True,
level=logging_level,
format="%(message)s",
handlers=[
logging.FileHandler(log_file, "w", "utf-8"),
logging.StreamHandler(sys.stdout),
],
)
def test_remote_config_extends_success_2(self):
remote_config = self.test_folder + "base2.mega-linter.yml"
os.environ["MEGALINTER_CONFIG"] = remote_config
config.init_config()
self.assertEqual("(base)", config.get("FILTER_REGEX_INCLUDE"))
self.assertEqual("(extension2)", config.get("FILTER_REGEX_EXCLUDE"))
self.assertEqual("true", config.get("SHOW_ELAPSED_TIME"))
def linter_test_setup(params=None):
config.init_config(None)
if params is None:
params = {}
# Root to lint
sub_lint_root = (params["sub_lint_root"] if "sub_lint_root" in params else
f"{os.path.sep}.automation{os.path.sep}test")
# Ignore report folder
config.set_value("FILTER_REGEX_EXCLUDE", "\\/(report)\\/")
# TAP Output deactivated by default
config.set_value("OUTPUT_FORMAT", "text")
config.set_value("OUTPUT_DETAIL", "detailed")
# Root path of default rules
root_dir = ("/tmp/lint" if os.path.isdir("/tmp/lint") else os.path.relpath(
os.path.relpath(os.path.dirname(os.path.abspath(__file__))) +
"/../../../.."))
config.set_value("VALIDATE_ALL_CODEBASE", "true")
# Root path of files to lint
config.set_value(
"DEFAULT_WORKSPACE",
(config.get("DEFAULT_WORKSPACE") +
sub_lint_root if config.exists("DEFAULT_WORKSPACE")
and os.path.isdir(config.get("DEFAULT_WORKSPACE") + sub_lint_root)
else root_dir + sub_lint_root),
)
assert os.path.isdir(
config.get("DEFAULT_WORKSPACE")), ("DEFAULT_WORKSPACE " +
config.get("DEFAULT_WORKSPACE") +
" is not a valid folder")
def list_files_git_diff(self):
# List all updated files from git
logging.info(
"Listing updated files in ["
+ self.github_workspace
+ "] using git diff, then filter with:"
)
repo = git.Repo(os.path.realpath(self.github_workspace))
default_branch = config.get("DEFAULT_BRANCH", "master")
current_branch = config.get("GITHUB_SHA", "")
if current_branch == "":
current_branch = repo.active_branch.commit.hexsha
try:
repo.git.pull()
except git.GitCommandError:
try:
repo.git.checkout(current_branch)
repo.git.pull()
except git.GitCommandError:
logging.info(f"Warning: Unable to pull current branch {current_branch}")
repo.git.checkout(default_branch)
diff = repo.git.diff(f"{default_branch}...{current_branch}", name_only=True)
repo.git.checkout(current_branch)
logging.info(f"Git diff :\n{diff}")
all_files = list()
for diff_line in diff.splitlines():
if os.path.isfile(self.workspace + os.path.sep + diff_line):
all_files += [self.workspace + os.path.sep + diff_line]
return all_files
def display_header():
# Header prints
logging.info(utils.format_hyphens(""))
logging.info(utils.format_hyphens("Mega-Linter"))
logging.info(utils.format_hyphens(""))
logging.info(
" - Image Creation Date: " + config.get("BUILD_DATE", "No docker image")
)
logging.info(
" - Image Revision: " + config.get("BUILD_REVISION", "No docker image")
)
logging.info(
" - Image Version: " + config.get("BUILD_VERSION", "No docker image")
)
logging.info(utils.format_hyphens(""))
logging.info("The Mega-Linter documentation can be found at:")
logging.info(" - https://nvuillam.github.io/mega-linter")
logging.info(utils.format_hyphens(""))
logging.info("GITHUB_REPOSITORY: " + os.environ.get("GITHUB_REPOSITORY", ""))
logging.info("GITHUB_SHA: " + os.environ.get("GITHUB_SHA", ""))
logging.info("GITHUB_TOKEN: " + os.environ.get("GITHUB_TOKEN", ""))
logging.info("GITHUB_RUN_ID: " + os.environ.get("GITHUB_RUN_ID", ""))
logging.info("PAT: " + "set" if os.environ.get("PAT", "") != "" else "")
# Display config variables for debug mode
for name, value in sorted(config.get_config().items()):
logging.debug("" + name + "=" + str(value))
logging.debug(utils.format_hyphens(""))
logging.info("")
def __init__(self, params=None):
if params is None:
params = {}
self.workspace = self.get_workspace()
config.init_config(self.workspace) # Initialize runtime config
self.github_workspace = config.get("GITHUB_WORKSPACE", self.workspace)
self.report_folder = config.get(
"REPORT_OUTPUT_FOLDER",
config.get("OUTPUT_FOLDER", self.github_workspace + os.path.sep + "report"),
)
self.initialize_logger()
self.display_header()
# Mega-Linter default rules location
self.default_rules_location = (
"/action/lib/.automation"
if os.path.isdir("/action/lib/.automation")
else os.path.relpath(
os.path.relpath(
os.path.dirname(os.path.abspath(__file__)) + "/../TEMPLATES"
)
)
)
# User-defined rules location
self.linter_rules_path = self.github_workspace + os.path.sep + ".github/linters"
self.validate_all_code_base = True
self.filter_regex_include = None
self.filter_regex_exclude = None
self.cli = params["cli"] if "cli" in params else False
self.default_linter_activation = True
# Get enable / disable vars
self.enable_descriptors = config.get_list("ENABLE", [])
self.enable_linters = config.get_list("ENABLE_LINTERS", [])
self.disable_descriptors = config.get_list("DISABLE", [])
self.disable_linters = config.get_list("DISABLE_LINTERS", [])
self.manage_default_linter_activation()
self.apply_fixes = config.get_list("APPLY_FIXES", "none")
self.show_elapsed_time = (
config.get("SHOW_ELAPSED_TIME", "false") == "true"
or config.get("LOG_LEVEL", "DEBUG") == "DEBUG"
)
# Load optional configuration
self.load_config_vars()
# Runtime properties
self.reporters = []
self.linters = []
self.file_extensions = []
self.file_names_regex = []
self.status = "success"
self.return_code = 0
self.has_updated_sources = 0
self.flavor_suggestions = None
# Initialize plugins
plugin_factory.initialize_plugins()
# Initialize linters and gather criteria to browse files
self.load_linters()
self.compute_file_extensions()
# Load Mega-Linter reporters
self.load_reporters()
def manage_activation(self, params):
# Default value is false in case ENABLE variables are used
if len(params["enable_descriptors"]) > 0 or len(
params["enable_linters"]) > 0:
self.is_active = False
# Activate or not the linter
if self.name in params["enable_linters"]:
self.is_active = True
elif self.name in params["disable_linters"]:
self.is_active = False
elif (self.descriptor_id in params["disable_descriptors"]
or self.name in params["disable_linters"]):
self.is_active = False
elif self.descriptor_id in params["enable_descriptors"]:
self.is_active = True
elif (config.exists("VALIDATE_" + self.name)
and config.get("VALIDATE_" + self.name) == "false"):
self.is_active = False
elif (config.exists("VALIDATE_" + self.descriptor_id)
and config.get("VALIDATE_" + self.descriptor_id) == "false"):
self.is_active = False
elif (config.exists("VALIDATE_" + self.name)
and config.get("VALIDATE_" + self.name) == "true"):
self.is_active = True
elif (config.exists("VALIDATE_" + self.descriptor_id)
and config.get("VALIDATE_" + self.descriptor_id) == "true"):
self.is_active = True
# check activation rules
if self.is_active is True and len(self.activation_rules) > 0:
self.is_active = utils.check_activation_rules(
self.activation_rules, self)
def manage_activation(self):
if config.get("GITHUB_COMMENT_REPORTER", "true") != "true":
self.is_active = False
elif (
config.get("POST_GITHUB_COMMENT", "true") == "true"
): # Legacy - true by default
self.is_active = True
def manage_activation(self, params):
if self.name in params["enable_linters"]:
self.is_active = True
elif self.name in params["disable_linters"]:
self.is_active = False
elif (
self.descriptor_id in params["disable_descriptors"]
or self.name in params["disable_linters"]
):
self.is_active = False
elif self.descriptor_id in params["enable_descriptors"]:
self.is_active = True
elif (
config.exists("VALIDATE_" + self.name)
and config.get("VALIDATE_" + self.name) == "false"
):
self.is_active = False
elif (
config.exists("VALIDATE_" + self.descriptor_id)
and config.get("VALIDATE_" + self.descriptor_id) == "false"
):
self.is_active = False
elif (
config.exists("VALIDATE_" + self.name)
and config.get("VALIDATE_" + self.name) == "true"
):
self.is_active = True
elif (
config.exists("VALIDATE_" + self.descriptor_id)
and config.get("VALIDATE_" + self.descriptor_id) == "true"
):
self.is_active = True
def list_files_git_diff(self):
# List all updated files from git
logging.info("Listing updated files in [" + self.github_workspace +
"] using git diff.")
repo = git.Repo(os.path.realpath(self.github_workspace))
# Add auth header if necessary
if config.get("GIT_AUTHORIZATION_BEARER", "") != "":
auth_bearer = "Authorization: Bearer " + config.get(
"GIT_AUTHORIZATION_BEARER")
repo.config_writer().set_value("http", "extraheader",
auth_bearer).release()
self.has_git_extraheader = True
# Fetch base branch content
default_branch = config.get("DEFAULT_BRANCH", "HEAD")
default_branch_remote = f"origin/{default_branch}"
if default_branch_remote not in [ref.name for ref in repo.refs]:
remote_ref = ("HEAD" if default_branch == "HEAD" else
f"refs/heads/{default_branch}")
local_ref = f"refs/remotes/{default_branch_remote}"
# Try to fetch default_branch from origin, because it isn't cached locally.
repo.git.fetch("origin", f"{remote_ref}:{local_ref}")
# Make git diff to list files
diff = repo.git.diff(default_branch_remote, name_only=True)
logging.info(f"Modified files:\n{diff}")
all_files = list()
for diff_line in diff.splitlines():
if os.path.isfile(self.workspace + os.path.sep + diff_line):
all_files += [self.workspace + os.path.sep + diff_line]
return all_files
def manage_activation(self):
if config.get("EMAIL_REPORTER", "true") != "true":
self.is_active = False
elif config.get("EMAIL_REPORTER_EMAIL", "none") == "none":
logging.info(
"To receive reports as email, please set variable EMAIL_REPORTER_EMAIL"
)
self.is_active = False
def linter_test_setup(params=None):
for key in [
"MEGALINTER_CONFIG",
"EXTENDS",
"FILTER_REGEX_INCLUDE",
"FILTER_REGEX_EXCLUDE",
"SHOW_ELAPSED_TIME",
]:
if key in os.environ:
del os.environ[key]
config.delete()
if params is None:
params = {}
# Root to lint
sub_lint_root = (
params["sub_lint_root"]
if "sub_lint_root" in params
else f"{os.path.sep}.automation{os.path.sep}test"
)
# Root path of default rules
root_dir = (
"/tmp/lint"
if os.path.isdir("/tmp/lint")
else os.path.relpath(
os.path.relpath(os.path.dirname(os.path.abspath(__file__))) + "/../../../.."
)
)
workspace = None
config_file_path = root_dir + sub_lint_root + os.path.sep + ".mega-linter.yml"
if os.path.isfile(config_file_path):
workspace = root_dir + sub_lint_root
elif params.get("required_config_file", False) is True:
raise Exception(
f"[test] There should be a .mega-linter.yml file in test folder {config_file_path}"
)
config.init_config(workspace)
# Ignore report folder
config.set_value("FILTER_REGEX_EXCLUDE", r"\/report\/")
# TAP Output deactivated by default
config.set_value("OUTPUT_FORMAT", "text")
config.set_value("OUTPUT_DETAIL", "detailed")
config.set_value("PLUGINS", "")
config.set_value("VALIDATE_ALL_CODEBASE", "true")
# Root path of files to lint
config.set_value(
"DEFAULT_WORKSPACE",
(
config.get("DEFAULT_WORKSPACE") + sub_lint_root
if config.exists("DEFAULT_WORKSPACE")
and os.path.isdir(config.get("DEFAULT_WORKSPACE") + sub_lint_root)
else root_dir + sub_lint_root
),
)
assert os.path.isdir(config.get("DEFAULT_WORKSPACE")), (
"DEFAULT_WORKSPACE "
+ config.get("DEFAULT_WORKSPACE")
+ " is not a valid folder"
)
def run(self):
# Collect files for each identified linter
self.collect_files()
# Process linters serial or parallel according to configuration
active_linters = []
linters_do_fixes = False
for linter in self.linters:
if linter.is_active is True:
active_linters += [linter]
if linter.apply_fixes is True:
linters_do_fixes = True
# Initialize reports
for reporter in self.reporters:
reporter.initialize()
# Display warning if selected flavors does not match all linters
if flavor_factory.check_active_linters_match_flavor(active_linters) is False:
active_linters = [
linter for linter in active_linters if linter.is_active is True
]
if config.get("PARALLEL", "true") == "true" and len(active_linters) > 1:
self.process_linters_parallel(active_linters, linters_do_fixes)
else:
self.process_linters_serial(active_linters, linters_do_fixes)
# Update main Mega-Linter status according to results of linters run
for linter in self.linters:
if linter.status != "success":
self.status = "error"
if linter.return_code != 0:
self.return_code = linter.return_code
if linter.number_fixed > 0:
self.has_updated_sources = 1
# Sort linters before reports production
self.linters = sorted(self.linters, key=lambda l: (l.descriptor_id, l.name))
# Check if a Mega-Linter flavor can be used for this repo, except if:
# - FLAVOR_SUGGESTIONS: false is defined
# - VALIDATE_ALL_CODE_BASE is false, or diff failed (we don't have all the files to calculate the suggestion)
if (
self.validate_all_code_base is True
and config.get("FLAVOR_SUGGESTIONS", "true") == "true"
):
self.flavor_suggestions = flavor_factory.get_megalinter_flavor_suggestions(
active_linters
)
# Generate reports
for reporter in self.reporters:
reporter.produce_report()
# Manage return code
self.check_results()
def get_workspace(self):
default_workspace = config.get("DEFAULT_WORKSPACE", "")
github_workspace = config.get("GITHUB_WORKSPACE", "")
# Github action run without override of DEFAULT_WORKSPACE and using /tmp/lint
if (
default_workspace == ""
and github_workspace != ""
and os.path.isdir(github_workspace + "/tmp/lint")
):
logging.debug(
"Context: Github action run without override of DEFAULT_WORKSPACE and using /tmp/lint"
)
return github_workspace + "/tmp/lint"
# Docker run without override of DEFAULT_WORKSPACE
elif default_workspace != "" and os.path.isdir(
"/tmp/lint" + os.path.sep + default_workspace
):
logging.debug("Context: Docker run without override of DEFAULT_WORKSPACE")
return default_workspace + "/tmp/lint" + os.path.sep + default_workspace
# Docker run with override of DEFAULT_WORKSPACE for test cases
elif default_workspace != "" and os.path.isdir(default_workspace):
logging.debug(
"Context: Docker run with override of DEFAULT_WORKSPACE for test cases"
)
return default_workspace
# Docker run test classes without override of DEFAULT_WORKSPACE
elif os.path.isdir("/tmp/lint"):
logging.debug(
"Context: Docker run test classes without override of DEFAULT_WORKSPACE"
)
return "/tmp/lint"
# Github action with override of DEFAULT_WORKSPACE
elif (
default_workspace != ""
and github_workspace != ""
and os.path.isdir(github_workspace + os.path.sep + default_workspace)
):
logging.debug("Context: Github action with override of DEFAULT_WORKSPACE")
return github_workspace + os.path.sep + default_workspace
# Github action without override of DEFAULT_WORKSPACE and NOT using /tmp/lint
elif (
default_workspace == ""
and github_workspace != ""
and github_workspace != "/"
and os.path.isdir(github_workspace)
):
logging.debug(
"Context: Github action without override of DEFAULT_WORKSPACE and NOT using /tmp/lint"
)
return github_workspace
# Unable to identify workspace
else:
raise FileNotFoundError(
f"Unable to find a workspace to lint \n"
f"DEFAULT_WORKSPACE: {default_workspace}\n"
f"GITHUB_WORKSPACE: {github_workspace}"
)
def __init__(self, params=None):
# Activate console output by default
self.is_active = True
self.report_type = "simple"
if config.get("OUTPUT_DETAIL", "") == "detailed":
self.report_type = "detailed"
if config.get("PRINT_ALL_FILES", "") == "false":
self.print_all_files = False
super().__init__(params)
def manage_default_linter_activation(self):
# If at least one language/linter is activated with VALIDATE_XXX , all others are deactivated by default
if len(self.enable_descriptors) > 0 or len(self.enable_linters) > 0:
self.default_linter_activation = False
# V3 legacy variables
for env_var in config.get():
if env_var.startswith("VALIDATE_") and env_var != "VALIDATE_ALL_CODEBASE":
if config.get(env_var) == "true":
self.default_linter_activation = False
def manage_activation(self):
# Super-Linter legacy variables
output_format = config.get("OUTPUT_FORMAT", "")
if output_format.startswith("text"):
self.is_active = True
# MegaLinter vars (true by default)
elif config.get("TEXT_REPORTER", "true") != "true":
self.is_active = False
else:
self.is_active = True
def manage_activation(self):
# Super-Linter legacy variables
output_format = config.get("OUTPUT_FORMAT", "")
if output_format.startswith("tap"):
self.is_active = True
if config.get("OUTPUT_DETAIL", "") == "detailed":
self.report_type = "detailed"
# Mega-Linter vars (false by default)
elif config.get("TEXT_REPORTER", "false") == "true":
self.is_active = True
else:
self.is_active = False
def load_linters(self):
# Linters init params
linter_init_params = {
"master": self,
"linter_rules_path": self.linter_rules_path,
"default_rules_location": self.default_rules_location,
"default_linter_activation": self.default_linter_activation,
"enable_descriptors": self.enable_descriptors,
"enable_linters": self.enable_linters,
"disable_descriptors": self.disable_descriptors,
"disable_linters": self.disable_linters,
"workspace": self.workspace,
"github_workspace": self.github_workspace,
"report_folder": self.report_folder,
"apply_fixes": self.apply_fixes,
"show_elapsed_time": self.show_elapsed_time,
}
# Build linters from descriptor files
# if flavor selected and no flavor suggestion, ignore linters that are not in current flavor)
if (self.megalinter_flavor != "all"
and config.get("FLAVOR_SUGGESTIONS", "true") != "true"):
all_linters = linter_factory.list_flavor_linters(
linter_init_params, self.megalinter_flavor)
else:
all_linters = linter_factory.list_all_linters(linter_init_params)
skipped_linters = []
# Remove inactive or disabled linters
for linter in all_linters:
linter.master = self
if linter.is_active is False or linter.disabled is True:
skipped_linters += [linter.name]
if linter.disabled is True:
logging.warning(
f"{linter.name} has been temporary disabled in MegaLinter, please use a "
"previous MegaLinter version or wait for the next one !"
)
continue
self.linters += [linter]
# Display skipped linters in log
show_skipped_linters = config.get("SHOW_SKIPPED_LINTERS",
"true") == "true"
if len(skipped_linters) > 0 and show_skipped_linters:
skipped_linters.sort()
logging.info("Skipped linters: " + ", ".join(skipped_linters))
# Sort linters by language and linter_name
self.linters = sorted(self.linters,
key=lambda l:
(l.processing_order, l.descriptor_id))
def produce_report(self):
if (
config.exists("GITHUB_REPOSITORY")
and config.exists("GITHUB_SHA")
and config.exists("GITHUB_TOKEN")
and config.exists("GITHUB_RUN_ID")
):
github_repo = config.get("GITHUB_REPOSITORY")
sha = config.get("GITHUB_SHA")
run_id = config.get("GITHUB_RUN_ID")
success_msg = "No errors were found in the linting process"
error_not_blocking = "Errors were detected but are considered not blocking"
error_msg = "Errors were detected, please view logs"
url = f"{self.github_api_url}/repos/{github_repo}/statuses/{sha}"
headers = {
"accept": "application/vnd.github.v3+json",
"authorization": f"Bearer {config.get('GITHUB_TOKEN')}",
"content-type": "application/json",
}
target_url = f"https://github.com/{github_repo}/actions/runs/{run_id}"
description = (
success_msg
if self.master.status == "success" and self.master.return_code == 0
else error_not_blocking
if self.master.status == "error" and self.master.return_code == 0
else error_msg
)
if self.master.show_elapsed_time is True:
description += f" ({str(round(self.master.elapsed_time_s, 2))}s)"
data = {
"state": "success" if self.master.return_code == 0 else "error",
"target_url": target_url,
"description": description,
"context": f"--> Lint: {self.master.descriptor_id} with {self.master.linter_name}",
}
response = requests.post(url, headers=headers, json=data)
if 200 <= response.status_code < 299:
logging.debug(
f"Successfully posted Github Status for {self.master.descriptor_id} with {self.master.linter_name}"
)
else:
logging.error(
f"Error posting Github Status for {self.master.descriptor_id}"
f"with {self.master.linter_name}: {response.status_code}"
)
logging.error(f"GitHub API response: {response.text}")
else:
logging.debug(
f"Skipped post of Github Status for {self.master.descriptor_id} with {self.master.linter_name}"
)
def load_linters(self):
# Linters init params
linter_init_params = {
"linter_rules_path": self.linter_rules_path,
"default_rules_location": self.default_rules_location,
"default_linter_activation": self.default_linter_activation,
"enable_descriptors": self.enable_descriptors,
"enable_linters": self.enable_linters,
"disable_descriptors": self.disable_descriptors,
"disable_linters": self.disable_linters,
"workspace": self.workspace,
"github_workspace": self.github_workspace,
"report_folder": self.report_folder,
"apply_fixes": self.apply_fixes,
"show_elapsed_time": self.show_elapsed_time,
}
# Build linters from descriptor files
all_linters = linter_factory.list_all_linters(linter_init_params)
skipped_linters = []
for linter in all_linters:
if linter.is_active is False:
skipped_linters += [linter.name]
continue
self.linters += [linter]
# Display skipped linters in log
show_skipped_linters = config.get("SHOW_SKIPPED_LINTERS", "true") == "true"
if len(skipped_linters) > 0 and show_skipped_linters:
skipped_linters.sort()
logging.info("Skipped linters: " + ", ".join(skipped_linters))
# Sort linters by language and linter_name
self.linters = sorted(
self.linters, key=lambda l: (l.processing_order, l.descriptor_id)
)
def __init__(self, params=None):
# Activate console output by default
self.is_active = True
self.report_type = "simple"
if config.get("OUTPUT_DETAIL", "") == "detailed":
self.report_type = "detailed"
super().__init__(params)
def produce_report(self):
# Doc URL
lang_lower = self.master.descriptor_id.lower()
linter_name_lower = self.master.linter_name.lower().replace("-", "_")
doc_name = f"{lang_lower}_{linter_name_lower}"
doc_url = f"https://nvuillam.github.io/mega-linter/descriptors/{doc_name}/"
# Finalize lines
text_report_lines = [
f"Results of {self.master.linter_name} linter (version {self.master.get_linter_version()})",
f"See documentation on {doc_url}",
"-----------------------------------------------",
"",
]
text_report_lines += self.report_items
text_report_lines += self.master.complete_text_reporter_report(self)
text_report_sub_folder = config.get("TEXT_REPORTER_SUB_FOLDER",
"linters_logs")
text_file_name = (
f"{self.report_folder}{os.path.sep}"
f"{text_report_sub_folder}{os.path.sep}"
f"{self.master.status.upper()}-{self.master.name}.log")
if not os.path.isdir(os.path.dirname(text_file_name)):
os.makedirs(os.path.dirname(text_file_name), exist_ok=True)
with open(text_file_name, "w", encoding="utf-8") as text_file:
text_file_content = "\n".join(text_report_lines) + "\n"
text_file.write(text_file_content)
logging.debug(f"Generated {self.name} report: {text_file_name}")
def manage_docker_command(self, command):
if self.cli_docker_image is None:
return command
docker_command = ["docker", "run"]
if hasattr(self, "workspace"):
volume_root = config.get("MEGALINTER_VOLUME_ROOT", "")
if volume_root != "":
workspace_value = (volume_root + "/" +
self.workspace.replace("/tmp/lint", ""))
else:
workspace_value = self.workspace
else:
workspace_value = "/tmp/lint"
docker_command += map(
lambda arg, w=workspace_value: arg.replace("{{WORKSPACE}}", w),
self.cli_docker_args,
)
docker_command += [
f"{self.cli_docker_image}:{self.cli_docker_image_version}"
]
if type(command) == str:
command = " ".join(docker_command) + " " + command
else:
command = docker_command + command # ["ls", "-A", "/tmp/lint"]
return command
def test_linter_failure(linter, test_self):
if linter.disabled is True:
raise unittest.SkipTest("Linter has been disabled")
test_folder = linter.test_folder
workspace = config.get("DEFAULT_WORKSPACE") + os.path.sep + test_folder
if os.path.isdir(workspace + os.path.sep + "bad"):
workspace = workspace + os.path.sep + "bad"
workspace = manage_copy_sources(workspace)
tmp_report_folder = tempfile.gettempdir() + os.path.sep + str(uuid.uuid4())
assert os.path.isdir(workspace), f"Test folder {workspace} is not existing"
if os.path.isfile(workspace + os.path.sep + "no_test_failure"):
raise unittest.SkipTest(
f"Skip failure test for {linter}: no_test_failure found in test folder"
)
linter_name = linter.linter_name
env_vars_failure = {
"DEFAULT_WORKSPACE": workspace,
"FILTER_REGEX_INCLUDE": r"(bad)",
"OUTPUT_FORMAT": "text",
"OUTPUT_DETAIL": "detailed",
"REPORT_OUTPUT_FOLDER": tmp_report_folder,
"LOG_LEVEL": "DEBUG",
"ENABLE_LINTERS": linter.name,
}
if linter.lint_all_other_linters_files is not False:
env_vars_failure["ENABLE_LINTERS"] += ",JAVASCRIPT_ES"
env_vars_failure.update(linter.test_variables)
mega_linter, output = call_mega_linter(env_vars_failure)
# Check linter run
test_self.assertTrue(
len(mega_linter.linters) > 0, "Linters have been created and run"
)
# Check console output
if linter.cli_lint_mode == "file":
if len(linter.file_names_regex) > 0 and len(linter.file_extensions) == 0:
test_self.assertRegex(
output, rf"\[{linter_name}\] .*{linter.file_names_regex[0]}.* - ERROR"
)
test_self.assertNotRegex(
output, rf"\[{linter_name}\] .*{linter.file_names_regex[0]}.* - SUCCESS"
)
else:
test_self.assertRegex(output, rf"\[{linter_name}\] .*bad.* - ERROR")
test_self.assertNotRegex(output, rf"\[{linter_name}\] .*bad.* - SUCCESS")
else:
test_self.assertRegex(
output,
rf"Linted \[{linter.descriptor_id}\] files with \[{linter_name}\]: Found",
)
# Check text reporter output log
report_file_name = f"ERROR-{linter.name}.log"
text_report_file = (
f"{tmp_report_folder}{os.path.sep}linters_logs"
f"{os.path.sep}{report_file_name}"
)
test_self.assertTrue(
os.path.isfile(text_report_file),
f"Unable to find text report {text_report_file}",
)
copy_logs_for_doc(text_report_file, test_folder, report_file_name)
def produce_report(self):
if self.master.cli_lint_mode == "project":
return
tap_report_lines = [
"TAP version 13", f"1..{str(len(self.master.files))}"
]
# Convert file results in TAP
for index, file_result in enumerate(self.master.files_lint_results):
file_nm = utils.normalize_log_string(file_result["file"])
tap_status = "ok" if file_result["status_code"] == 0 else "not ok"
file_tap_lines = [f"{tap_status} {str(index + 1)} - {file_nm}"]
if (self.report_type == "detailed" and file_result["stdout"] != ""
and file_result["status_code"] != 0):
std_out_tap = (file_result["stdout"].rstrip(f" {os.linesep}") +
os.linesep)
std_out_tap = "\\n".join(std_out_tap.splitlines())
std_out_tap = std_out_tap.replace(":", " ")
detailed_lines = [
" ---", f" message: {std_out_tap}", " ..."
]
file_tap_lines += detailed_lines
tap_report_lines += file_tap_lines
# Write TAP file
tap_report_sub_folder = config.get("TAP_REPORTER_SUB_FOLDER", "tap")
tap_file_name = (f"{self.report_folder}{os.path.sep}"
f"{tap_report_sub_folder}{os.path.sep}"
f"mega-linter-{self.master.name}.tap")
if not os.path.isdir(os.path.dirname(tap_file_name)):
os.makedirs(os.path.dirname(tap_file_name), exist_ok=True)
with open(tap_file_name, "w", encoding="utf-8") as tap_file:
tap_file_content = "\n".join(tap_report_lines) + "\n"
tap_file.write(tap_file_content)
logging.info(
f"[Tap Reporter] Generated {self.name} report: {tap_file_name}"
)
def check_active_linters_match_flavor(active_linters):
flavor = get_image_flavor()
if flavor == "all":
logging.debug('MegaLinter flavor is "all", no need to check match with linters')
return True
all_flavors = get_all_flavors()
flavor_linters = all_flavors[flavor]["linters"]
missing_linters = []
for active_linter in active_linters:
if active_linter.name not in flavor_linters:
missing_linters += [active_linter.name]
active_linter.is_active = False
if len(missing_linters) > 0:
missing_linters_str = ",".join(missing_linters)
logging.warning(
f"MegaLinter flavor [{flavor}] does not contain linters {missing_linters_str}.\n"
"As they are not available in this docker image, they will not be processed\n"
"To solve this problem, please either: \n"
f"- use default flavor {ML_REPO}\n"
"- add ignored linters in DISABLE or DISABLE_LINTERS variables in your .mega-linter.yml config file "
"located in your root directory\n"
"- ignore this message by setting config variable FLAVOR_SUGGESTIONS to false"
)
if config.get("FAIL_IF_MISSING_LINTER_IN_FLAVOR", "") == "true":
logging.error(
'Missing linter and FAIL_IF_MISSING_LINTER_IN_FLAVOR has been set to "true": Stop run'
)
sys.exit(84)
return False
return True
def check_results(self):
print(f"::set-output name=has_updated_sources::{str(self.has_updated_sources)}")
if self.status == "success":
logging.info("✅ Successfully linted all files without errors")
config.delete()
elif self.status == "warning":
logging.warning("◬ Successfully linted all files, but with ignored errors")
config.delete()
else:
logging.error("❌ Error(s) have been found during linting")
logging.warning(
"To disable linters or customize their checks, you can use a .mega-linter.yml file "
"at the root of your repository"
)
logging.warning(
"More info at https://nvuillam.github.io/mega-linter/configuration/"
)
if self.cli is True:
if config.get("DISABLE_ERRORS", "false") == "true":
config.delete()
sys.exit(0)
else:
config.delete()
sys.exit(self.return_code)
config.delete()
def load_config_vars(self):
# Linter rules root path
if config.exists("LINTER_RULES_PATH"):
self.linter_rules_path = (self.github_workspace + os.path.sep +
config.get("LINTER_RULES_PATH"))
# Filtering regex (inclusion)
if config.exists("FILTER_REGEX_INCLUDE"):
self.filter_regex_include = config.get("FILTER_REGEX_INCLUDE")
# Filtering regex (exclusion)
if config.exists("FILTER_REGEX_EXCLUDE"):
self.filter_regex_exclude = config.get("FILTER_REGEX_EXCLUDE")
# Disable all fields validation if VALIDATE_ALL_CODEBASE is 'false'
if (config.exists("VALIDATE_ALL_CODEBASE")
and config.get("VALIDATE_ALL_CODEBASE") == "false"):
self.validate_all_code_base = False
