def are_modules_installed_for_verify(content_path: str) -> bool:
""" Check the following:
1. npm packages installed - see packs var for specific pack details.
2. node interperter exists.
Returns:
bool: True If all req ok else False
"""
missing_module = []
valid = True
# Check node exist
stdout, stderr, exit_code = run_command_os('node -v', cwd=content_path)
if exit_code:
print_warning(
f'There is no node installed on the machine, Test Skipped, error - {stderr}, {stdout}'
)
valid = False
else:
# Check npm modules exsits
stdout, stderr, exit_code = run_command_os(
f'npm ls --json {" ".join(REQUIRED_MDX_PACKS)}',
cwd=content_path)
if exit_code: # all are missinig
missing_module.extend(REQUIRED_MDX_PACKS)
else:
deps = json.loads(stdout).get('dependencies', {})
for pack in REQUIRED_MDX_PACKS:
if pack not in deps:
missing_module.append(pack)
if missing_module:
valid = False
print_warning(
f"The npm modules: {missing_module} are not installed, Readme mdx validation skipped. Use "
f"'npm install' to install all required node dependencies")
return valid
def are_modules_installed_for_verify(self) -> bool:
""" Check the following:
1. npm packages installed - see packs var for specific pack details.
2. node interperter exists.
Returns:
bool: True If all req ok else False
"""
missing_module = []
valid = True
# Check node exist
stdout, stderr, exit_code = run_command_os('node -v',
cwd=self.content_path)
if exit_code:
print_warning(
f'There is no node installed on the machine, Test Skipped, error - {stderr}, {stdout}'
)
valid = False
else:
# Check npm modules exsits
packs = ['@mdx-js/mdx', 'fs-extra', 'commander']
for pack in packs:
stdout, stderr, exit_code = run_command_os(
f'npm ls {pack}', cwd=self.content_path)
if exit_code:
missing_module.append(pack)
if missing_module:
valid = False
print_warning(
f"The npm modules: {missing_module} are not installed, Test Skipped, use "
f"'npm install <module>' to install all required node dependencies"
)
return valid
def validate_env() -> None:
"""Packs which use python2 will need to be run inside virtual environment including python2 as main
and the specified req
"""
wrn_msg = 'demisto-sdk lint not in virtual environment, Python2 lints will fail, use "source .hooks/bootstrap"' \
' to create the virtual environment'
command = "python -c \"import sys; print('{}.{}'.format(sys.version_info[0], sys.version_info[1]))\""
stdout, stderr, exit_code = run_command_os(command, cwd=Path().cwd())
if "2" not in stdout:
print_warning(wrn_msg)
else:
stdout, stderr, exit_code = run_command_os("pip3 freeze", cwd=Path().cwd())
for req in PYTHON2_REQ:
if req not in stdout:
print_warning(wrn_msg)
def _run_vulture(self, py_num: float,
lint_files: List[Path]) -> Tuple[int, str]:
""" Run mypy in pack dir
Args:
py_num(float): The python version in use
lint_files(List[Path]): file to perform lint
Returns:
int: 0 on successful else 1, errors
str: Vulture errors
"""
log_prompt = f"{self._pack_name} - Vulture"
logger.info(f"{log_prompt} - Start")
stdout, stderr, exit_code = run_command_os(
command=build_vulture_command(files=lint_files,
pack_path=self._pack_abs_dir,
py_num=py_num),
cwd=self._pack_abs_dir)
logger.debug(f"{log_prompt} - Finished exit-code: {exit_code}")
logger.debug(
f"{log_prompt} - Finished stdout: {RL if stdout else ''}{stdout}")
logger.debug(
f"{log_prompt} - Finished stderr: {RL if stderr else ''}{stderr}")
if stderr or exit_code:
logger.info(f"{log_prompt}- Finished Finished errors found")
if stderr:
return FAIL, stderr
else:
return FAIL, stdout
logger.info(f"{log_prompt} - Successfully finished")
return SUCCESS, ""
def _run_bandit(self, lint_files: List[Path]) -> Tuple[int, str]:
""" Run bandit in pack dir
Args:
lint_files(List[Path]): file to perform lint
Returns:
int: 0 on successful else 1, errors
str: Bandit errors
"""
log_prompt = f"{self._pack_name} - Bandit"
logger.info(f"{log_prompt} - Start")
stdout, stderr, exit_code = run_command_os(
command=build_bandit_command(lint_files), cwd=self._pack_abs_dir)
logger.debug(f"{log_prompt} - Finished exit-code: {exit_code}")
logger.debug(
f"{log_prompt} - Finished stdout: {RL if stdout else ''}{stdout}")
logger.debug(
f"{log_prompt} - Finished stderr: {RL if stderr else ''}{stderr}")
if stderr or exit_code:
logger.info(f"{log_prompt}- Finished Finished errors found")
if stderr:
return FAIL, stderr
else:
return FAIL, stdout
logger.info(f"{log_prompt} - Successfully finished")
return SUCCESS, ""
def test_run_command_os(command, cwd):
"""Tests a simple command, to check if it works
"""
stdout, stderr, return_code = run_command_os(command, cwd=cwd)
assert 0 == return_code
assert stdout
assert not stderr
def _run_xsoar_linter(self, py_num: float,
lint_files: List[Path]) -> Tuple[int, str]:
""" Runs Xsaor linter in pack dir
Args:
lint_files(List[Path]): file to perform lint
Returns:
int: 0 on successful else 1, errors
str: Xsoar linter errors
"""
status = SUCCESS
FAIL_PYLINT = 0b10
with pylint_plugin(self._pack_abs_dir):
log_prompt = f"{self._pack_name} - XSOAR Linter"
logger.info(f"{log_prompt} - Start")
myenv = os.environ.copy()
if myenv.get('PYTHONPATH'):
myenv['PYTHONPATH'] += ':' + str(self._pack_abs_dir)
else:
myenv['PYTHONPATH'] = str(self._pack_abs_dir)
if self._facts['is_long_running']:
myenv['LONGRUNNING'] = 'True'
if py_num < 3:
myenv['PY2'] = 'True'
stdout, stderr, exit_code = run_command_os(
command=build_xsoar_linter_command(
lint_files, py_num,
self._facts.get('support_level', 'base')),
cwd=self._pack_abs_dir,
env=myenv)
if exit_code & FAIL_PYLINT:
logger.info(f"{log_prompt}- Finished errors found")
status = FAIL
if exit_code & WARNING:
logger.info(f"{log_prompt} - Finished warnings found")
if not status:
status = WARNING
# if pylint did not run and failure exit code has been returned from run commnad
elif exit_code & FAIL:
status = FAIL
# for contrib prs which are not merged from master and do not have pylint in dev-requirements-py2.
if os.environ.get('CI'):
stdout = "Xsoar linter could not run, Please merge from master"
else:
stdout = "Xsoar linter could not run, please make sure you have" \
" the necessary Pylint version for both py2 and py3"
logger.info(f"{log_prompt}- Finished errors found")
logger.debug(f"{log_prompt} - Finished exit-code: {exit_code}")
logger.debug(
f"{log_prompt} - Finished stdout: {RL if stdout else ''}{stdout}")
logger.debug(
f"{log_prompt} - Finished stderr: {RL if stderr else ''}{stderr}")
if not exit_code:
logger.info(f"{log_prompt} - Successfully finished")
return status, stdout
def mdx_verify(self) -> bool:
mdx_parse = Path(__file__).parent.parent / 'mdx-parse.js'
# run the java script mdx parse validator
_, stderr, is_not_valid = run_command_os(
f'node {mdx_parse} -f {self.file_path}',
cwd=self.content_path,
env=os.environ)
if is_not_valid:
error_message, error_code = Errors.readme_error(stderr)
if self.handle_error(error_message,
error_code,
file_path=self.file_path):
return False
return True
def mdx_verify(self) -> bool:
mdx_parse = Path(__file__).parent.parent / 'mdx-parse.js'
readme_content = self.fix_mdx()
with tempfile.NamedTemporaryFile('w+t') as fp:
fp.write(readme_content)
fp.flush()
# run the javascript mdx parse validator
_, stderr, is_not_valid = run_command_os(
f'node {mdx_parse} -f {fp.name}',
cwd=self.content_path,
env=os.environ)
if is_not_valid:
error_message, error_code = Errors.readme_error(stderr)
if self.handle_error(error_message,
error_code,
file_path=self.file_path):
return False
return True
def is_mdx_file(self) -> bool:
html = self.is_html_doc()
valid = self.are_modules_installed_for_verify()
if valid and not html:
mdx_parse = Path(__file__).parent.parent / 'mdx-parse.js'
# add to env var the directory of node modules
os.environ['NODE_PATH'] = str(
self.node_modules_path) + os.pathsep + os.getenv(
"NODE_PATH", "")
# run the java script mdx parse validator
_, stderr, is_valid = run_command_os(
f'node {mdx_parse} -f {self.file_path}',
cwd=self.content_path,
env=os.environ)
if is_valid:
print_error(
f'Failed verifying README.md, Path: {self.file_path}. Error Message is: {stderr}'
)
return False
return True
def is_mdx_file(self) -> bool:
html = self.is_html_doc()
valid = self.are_modules_installed_for_verify()
if valid and not html:
mdx_parse = Path(__file__).parent.parent / 'mdx-parse.js'
# add to env var the directory of node modules
os.environ['NODE_PATH'] = str(
self.node_modules_path) + os.pathsep + os.getenv(
"NODE_PATH", "")
# run the java script mdx parse validator
_, stderr, is_valid = run_command_os(
f'node {mdx_parse} -f {self.file_path}',
cwd=self.content_path,
env=os.environ)
if is_valid:
error_message, error_code = Errors.readme_error(stderr)
if self.handle_error(error_message,
error_code,
file_path=self.file_path):
return False
return True
