def are_modules_installed_for_verify(content_path: str) -> bool: """ Check the following: 1. npm packages installed - see packs var for specific pack details. 2. node interperter exists. Returns: bool: True If all req ok else False """ missing_module = [] valid = True # Check node exist stdout, stderr, exit_code = run_command_os('node -v', cwd=content_path) if exit_code: print_warning( f'There is no node installed on the machine, Test Skipped, error - {stderr}, {stdout}' ) valid = False else: # Check npm modules exsits stdout, stderr, exit_code = run_command_os( f'npm ls --json {" ".join(REQUIRED_MDX_PACKS)}', cwd=content_path) if exit_code: # all are missinig missing_module.extend(REQUIRED_MDX_PACKS) else: deps = json.loads(stdout).get('dependencies', {}) for pack in REQUIRED_MDX_PACKS: if pack not in deps: missing_module.append(pack) if missing_module: valid = False print_warning( f"The npm modules: {missing_module} are not installed, Readme mdx validation skipped. Use " f"'npm install' to install all required node dependencies") return valid
def are_modules_installed_for_verify(self) -> bool: """ Check the following: 1. npm packages installed - see packs var for specific pack details. 2. node interperter exists. Returns: bool: True If all req ok else False """ missing_module = [] valid = True # Check node exist stdout, stderr, exit_code = run_command_os('node -v', cwd=self.content_path) if exit_code: print_warning( f'There is no node installed on the machine, Test Skipped, error - {stderr}, {stdout}' ) valid = False else: # Check npm modules exsits packs = ['@mdx-js/mdx', 'fs-extra', 'commander'] for pack in packs: stdout, stderr, exit_code = run_command_os( f'npm ls {pack}', cwd=self.content_path) if exit_code: missing_module.append(pack) if missing_module: valid = False print_warning( f"The npm modules: {missing_module} are not installed, Test Skipped, use " f"'npm install <module>' to install all required node dependencies" ) return valid
def validate_env() -> None: """Packs which use python2 will need to be run inside virtual environment including python2 as main and the specified req """ wrn_msg = 'demisto-sdk lint not in virtual environment, Python2 lints will fail, use "source .hooks/bootstrap"' \ ' to create the virtual environment' command = "python -c \"import sys; print('{}.{}'.format(sys.version_info[0], sys.version_info[1]))\"" stdout, stderr, exit_code = run_command_os(command, cwd=Path().cwd()) if "2" not in stdout: print_warning(wrn_msg) else: stdout, stderr, exit_code = run_command_os("pip3 freeze", cwd=Path().cwd()) for req in PYTHON2_REQ: if req not in stdout: print_warning(wrn_msg)
def _run_vulture(self, py_num: float, lint_files: List[Path]) -> Tuple[int, str]: """ Run mypy in pack dir Args: py_num(float): The python version in use lint_files(List[Path]): file to perform lint Returns: int: 0 on successful else 1, errors str: Vulture errors """ log_prompt = f"{self._pack_name} - Vulture" logger.info(f"{log_prompt} - Start") stdout, stderr, exit_code = run_command_os( command=build_vulture_command(files=lint_files, pack_path=self._pack_abs_dir, py_num=py_num), cwd=self._pack_abs_dir) logger.debug(f"{log_prompt} - Finished exit-code: {exit_code}") logger.debug( f"{log_prompt} - Finished stdout: {RL if stdout else ''}{stdout}") logger.debug( f"{log_prompt} - Finished stderr: {RL if stderr else ''}{stderr}") if stderr or exit_code: logger.info(f"{log_prompt}- Finished Finished errors found") if stderr: return FAIL, stderr else: return FAIL, stdout logger.info(f"{log_prompt} - Successfully finished") return SUCCESS, ""
def _run_bandit(self, lint_files: List[Path]) -> Tuple[int, str]: """ Run bandit in pack dir Args: lint_files(List[Path]): file to perform lint Returns: int: 0 on successful else 1, errors str: Bandit errors """ log_prompt = f"{self._pack_name} - Bandit" logger.info(f"{log_prompt} - Start") stdout, stderr, exit_code = run_command_os( command=build_bandit_command(lint_files), cwd=self._pack_abs_dir) logger.debug(f"{log_prompt} - Finished exit-code: {exit_code}") logger.debug( f"{log_prompt} - Finished stdout: {RL if stdout else ''}{stdout}") logger.debug( f"{log_prompt} - Finished stderr: {RL if stderr else ''}{stderr}") if stderr or exit_code: logger.info(f"{log_prompt}- Finished Finished errors found") if stderr: return FAIL, stderr else: return FAIL, stdout logger.info(f"{log_prompt} - Successfully finished") return SUCCESS, ""
def test_run_command_os(command, cwd): """Tests a simple command, to check if it works """ stdout, stderr, return_code = run_command_os(command, cwd=cwd) assert 0 == return_code assert stdout assert not stderr
def _run_xsoar_linter(self, py_num: float, lint_files: List[Path]) -> Tuple[int, str]: """ Runs Xsaor linter in pack dir Args: lint_files(List[Path]): file to perform lint Returns: int: 0 on successful else 1, errors str: Xsoar linter errors """ status = SUCCESS FAIL_PYLINT = 0b10 with pylint_plugin(self._pack_abs_dir): log_prompt = f"{self._pack_name} - XSOAR Linter" logger.info(f"{log_prompt} - Start") myenv = os.environ.copy() if myenv.get('PYTHONPATH'): myenv['PYTHONPATH'] += ':' + str(self._pack_abs_dir) else: myenv['PYTHONPATH'] = str(self._pack_abs_dir) if self._facts['is_long_running']: myenv['LONGRUNNING'] = 'True' if py_num < 3: myenv['PY2'] = 'True' stdout, stderr, exit_code = run_command_os( command=build_xsoar_linter_command( lint_files, py_num, self._facts.get('support_level', 'base')), cwd=self._pack_abs_dir, env=myenv) if exit_code & FAIL_PYLINT: logger.info(f"{log_prompt}- Finished errors found") status = FAIL if exit_code & WARNING: logger.info(f"{log_prompt} - Finished warnings found") if not status: status = WARNING # if pylint did not run and failure exit code has been returned from run commnad elif exit_code & FAIL: status = FAIL # for contrib prs which are not merged from master and do not have pylint in dev-requirements-py2. if os.environ.get('CI'): stdout = "Xsoar linter could not run, Please merge from master" else: stdout = "Xsoar linter could not run, please make sure you have" \ " the necessary Pylint version for both py2 and py3" logger.info(f"{log_prompt}- Finished errors found") logger.debug(f"{log_prompt} - Finished exit-code: {exit_code}") logger.debug( f"{log_prompt} - Finished stdout: {RL if stdout else ''}{stdout}") logger.debug( f"{log_prompt} - Finished stderr: {RL if stderr else ''}{stderr}") if not exit_code: logger.info(f"{log_prompt} - Successfully finished") return status, stdout
def mdx_verify(self) -> bool: mdx_parse = Path(__file__).parent.parent / 'mdx-parse.js' # run the java script mdx parse validator _, stderr, is_not_valid = run_command_os( f'node {mdx_parse} -f {self.file_path}', cwd=self.content_path, env=os.environ) if is_not_valid: error_message, error_code = Errors.readme_error(stderr) if self.handle_error(error_message, error_code, file_path=self.file_path): return False return True
def mdx_verify(self) -> bool: mdx_parse = Path(__file__).parent.parent / 'mdx-parse.js' readme_content = self.fix_mdx() with tempfile.NamedTemporaryFile('w+t') as fp: fp.write(readme_content) fp.flush() # run the javascript mdx parse validator _, stderr, is_not_valid = run_command_os( f'node {mdx_parse} -f {fp.name}', cwd=self.content_path, env=os.environ) if is_not_valid: error_message, error_code = Errors.readme_error(stderr) if self.handle_error(error_message, error_code, file_path=self.file_path): return False return True
def is_mdx_file(self) -> bool: html = self.is_html_doc() valid = self.are_modules_installed_for_verify() if valid and not html: mdx_parse = Path(__file__).parent.parent / 'mdx-parse.js' # add to env var the directory of node modules os.environ['NODE_PATH'] = str( self.node_modules_path) + os.pathsep + os.getenv( "NODE_PATH", "") # run the java script mdx parse validator _, stderr, is_valid = run_command_os( f'node {mdx_parse} -f {self.file_path}', cwd=self.content_path, env=os.environ) if is_valid: print_error( f'Failed verifying README.md, Path: {self.file_path}. Error Message is: {stderr}' ) return False return True
def is_mdx_file(self) -> bool: html = self.is_html_doc() valid = self.are_modules_installed_for_verify() if valid and not html: mdx_parse = Path(__file__).parent.parent / 'mdx-parse.js' # add to env var the directory of node modules os.environ['NODE_PATH'] = str( self.node_modules_path) + os.pathsep + os.getenv( "NODE_PATH", "") # run the java script mdx parse validator _, stderr, is_valid = run_command_os( f'node {mdx_parse} -f {self.file_path}', cwd=self.content_path, env=os.environ) if is_valid: error_message, error_code = Errors.readme_error(stderr) if self.handle_error(error_message, error_code, file_path=self.file_path): return False return True