def _gather_facts(self, modules: dict) -> bool:
""" Gathering facts about the package - python version, docker images, valid docker image, yml parsing
Args:
modules(dict): Test mandatory modules to be ignore in lint check
Returns:
bool: Indicating if to continue further or not, if False exit Thread, Else continue.
"""
# Looking for pkg yaml
yml_file: Optional[Path] = self._pack_abs_dir.glob(
[r'*.yaml', r'*.yml', r'!*unified*.yml'], flags=NEGATE)
if not yml_file:
logger.info(
f"{self._pack_abs_dir} - Skipping no yaml file found {yml_file}"
)
self._pkg_lint_status["errors"].append(
'Unable to find yml file in package')
return True
else:
try:
yml_file = next(yml_file)
except StopIteration:
return True
# Get pack name
self._pack_name = yml_file.stem
log_prompt = f"{self._pack_name} - Facts"
self._pkg_lint_status["pkg"] = yml_file.stem
logger.info(f"{log_prompt} - Using yaml file {yml_file}")
# Parsing pack yaml - in order to verify if check needed
try:
script_obj: Dict = {}
yml_obj: Dict = YAML().load(yml_file)
if isinstance(yml_obj, dict):
script_obj = yml_obj.get('script', {}) if isinstance(
yml_obj.get('script'), dict) else yml_obj
self._pkg_lint_status["pack_type"] = script_obj.get('type')
except (FileNotFoundError, IOError, KeyError):
self._pkg_lint_status["errors"].append(
'Unable to parse package yml')
return True
# return no check needed if not python pack
if self._pkg_lint_status["pack_type"] not in (TYPE_PYTHON, TYPE_PWSH):
logger.info(
f"{log_prompt} - Skipping due to not Python, Powershell package - Pack is"
f" {self._pkg_lint_status['pack_type']}")
return True
# Docker images
if self._facts["docker_engine"]:
logger.info(
f"{log_prompt} - Pulling docker images, can take up to 1-2 minutes if not exists locally "
)
self._facts["images"] = [[
image, -1
] for image in get_all_docker_images(script_obj=script_obj)]
# Gather environment variables for docker execution
self._facts["env_vars"] = {
"CI":
os.getenv("CI", False),
"DEMISTO_LINT_UPDATE_CERTS":
os.getenv('DEMISTO_LINT_UPDATE_CERTS', "yes")
}
lint_files = set()
# Facts for python pack
if self._pkg_lint_status["pack_type"] == TYPE_PYTHON:
if self._facts["docker_engine"]:
# Getting python version from docker image - verifying if not valid docker image configured
for image in self._facts["images"]:
py_num: float = get_python_version_from_image(
image=image[0])
image[1] = py_num
logger.info(
f"{self._pack_name} - Facts - {image[0]} - Python {py_num}"
)
if not self._facts["python_version"]:
self._facts["python_version"] = py_num
# Checking whatever *test* exists in package
self._facts["test"] = True if next(
self._pack_abs_dir.glob([r'test_*.py', r'*_test.py']),
None) else False
if self._facts["test"]:
logger.info(f"{log_prompt} - Tests found")
else:
logger.info(f"{log_prompt} - Tests not found")
# Gather package requirements embedded test-requirements.py file
test_requirements = self._pack_abs_dir / 'test-requirements.txt'
if test_requirements.exists():
try:
additional_req = test_requirements.read_text(
encoding='utf-8').strip().split('\n')
self._facts["additional_requirements"].extend(
additional_req)
logger.info(
f"{log_prompt} - Additional package Pypi packages found - {additional_req}"
)
except (FileNotFoundError, IOError):
self._pkg_lint_status["errors"].append(
'Unable to parse test-requirements.txt in package')
# Get lint files
lint_files = set(
self._pack_abs_dir.glob(["*.py", "!__init__.py", "!*.tmp"],
flags=NEGATE))
# Facts for Powershell pack
elif self._pkg_lint_status["pack_type"] == TYPE_PWSH:
# Get lint files
lint_files = set(
self._pack_abs_dir.glob([
"*.ps1", "!*Tests.ps1", "CommonServerPowerShell.ps1",
"demistomock.ps1'"
],
flags=NEGATE))
# Add CommonServer to the lint checks
if 'commonserver' in self._pack_abs_dir.name.lower():
# Powershell
if self._pkg_lint_status["pack_type"] == TYPE_PWSH:
self._facts["lint_files"] = [
Path(self._pack_abs_dir / 'CommonServerPowerShell.ps1')
]
# Python
elif self._pkg_lint_status["pack_type"] == TYPE_PYTHON:
self._facts["lint_files"] = [
Path(self._pack_abs_dir / 'CommonServerPython.py')
]
else:
test_modules = {
self._pack_abs_dir / module.name
for module in modules.keys()
}
lint_files = lint_files.difference(test_modules)
self._facts["lint_files"] = list(lint_files)
if self._facts["lint_files"]:
for lint_file in self._facts["lint_files"]:
logger.info(f"{log_prompt} - Lint file {lint_file}")
else:
logger.info(f"{log_prompt} - Lint files not found")
self._split_lint_files()
return False
def run_dev_packages(self) -> int:
return_code = 0
# load yaml
_, yml_path = get_yml_paths_in_dir(
self.project_dir, Errors.no_yml_file(self.project_dir))
if not yml_path:
return 1
print_v('Using yaml file: {}'.format(yml_path))
with open(yml_path, 'r') as yml_file:
yml_data = yaml.safe_load(yml_file)
script_obj = yml_data
if isinstance(script_obj.get('script'), dict):
script_obj = script_obj.get('script')
script_type = script_obj.get('type')
if script_type != 'python':
if script_type == 'powershell':
# TODO powershell linting
return 0
print(
'Script is not of type "python". Found type: {}. Nothing to do.'
.format(script_type))
return 0
dockers = get_all_docker_images(script_obj)
py_num = get_python_version(dockers[0], self.log_verbose)
self.lock.acquire()
print_color(
"============ Starting process for: {} ============\n".format(
self.project_dir), LOG_COLORS.YELLOW)
if self.lock.locked():
self.lock.release()
self._setup_dev_files(py_num)
if self.run_args['flake8']:
result_val = self.run_flake8(py_num)
if result_val:
return_code = result_val
if self.run_args['mypy']:
result_val = self.run_mypy(py_num)
if result_val:
return_code = result_val
if self.run_args['bandit']:
result_val = self.run_bandit(py_num)
if result_val:
return_code = result_val
for docker in dockers:
for try_num in (1, 2):
print_v("Using docker image: {}".format(docker))
py_num = get_python_version(docker, self.log_verbose)
try:
if self.run_args['tests'] or self.run_args['pylint']:
if py_num == 2.7:
requirements = self.requirements_2
else:
requirements = self.requirements_3
docker_image_created = self._docker_image_create(
docker, requirements)
output, status_code = self._docker_run(
docker_image_created)
self.lock.acquire()
print_color(
"\n========== Running tests/pylint for: {} ========="
.format(self.project_dir), LOG_COLORS.YELLOW)
if status_code == 1:
raise subprocess.CalledProcessError(*output)
else:
print(output)
print_color(
"============ Finished process for: {} "
"with docker: {} ============\n".format(
self.project_dir, docker),
LOG_COLORS.GREEN)
if self.lock.locked():
self.lock.release()
break # all is good no need to retry
except subprocess.CalledProcessError as ex:
if ex.output:
print_color(
"=========================== ERROR IN {}==========================="
"\n{}\n".format(self.project_dir, ex.output),
LOG_COLORS.RED)
else:
print_color(
"========= Test Failed on {}, Look at the error/s above ========\n"
.format(self.project_dir), LOG_COLORS.RED)
return_code = 1
if not self.log_verbose:
sys.stderr.write(
"Need a more detailed log? try running with the -v options as so: \n{} -v\n\n"
.format(" ".join(sys.argv[:])))
if self.lock.locked():
self.lock.release()
# circle ci docker setup sometimes fails on
if try_num > 1 or not ex.output or 'read: connection reset by peer' not in ex.output:
return 2
else:
sys.stderr.write(
"Retrying as failure seems to be docker communication related...\n"
)
finally:
sys.stdout.flush()
sys.stderr.flush()
return return_code
def run_dev_packages(self) -> int:
return_code = 0
supported_types = (TYPE_PYTHON, TYPE_PWSH)
if self.script_type not in supported_types:
print_warning(
f'Script is not of types: {supported_types}. Found type: {self.script_type}. Nothing to do.'
)
return 0
dockers = get_all_docker_images(self.script_obj)
print_color(
"============ Starting process for: {} ============\n".format(
self.project_dir), LOG_COLORS.YELLOW)
if self.script_type == TYPE_PYTHON:
return_code = self.run_py_non_docker_tasks(dockers)
if self.script_type == TYPE_PWSH:
self._setup_dev_files_pwsh()
for docker in dockers:
for try_num in (1, 2):
print_v("Using docker image: {}".format(docker))
try:
if self.run_args['tests'] or self.run_args['pylint']:
docker_image_created = self._docker_image_create(
docker)
output, status_code = self._docker_run(
docker_image_created)
with self.lock:
print_color(
"\n========== Running tests/pylint for: {} ========="
.format(self.project_dir), LOG_COLORS.YELLOW)
if status_code == 1:
raise subprocess.CalledProcessError(*output)
else:
print(output)
print_color(
"============ Finished process for: {} "
"with docker: {} ============\n".format(
self.project_dir, docker),
LOG_COLORS.GREEN)
break # all is good no need to retry
except subprocess.CalledProcessError as ex:
with self.lock:
if ex.output:
print_color(
"=========================== ERROR IN {}==========================="
"\n{}\n".format(self.project_dir, ex.output),
LOG_COLORS.RED)
else:
print_color(
"========= Test Failed on {}, Look at the error/s above ========\n"
.format(self.project_dir), LOG_COLORS.RED)
return_code = 1
if not get_log_verbose():
sys.stderr.write(
"Need a more detailed log? try running with the -v options as so: \n{} -v\n\n"
.format(" ".join(sys.argv[:])))
# circle ci docker setup sometimes fails on
if try_num > 1 or not ex.output or 'read: connection reset by peer' not in ex.output:
return 2
else:
sys.stderr.write(
"Retrying as failure seems to be docker communication related...\n"
)
finally:
sys.stdout.flush()
sys.stderr.flush()
return return_code
def extract_to_package_format(self) -> int:
"""Extracts the self.input yml file into several files according to the Demisto standard of the package format.
Returns:
int. status code for the operation.
"""
try:
output_path = self.get_output_path()
except ValueError as ex:
print_error(str(ex))
return 1
self.print_logs("Starting migration of: {} to dir: {}".format(self.input, output_path), log_color=LOG_COLORS.NATIVE)
os.makedirs(output_path, exist_ok=True)
base_name = os.path.basename(output_path) if not self.base_name else self.base_name
code_file = "{}/{}".format(output_path, base_name)
self.extract_code(code_file)
script = self.yml_data['script']
lang_type: str = script['type'] if self.file_type == 'integration' else self.yml_data['type']
code_file = f"{code_file}{TYPE_TO_EXTENSION[lang_type]}"
self.extract_image("{}/{}_image.png".format(output_path, base_name))
self.extract_long_description("{}/{}_description.md".format(output_path, base_name))
yaml_out = "{}/{}.yml".format(output_path, base_name)
self.print_logs("Creating yml file: {} ...".format(yaml_out), log_color=LOG_COLORS.NATIVE)
ryaml = YAML()
ryaml.preserve_quotes = True
with open(self.input, 'r') as yf:
yaml_obj = ryaml.load(yf)
script_obj = yaml_obj
if self.file_type == 'integration':
script_obj = yaml_obj['script']
if 'image' in yaml_obj:
del yaml_obj['image']
if 'detaileddescription' in yaml_obj:
del yaml_obj['detaileddescription']
script_obj['script'] = SingleQuotedScalarString('')
code_type = script_obj['type']
if code_type == TYPE_PWSH and not yaml_obj.get('fromversion'):
self.print_logs("Setting fromversion for PowerShell to: 5.5.0", log_color=LOG_COLORS.NATIVE)
yaml_obj['fromversion'] = "5.5.0"
with open(yaml_out, 'w') as yf:
ryaml.dump(yaml_obj, yf)
# check if there is a README and if found, set found_readme to True
found_readme = False
if self.readme:
yml_readme = os.path.splitext(self.input)[0] + '_README.md'
readme = output_path + '/README.md'
if os.path.exists(yml_readme):
found_readme = True
self.print_logs(f"Copying {readme} to {readme}", log_color=LOG_COLORS.NATIVE)
shutil.copy(yml_readme, readme)
else:
# open an empty file
with open(readme, 'w'):
pass
# Python code formatting and dev env setup
if code_type == TYPE_PYTHON:
if self.basic_fmt:
self.print_logs("Running autopep8 on file: {} ...".format(code_file), log_color=LOG_COLORS.NATIVE)
try:
subprocess.call(["autopep8", "-i", "--max-line-length", "130", code_file])
except FileNotFoundError:
self.print_logs("autopep8 skipped! It doesn't seem you have autopep8 installed.\n"
"Make sure to install it with: pip install autopep8.\n"
"Then run: autopep8 -i {}".format(code_file), LOG_COLORS.YELLOW)
if self.pipenv:
if self.basic_fmt:
self.print_logs("Running isort on file: {} ...".format(code_file), LOG_COLORS.NATIVE)
try:
subprocess.call(["isort", code_file])
except FileNotFoundError:
self.print_logs("isort skipped! It doesn't seem you have isort installed.\n"
"Make sure to install it with: pip install isort.\n"
"Then run: isort {}".format(code_file), LOG_COLORS.YELLOW)
self.print_logs("Detecting python version and setting up pipenv files ...", log_color=LOG_COLORS.NATIVE)
docker = get_all_docker_images(script_obj)[0]
py_ver = get_python_version(docker, self.config.log_verbose)
pip_env_dir = get_pipenv_dir(py_ver, self.config.envs_dirs_base)
self.print_logs("Copying pipenv files from: {}".format(pip_env_dir), log_color=LOG_COLORS.NATIVE)
shutil.copy("{}/Pipfile".format(pip_env_dir), output_path)
shutil.copy("{}/Pipfile.lock".format(pip_env_dir), output_path)
env = os.environ.copy()
env["PIPENV_IGNORE_VIRTUALENVS"] = "1"
try:
subprocess.call(["pipenv", "install", "--dev"], cwd=output_path, env=env)
self.print_logs("Installing all py requirements from docker: [{}] into pipenv".format(docker),
LOG_COLORS.NATIVE)
requirements = get_pip_requirements(docker)
fp = tempfile.NamedTemporaryFile(delete=False)
fp.write(requirements.encode('utf-8'))
fp.close()
try:
subprocess.check_call(["pipenv", "install", "-r", fp.name], cwd=output_path, env=env)
except Exception:
self.print_logs("Failed installing requirements in pipenv.\n "
"Please try installing manually after extract ends\n", LOG_COLORS.RED)
os.unlink(fp.name)
self.print_logs("Installing flake8 for linting", log_color=LOG_COLORS.NATIVE)
subprocess.call(["pipenv", "install", "--dev", "flake8"], cwd=output_path, env=env)
except FileNotFoundError as err:
self.print_logs("pipenv install skipped! It doesn't seem you have pipenv installed.\n"
"Make sure to install it with: pip3 install pipenv.\n"
f"Then run in the package dir: pipenv install --dev\n.Err: {err}", LOG_COLORS.YELLOW)
arg_path = os.path.relpath(output_path)
self.print_logs("\nCompleted: setting up package: {}\n".format(arg_path), LOG_COLORS.GREEN)
next_steps: str = "Next steps: \n" \
"* Install additional py packages for unit testing (if needed): cd {};" \
" pipenv install <package>\n".format(arg_path) if code_type == TYPE_PYTHON else ''
next_steps += "* Create unit tests\n" \
"* Check linting and unit tests by running: demisto-sdk lint -i {}\n".format(arg_path)
next_steps += "* When ready, remove from git the old yml and/or README and add the new package:\n" \
" git rm {}\n".format(self.input)
if found_readme:
next_steps += " git rm {}\n".format(os.path.splitext(self.input)[0] + '_README.md')
next_steps += " git add {}\n".format(arg_path)
self.print_logs(next_steps, log_color=LOG_COLORS.NATIVE)
else:
self.print_logs("Skipping pipenv and requirements installation - Note: no Pipfile will be created",
log_color=LOG_COLORS.YELLOW)
self.print_logs(f"Finished splitting the yml file - you can find the split results here: {output_path}",
log_color=LOG_COLORS.GREEN)
return 0
def extract_to_package_format(self) -> int:
"""Extracts the self.input yml file into several files according to the Demisto standard of the package format.
Returns:
int. status code for the operation.
"""
print("Starting migration of: {} to dir: {}".format(
self.input, self.output))
arg_path = self.output
output_path = os.path.abspath(self.output)
os.makedirs(output_path, exist_ok=True)
base_name = os.path.basename(output_path)
code_file = "{}/{}.py".format(output_path, base_name)
self.extract_code(code_file)
self.extract_image("{}/{}_image.png".format(output_path, base_name))
self.extract_long_description("{}/{}_description.md".format(
output_path, base_name))
yaml_out = "{}/{}.yml".format(output_path, base_name)
print("Creating yml file: {} ...".format(yaml_out))
ryaml = YAML()
ryaml.preserve_quotes = True
with open(self.input, 'r') as yf:
yaml_obj = ryaml.load(yf)
script_obj = yaml_obj
if self.file_type == 'integration':
script_obj = yaml_obj['script']
del yaml_obj['image']
if 'detaileddescription' in yaml_obj:
del yaml_obj['detaileddescription']
if script_obj['type'] != 'python':
print(
'Script is not of type "python". Found type: {}. Nothing to do.'
.format(script_obj['type']))
return 1
script_obj['script'] = SingleQuotedScalarString('')
with open(yaml_out, 'w') as yf:
ryaml.dump(yaml_obj, yf)
print("Running autopep8 on file: {} ...".format(code_file))
try:
subprocess.call(
["autopep8", "-i", "--max-line-length", "130", code_file])
except FileNotFoundError:
print_color(
"autopep8 skipped! It doesn't seem you have autopep8 installed.\n"
"Make sure to install it with: pip install autopep8.\n"
"Then run: autopep8 -i {}".format(code_file),
LOG_COLORS.YELLOW)
print("Detecting python version and setting up pipenv files ...")
docker = get_all_docker_images(script_obj)[0]
py_ver = get_python_version(docker, self.config.log_verbose)
pip_env_dir = get_pipenv_dir(py_ver, self.config.envs_dirs_base)
print("Copying pipenv files from: {}".format(pip_env_dir))
shutil.copy("{}/Pipfile".format(pip_env_dir), output_path)
shutil.copy("{}/Pipfile.lock".format(pip_env_dir), output_path)
try:
subprocess.call(["pipenv", "install", "--dev"], cwd=output_path)
print(
"Installing all py requirements from docker: [{}] into pipenv".
format(docker))
requirements = subprocess.check_output(
[
"docker", "run", "--rm", docker, "pip", "freeze",
"--disable-pip-version-check"
],
universal_newlines=True,
stderr=subprocess.DEVNULL).strip()
fp = tempfile.NamedTemporaryFile(delete=False)
fp.write(requirements.encode('utf-8'))
fp.close()
try:
subprocess.check_call(["pipenv", "install", "-r", fp.name],
cwd=output_path)
except Exception:
print_color(
"Failed installing requirements in pipenv.\n "
"Please try installing manually after extract ends\n",
LOG_COLORS.RED)
os.unlink(fp.name)
print("Installing flake8 for linting")
subprocess.call(["pipenv", "install", "--dev", "flake8"],
cwd=output_path)
except FileNotFoundError:
print_color(
"pipenv install skipped! It doesn't seem you have pipenv installed.\n"
"Make sure to install it with: pip3 install pipenv.\n"
"Then run in the package dir: pipenv install --dev",
LOG_COLORS.YELLOW)
# check if there is a changelog
yml_changelog = os.path.splitext(self.input)[0] + '_CHANGELOG.md'
changelog = arg_path + '/CHANGELOG.md'
if os.path.exists(yml_changelog):
shutil.copy(yml_changelog, changelog)
else:
with open(changelog, 'wt', encoding='utf-8') as changelog_file:
changelog_file.write("## [Unreleased]\n-\n")
print_color("\nCompleted: setting up package: {}\n".format(arg_path),
LOG_COLORS.GREEN)
print(
"Next steps: \n",
"* Install additional py packages for unit testing (if needed): cd {}; pipenv install <package>\n"
.format(arg_path),
"* Create unit tests\n",
"* Check linting and unit tests by running: demisto-sdk lint -d {}\n"
.format(arg_path),
"* When ready rm from git the source yml and add the new package:\n",
" git rm {}\n".format(self.input),
" git add {}\n".format(arg_path),
sep='')
return 0