def extract_to_package_format(self) -> int:
"""Extracts the self.input yml file into several files according to the Demisto standard of the package format.
Returns:
int. status code for the operation.
"""
try:
output_path = self.get_output_path()
except ValueError as ex:
print_error(str(ex))
return 1
self.print_logs("Starting migration of: {} to dir: {}".format(self.input, output_path), log_color=LOG_COLORS.NATIVE)
os.makedirs(output_path, exist_ok=True)
base_name = os.path.basename(output_path) if not self.base_name else self.base_name
code_file = "{}/{}".format(output_path, base_name)
self.extract_code(code_file)
script = self.yml_data['script']
lang_type: str = script['type'] if self.file_type == 'integration' else self.yml_data['type']
code_file = f"{code_file}{TYPE_TO_EXTENSION[lang_type]}"
self.extract_image("{}/{}_image.png".format(output_path, base_name))
self.extract_long_description("{}/{}_description.md".format(output_path, base_name))
yaml_out = "{}/{}.yml".format(output_path, base_name)
self.print_logs("Creating yml file: {} ...".format(yaml_out), log_color=LOG_COLORS.NATIVE)
ryaml = YAML()
ryaml.preserve_quotes = True
with open(self.input, 'r') as yf:
yaml_obj = ryaml.load(yf)
script_obj = yaml_obj
if self.file_type == 'integration':
script_obj = yaml_obj['script']
if 'image' in yaml_obj:
del yaml_obj['image']
if 'detaileddescription' in yaml_obj:
del yaml_obj['detaileddescription']
script_obj['script'] = SingleQuotedScalarString('')
code_type = script_obj['type']
if code_type == TYPE_PWSH and not yaml_obj.get('fromversion'):
self.print_logs("Setting fromversion for PowerShell to: 5.5.0", log_color=LOG_COLORS.NATIVE)
yaml_obj['fromversion'] = "5.5.0"
with open(yaml_out, 'w') as yf:
ryaml.dump(yaml_obj, yf)
# check if there is a README and if found, set found_readme to True
found_readme = False
if self.readme:
yml_readme = os.path.splitext(self.input)[0] + '_README.md'
readme = output_path + '/README.md'
if os.path.exists(yml_readme):
found_readme = True
self.print_logs(f"Copying {readme} to {readme}", log_color=LOG_COLORS.NATIVE)
shutil.copy(yml_readme, readme)
else:
# open an empty file
with open(readme, 'w'):
pass
# Python code formatting and dev env setup
if code_type == TYPE_PYTHON:
if self.basic_fmt:
self.print_logs("Running autopep8 on file: {} ...".format(code_file), log_color=LOG_COLORS.NATIVE)
try:
subprocess.call(["autopep8", "-i", "--max-line-length", "130", code_file])
except FileNotFoundError:
self.print_logs("autopep8 skipped! It doesn't seem you have autopep8 installed.\n"
"Make sure to install it with: pip install autopep8.\n"
"Then run: autopep8 -i {}".format(code_file), LOG_COLORS.YELLOW)
if self.pipenv:
if self.basic_fmt:
self.print_logs("Running isort on file: {} ...".format(code_file), LOG_COLORS.NATIVE)
try:
subprocess.call(["isort", code_file])
except FileNotFoundError:
self.print_logs("isort skipped! It doesn't seem you have isort installed.\n"
"Make sure to install it with: pip install isort.\n"
"Then run: isort {}".format(code_file), LOG_COLORS.YELLOW)
self.print_logs("Detecting python version and setting up pipenv files ...", log_color=LOG_COLORS.NATIVE)
docker = get_all_docker_images(script_obj)[0]
py_ver = get_python_version(docker, self.config.log_verbose)
pip_env_dir = get_pipenv_dir(py_ver, self.config.envs_dirs_base)
self.print_logs("Copying pipenv files from: {}".format(pip_env_dir), log_color=LOG_COLORS.NATIVE)
shutil.copy("{}/Pipfile".format(pip_env_dir), output_path)
shutil.copy("{}/Pipfile.lock".format(pip_env_dir), output_path)
env = os.environ.copy()
env["PIPENV_IGNORE_VIRTUALENVS"] = "1"
try:
subprocess.call(["pipenv", "install", "--dev"], cwd=output_path, env=env)
self.print_logs("Installing all py requirements from docker: [{}] into pipenv".format(docker),
LOG_COLORS.NATIVE)
requirements = get_pip_requirements(docker)
fp = tempfile.NamedTemporaryFile(delete=False)
fp.write(requirements.encode('utf-8'))
fp.close()
try:
subprocess.check_call(["pipenv", "install", "-r", fp.name], cwd=output_path, env=env)
except Exception:
self.print_logs("Failed installing requirements in pipenv.\n "
"Please try installing manually after extract ends\n", LOG_COLORS.RED)
os.unlink(fp.name)
self.print_logs("Installing flake8 for linting", log_color=LOG_COLORS.NATIVE)
subprocess.call(["pipenv", "install", "--dev", "flake8"], cwd=output_path, env=env)
except FileNotFoundError as err:
self.print_logs("pipenv install skipped! It doesn't seem you have pipenv installed.\n"
"Make sure to install it with: pip3 install pipenv.\n"
f"Then run in the package dir: pipenv install --dev\n.Err: {err}", LOG_COLORS.YELLOW)
arg_path = os.path.relpath(output_path)
self.print_logs("\nCompleted: setting up package: {}\n".format(arg_path), LOG_COLORS.GREEN)
next_steps: str = "Next steps: \n" \
"* Install additional py packages for unit testing (if needed): cd {};" \
" pipenv install <package>\n".format(arg_path) if code_type == TYPE_PYTHON else ''
next_steps += "* Create unit tests\n" \
"* Check linting and unit tests by running: demisto-sdk lint -i {}\n".format(arg_path)
next_steps += "* When ready, remove from git the old yml and/or README and add the new package:\n" \
" git rm {}\n".format(self.input)
if found_readme:
next_steps += " git rm {}\n".format(os.path.splitext(self.input)[0] + '_README.md')
next_steps += " git add {}\n".format(arg_path)
self.print_logs(next_steps, log_color=LOG_COLORS.NATIVE)
else:
self.print_logs("Skipping pipenv and requirements installation - Note: no Pipfile will be created",
log_color=LOG_COLORS.YELLOW)
self.print_logs(f"Finished splitting the yml file - you can find the split results here: {output_path}",
log_color=LOG_COLORS.GREEN)
return 0
def extract_to_package_format(self) -> int:
"""Extracts the self.yml_path into several files according to the Demisto standard of the package format.
Returns:
int. status code for the operation.
"""
print("Starting migration of: {} to dir: {}".format(
self.yml_path, self.dest_path))
arg_path = self.dest_path
output_path = os.path.abspath(self.dest_path)
os.makedirs(output_path, exist_ok=True)
base_name = os.path.basename(output_path)
yml_type = self.get_yml_type()
code_file = "{}/{}.py".format(output_path, base_name)
self.extract_code(code_file)
self.extract_image("{}/{}_image.png".format(output_path, base_name))
self.extract_long_description("{}/{}_description.md".format(
output_path, base_name))
yaml_out = "{}/{}.yml".format(output_path, base_name)
print("Creating yml file: {} ...".format(yaml_out))
ryaml = YAML()
ryaml.preserve_quotes = True
with open(self.yml_path, 'r') as yf:
yaml_obj = ryaml.load(yf)
script_obj = yaml_obj
if yml_type == INTEGRATION:
script_obj = yaml_obj['script']
del yaml_obj['image']
if 'detaileddescription' in yaml_obj:
del yaml_obj['detaileddescription']
if script_obj['type'] != 'python':
print(
'Script is not of type "python". Found type: {}. Nothing to do.'
.format(script_obj['type']))
return 1
script_obj['script'] = SingleQuotedScalarString('')
with open(yaml_out, 'w') as yf:
ryaml.dump(yaml_obj, yf)
print("Running autopep8 on file: {} ...".format(code_file))
try:
subprocess.call(
["autopep8", "-i", "--max-line-length", "130", code_file])
except FileNotFoundError:
print_color(
"autopep8 skipped! It doesn't seem you have autopep8 installed.\n"
"Make sure to install it with: pip install autopep8.\n"
"Then run: autopep8 -i {}".format(code_file),
LOG_COLORS.YELLOW)
print("Detecting python version and setting up pipenv files ...")
docker = get_docker_images(script_obj)[0]
py_ver = get_python_version(docker, self.config.log_verbose)
pip_env_dir = get_pipenv_dir(py_ver, self.config.envs_dirs_base)
print("Copying pipenv files from: {}".format(pip_env_dir))
shutil.copy("{}/Pipfile".format(pip_env_dir), output_path)
shutil.copy("{}/Pipfile.lock".format(pip_env_dir), output_path)
try:
subprocess.call(["pipenv", "install", "--dev"], cwd=output_path)
print(
"Installing all py requirements from docker: [{}] into pipenv".
format(docker))
requirements = subprocess.check_output(
[
"docker", "run", "--rm", docker, "pip", "freeze",
"--disable-pip-version-check"
],
universal_newlines=True,
stderr=subprocess.DEVNULL).strip()
fp = tempfile.NamedTemporaryFile(delete=False)
fp.write(requirements.encode('utf-8'))
fp.close()
try:
subprocess.check_call(["pipenv", "install", "-r", fp.name],
cwd=output_path)
except Exception:
print_color(
"Failed installing requirements in pipenv.\n "
"Please try installing manually after extract ends\n",
LOG_COLORS.RED)
os.unlink(fp.name)
print("Installing flake8 for linting")
subprocess.call(["pipenv", "install", "--dev", "flake8"],
cwd=output_path)
except FileNotFoundError:
print_color(
"pipenv install skipped! It doesn't seem you have pipenv installed.\n"
"Make sure to install it with: pip3 install pipenv.\n"
"Then run in the package dir: pipenv install --dev",
LOG_COLORS.YELLOW)
# check if there is a changelog
yml_changelog = os.path.splitext(self.yml_path)[0] + '_CHANGELOG.md'
changelog = arg_path + '/CHANGELOG.md'
if os.path.exists(yml_changelog):
shutil.copy(yml_changelog, changelog)
else:
with open(changelog, 'wt', encoding='utf-8') as changelog_file:
changelog_file.write("## [Unreleased]\n-\n")
print_color("\nCompleted: setting up package: {}\n".format(arg_path),
LOG_COLORS.GREEN)
print(
"Next steps: \n",
"* Install additional py packages for unit testing (if needed): cd {}; pipenv install <package>\n"
.format(arg_path),
"* Create unit tests\n",
"* Check linting and unit tests by running: ./Tests/scripts/pkg_dev_test_tasks.py -d {}\n"
.format(arg_path),
"* When ready rm from git the source yml and add the new package:\n",
" git rm {}\n".format(self.yml_path),
" git add {}\n".format(arg_path),
sep='')
return 0