def extract_to_package_format(self) -> int: """Extracts the self.input yml file into several files according to the Demisto standard of the package format. Returns: int. status code for the operation. """ try: output_path = self.get_output_path() except ValueError as ex: print_error(str(ex)) return 1 self.print_logs("Starting migration of: {} to dir: {}".format(self.input, output_path), log_color=LOG_COLORS.NATIVE) os.makedirs(output_path, exist_ok=True) base_name = os.path.basename(output_path) if not self.base_name else self.base_name code_file = "{}/{}".format(output_path, base_name) self.extract_code(code_file) script = self.yml_data['script'] lang_type: str = script['type'] if self.file_type == 'integration' else self.yml_data['type'] code_file = f"{code_file}{TYPE_TO_EXTENSION[lang_type]}" self.extract_image("{}/{}_image.png".format(output_path, base_name)) self.extract_long_description("{}/{}_description.md".format(output_path, base_name)) yaml_out = "{}/{}.yml".format(output_path, base_name) self.print_logs("Creating yml file: {} ...".format(yaml_out), log_color=LOG_COLORS.NATIVE) ryaml = YAML() ryaml.preserve_quotes = True with open(self.input, 'r') as yf: yaml_obj = ryaml.load(yf) script_obj = yaml_obj if self.file_type == 'integration': script_obj = yaml_obj['script'] if 'image' in yaml_obj: del yaml_obj['image'] if 'detaileddescription' in yaml_obj: del yaml_obj['detaileddescription'] script_obj['script'] = SingleQuotedScalarString('') code_type = script_obj['type'] if code_type == TYPE_PWSH and not yaml_obj.get('fromversion'): self.print_logs("Setting fromversion for PowerShell to: 5.5.0", log_color=LOG_COLORS.NATIVE) yaml_obj['fromversion'] = "5.5.0" with open(yaml_out, 'w') as yf: ryaml.dump(yaml_obj, yf) # check if there is a README and if found, set found_readme to True found_readme = False if self.readme: yml_readme = os.path.splitext(self.input)[0] + '_README.md' readme = output_path + '/README.md' if os.path.exists(yml_readme): found_readme = True self.print_logs(f"Copying {readme} to {readme}", log_color=LOG_COLORS.NATIVE) shutil.copy(yml_readme, readme) else: # open an empty file with open(readme, 'w'): pass # Python code formatting and dev env setup if code_type == TYPE_PYTHON: if self.basic_fmt: self.print_logs("Running autopep8 on file: {} ...".format(code_file), log_color=LOG_COLORS.NATIVE) try: subprocess.call(["autopep8", "-i", "--max-line-length", "130", code_file]) except FileNotFoundError: self.print_logs("autopep8 skipped! It doesn't seem you have autopep8 installed.\n" "Make sure to install it with: pip install autopep8.\n" "Then run: autopep8 -i {}".format(code_file), LOG_COLORS.YELLOW) if self.pipenv: if self.basic_fmt: self.print_logs("Running isort on file: {} ...".format(code_file), LOG_COLORS.NATIVE) try: subprocess.call(["isort", code_file]) except FileNotFoundError: self.print_logs("isort skipped! It doesn't seem you have isort installed.\n" "Make sure to install it with: pip install isort.\n" "Then run: isort {}".format(code_file), LOG_COLORS.YELLOW) self.print_logs("Detecting python version and setting up pipenv files ...", log_color=LOG_COLORS.NATIVE) docker = get_all_docker_images(script_obj)[0] py_ver = get_python_version(docker, self.config.log_verbose) pip_env_dir = get_pipenv_dir(py_ver, self.config.envs_dirs_base) self.print_logs("Copying pipenv files from: {}".format(pip_env_dir), log_color=LOG_COLORS.NATIVE) shutil.copy("{}/Pipfile".format(pip_env_dir), output_path) shutil.copy("{}/Pipfile.lock".format(pip_env_dir), output_path) env = os.environ.copy() env["PIPENV_IGNORE_VIRTUALENVS"] = "1" try: subprocess.call(["pipenv", "install", "--dev"], cwd=output_path, env=env) self.print_logs("Installing all py requirements from docker: [{}] into pipenv".format(docker), LOG_COLORS.NATIVE) requirements = get_pip_requirements(docker) fp = tempfile.NamedTemporaryFile(delete=False) fp.write(requirements.encode('utf-8')) fp.close() try: subprocess.check_call(["pipenv", "install", "-r", fp.name], cwd=output_path, env=env) except Exception: self.print_logs("Failed installing requirements in pipenv.\n " "Please try installing manually after extract ends\n", LOG_COLORS.RED) os.unlink(fp.name) self.print_logs("Installing flake8 for linting", log_color=LOG_COLORS.NATIVE) subprocess.call(["pipenv", "install", "--dev", "flake8"], cwd=output_path, env=env) except FileNotFoundError as err: self.print_logs("pipenv install skipped! It doesn't seem you have pipenv installed.\n" "Make sure to install it with: pip3 install pipenv.\n" f"Then run in the package dir: pipenv install --dev\n.Err: {err}", LOG_COLORS.YELLOW) arg_path = os.path.relpath(output_path) self.print_logs("\nCompleted: setting up package: {}\n".format(arg_path), LOG_COLORS.GREEN) next_steps: str = "Next steps: \n" \ "* Install additional py packages for unit testing (if needed): cd {};" \ " pipenv install <package>\n".format(arg_path) if code_type == TYPE_PYTHON else '' next_steps += "* Create unit tests\n" \ "* Check linting and unit tests by running: demisto-sdk lint -i {}\n".format(arg_path) next_steps += "* When ready, remove from git the old yml and/or README and add the new package:\n" \ " git rm {}\n".format(self.input) if found_readme: next_steps += " git rm {}\n".format(os.path.splitext(self.input)[0] + '_README.md') next_steps += " git add {}\n".format(arg_path) self.print_logs(next_steps, log_color=LOG_COLORS.NATIVE) else: self.print_logs("Skipping pipenv and requirements installation - Note: no Pipfile will be created", log_color=LOG_COLORS.YELLOW) self.print_logs(f"Finished splitting the yml file - you can find the split results here: {output_path}", log_color=LOG_COLORS.GREEN) return 0
def extract_to_package_format(self) -> int: """Extracts the self.yml_path into several files according to the Demisto standard of the package format. Returns: int. status code for the operation. """ print("Starting migration of: {} to dir: {}".format( self.yml_path, self.dest_path)) arg_path = self.dest_path output_path = os.path.abspath(self.dest_path) os.makedirs(output_path, exist_ok=True) base_name = os.path.basename(output_path) yml_type = self.get_yml_type() code_file = "{}/{}.py".format(output_path, base_name) self.extract_code(code_file) self.extract_image("{}/{}_image.png".format(output_path, base_name)) self.extract_long_description("{}/{}_description.md".format( output_path, base_name)) yaml_out = "{}/{}.yml".format(output_path, base_name) print("Creating yml file: {} ...".format(yaml_out)) ryaml = YAML() ryaml.preserve_quotes = True with open(self.yml_path, 'r') as yf: yaml_obj = ryaml.load(yf) script_obj = yaml_obj if yml_type == INTEGRATION: script_obj = yaml_obj['script'] del yaml_obj['image'] if 'detaileddescription' in yaml_obj: del yaml_obj['detaileddescription'] if script_obj['type'] != 'python': print( 'Script is not of type "python". Found type: {}. Nothing to do.' .format(script_obj['type'])) return 1 script_obj['script'] = SingleQuotedScalarString('') with open(yaml_out, 'w') as yf: ryaml.dump(yaml_obj, yf) print("Running autopep8 on file: {} ...".format(code_file)) try: subprocess.call( ["autopep8", "-i", "--max-line-length", "130", code_file]) except FileNotFoundError: print_color( "autopep8 skipped! It doesn't seem you have autopep8 installed.\n" "Make sure to install it with: pip install autopep8.\n" "Then run: autopep8 -i {}".format(code_file), LOG_COLORS.YELLOW) print("Detecting python version and setting up pipenv files ...") docker = get_docker_images(script_obj)[0] py_ver = get_python_version(docker, self.config.log_verbose) pip_env_dir = get_pipenv_dir(py_ver, self.config.envs_dirs_base) print("Copying pipenv files from: {}".format(pip_env_dir)) shutil.copy("{}/Pipfile".format(pip_env_dir), output_path) shutil.copy("{}/Pipfile.lock".format(pip_env_dir), output_path) try: subprocess.call(["pipenv", "install", "--dev"], cwd=output_path) print( "Installing all py requirements from docker: [{}] into pipenv". format(docker)) requirements = subprocess.check_output( [ "docker", "run", "--rm", docker, "pip", "freeze", "--disable-pip-version-check" ], universal_newlines=True, stderr=subprocess.DEVNULL).strip() fp = tempfile.NamedTemporaryFile(delete=False) fp.write(requirements.encode('utf-8')) fp.close() try: subprocess.check_call(["pipenv", "install", "-r", fp.name], cwd=output_path) except Exception: print_color( "Failed installing requirements in pipenv.\n " "Please try installing manually after extract ends\n", LOG_COLORS.RED) os.unlink(fp.name) print("Installing flake8 for linting") subprocess.call(["pipenv", "install", "--dev", "flake8"], cwd=output_path) except FileNotFoundError: print_color( "pipenv install skipped! It doesn't seem you have pipenv installed.\n" "Make sure to install it with: pip3 install pipenv.\n" "Then run in the package dir: pipenv install --dev", LOG_COLORS.YELLOW) # check if there is a changelog yml_changelog = os.path.splitext(self.yml_path)[0] + '_CHANGELOG.md' changelog = arg_path + '/CHANGELOG.md' if os.path.exists(yml_changelog): shutil.copy(yml_changelog, changelog) else: with open(changelog, 'wt', encoding='utf-8') as changelog_file: changelog_file.write("## [Unreleased]\n-\n") print_color("\nCompleted: setting up package: {}\n".format(arg_path), LOG_COLORS.GREEN) print( "Next steps: \n", "* Install additional py packages for unit testing (if needed): cd {}; pipenv install <package>\n" .format(arg_path), "* Create unit tests\n", "* Check linting and unit tests by running: ./Tests/scripts/pkg_dev_test_tasks.py -d {}\n" .format(arg_path), "* When ready rm from git the source yml and add the new package:\n", " git rm {}\n".format(self.yml_path), " git add {}\n".format(arg_path), sep='') return 0