def __init__(
self,
input: str,
output: Optional[str] = None,
force: bool = False,
marketplace: Optional[str] = None,
):
directory_name = ''
# Changing relative path to current abspath fixed problem with default output file name.
input = os.path.abspath(input)
if not os.path.isdir(input):
print_error(UNSUPPORTED_INPUT_ERR_MSG)
sys.exit(1)
for optional_dir_name in DIR_TO_PREFIX:
if optional_dir_name in input:
directory_name = optional_dir_name
if not directory_name:
print_error(UNSUPPORTED_INPUT_ERR_MSG)
self.package_path = input
self.package_path = self.package_path.rstrip(os.sep)
self.use_force = force
self.dest_path = output
self.dir_name = ''
self.marketplace = marketplace
if marketplace:
MARKETPLACE_TAG_PARSER.marketplace = marketplace
yml_paths, self.yml_path = get_yml_paths_in_dir(
self.package_path, Errors.no_yml_file(self.package_path))
for path in yml_paths:
# The plugin creates a unified YML file for the package.
# In case this script runs locally and there is a unified YML file in the package we need to ignore it.
# Also,
# we don't take the unified file by default because
# there might be packages that were not created by the plugin.
if 'unified' not in path and os.path.basename(
os.path.dirname(path)) not in [
SCRIPTS_DIR, INTEGRATIONS_DIR
]:
self.yml_path = path
break
self.yaml = YAML_Handler(
width=50000
) # make sure long lines will not break (relevant for code section)
if self.yml_path:
with io.open(self.yml_path, 'r', encoding='utf8') as yml_file:
self.yml_data = self.yaml.load(yml_file)
else:
self.yml_data = {}
print_error(f'No yml found in path: {self.package_path}')
INVALID_INTEGRATION_YML_1, INVALID_INTEGRATION_YML_2,
INVALID_INTEGRATION_YML_3, INVALID_INTEGRATION_YML_4,
INVALID_LAYOUT_CONTAINER_PATH, INVALID_LAYOUT_PATH, INVALID_PLAYBOOK_PATH,
INVALID_REPUTATION_FILE, INVALID_WIDGET_PATH, LAYOUT_TARGET,
LAYOUTS_CONTAINER_TARGET, PLAYBOOK_PACK_TARGET, PLAYBOOK_TARGET,
VALID_DASHBOARD_PATH, VALID_INTEGRATION_ID_PATH,
VALID_INTEGRATION_TEST_PATH, VALID_LAYOUT_CONTAINER_PATH,
VALID_LAYOUT_PATH, VALID_PLAYBOOK_ARCSIGHT_ADD_DOMAIN_PATH,
VALID_PLAYBOOK_ID_PATH, VALID_REPUTATION_FILE, VALID_TEST_PLAYBOOK_PATH,
VALID_WIDGET_PATH, WIDGET_TARGET)
from TestSuite.json_based import JSONBased
from TestSuite.pack import Pack
from TestSuite.test_tools import ChangeCWD
json = JSON_Handler()
yaml = YAML_Handler()
class TestStructureValidator:
INPUTS_TARGETS = [
LAYOUTS_CONTAINER_TARGET,
LAYOUT_TARGET,
DASHBOARD_TARGET,
WIDGET_TARGET,
PLAYBOOK_TARGET,
INTEGRATION_TARGET,
INCIDENT_FIELD_TARGET,
PLAYBOOK_PACK_TARGET,
]
CREATED_DIRS = list() # type: List
class YAMLUnifier(ABC):
"""Interface to YAML objects that need to be unified
Attributes:
package_path (str): The directory path to the files to unify.
dest_path (str, optional): The output dir to write the unified YAML to.
use_force(bool): Forcefully overwrites the preexisting yml if one exists.
yaml(YAML_Handler): Wrapper object to handle YAML files.
yml_path(str): The YAML file path.
yml_data(dict): The YAML doucment Python object.
"""
def __init__(
self,
input: str,
output: Optional[str] = None,
force: bool = False,
):
directory_name = ''
# Changing relative path to current abspath fixed problem with default output file name.
input = os.path.abspath(input)
if not os.path.isdir(input):
print_error(UNSUPPORTED_INPUT_ERR_MSG)
sys.exit(1)
for optional_dir_name in DIR_TO_PREFIX:
if optional_dir_name in input:
directory_name = optional_dir_name
if not directory_name:
print_error(UNSUPPORTED_INPUT_ERR_MSG)
self.package_path = input
self.package_path = self.package_path.rstrip(os.sep)
self.use_force = force
self.dest_path = output
self.dir_name = ''
yml_paths, self.yml_path = get_yml_paths_in_dir(
self.package_path, Errors.no_yml_file(self.package_path))
for path in yml_paths:
# The plugin creates a unified YML file for the package.
# In case this script runs locally and there is a unified YML file in the package we need to ignore it.
# Also,
# we don't take the unified file by default because
# there might be packages that were not created by the plugin.
if 'unified' not in path and os.path.basename(
os.path.dirname(path)) not in [
SCRIPTS_DIR, INTEGRATIONS_DIR
]:
self.yml_path = path
break
self.yaml = YAML_Handler(
width=50000
) # make sure long lines will not break (relevant for code section)
if self.yml_path:
with io.open(self.yml_path, 'r', encoding='utf8') as yml_file:
self.yml_data = self.yaml.load(yml_file)
else:
self.yml_data = {}
print_error(f'No yml found in path: {self.package_path}')
@abstractmethod
def unify(self):
"""Merges the various components to create an output yml file."""
...
def _set_dest_path(
self,
file_name_suffix: Optional[str] = None,
):
"""Sets the target (destination) output path for the unified YAML, based on:
- Integration/Script directory name.
- Content item type (Integration/Script/Rule).
- Content item prefix (integration/script/parsingrule/modelingrule).
- Provided file name suffix.
Args:
file_name_suffix(str): An optional suffix to concat to the filename.
"""
package_dir_name = os.path.basename(self.package_path)
output_filename = '{}-{}.yml'.format(DIR_TO_PREFIX[self.dir_name],
package_dir_name)
if file_name_suffix:
# append suffix to output file name
output_filename = file_name_suffix.join(
os.path.splitext(output_filename))
if self.dest_path:
self.dest_path = os.path.join(self.dest_path, output_filename)
else:
self.dest_path = os.path.join(self.package_path, output_filename)
def _output_yaml(
self,
file_path: Optional[str],
file_data: dict,
):
"""Writes the YAML unified to the given path.
Checks whether the unified YAML already exists, and either fail or overwrite it forced to.
Args:
file_path(str): The file path to output the YAML to.
file_data(dict): The unified YAML contents.
"""
if os.path.isfile(
file_path) and not self.use_force: # type: ignore[arg-type]
raise ValueError(
f'Output file already exists: {self.dest_path}.'
' Make sure to remove this file from source control'
' or rename this package (for example if it is a v2).')
with io.open(file_path, mode='w',
encoding='utf-8') as file_: # type: ignore[arg-type]
self.yaml.dump(file_data, file_)
from typing import Optional, Union
from ruamel.yaml.scanner import ScannerError
from wcmatch.pathlib import EXTGLOB, NEGATE, Path
import demisto_sdk.commands.common.content.errors as exc
from demisto_sdk.commands.common.handlers import YAML_Handler
from .dictionary_based_object import DictionaryBasedObject
yaml = YAML_Handler(width=50000)
class YAMLObject(DictionaryBasedObject):
def __init__(self, path: Union[Path, str], file_name_prefix: str = ""):
super().__init__(path=path, file_name_prefix=file_name_prefix)
@staticmethod
def _fix_path(path: Union[Path, str]):
"""Find and validate object path is valid.
Rules:
1. Path exists.
2. One of the following options:
a. Path is a file.
b. Path is directory and file with a yml/yaml suffix exists in the given directory.
3. File suffix equal "yml" or "yaml".
Returns:
Path: valid file path.
from demisto_sdk.commands.common.constants import (GENERAL_DEFAULT_FROMVERSION,
VERSION_5_5_0)
from demisto_sdk.commands.common.handlers import YAML_Handler
from demisto_sdk.commands.common.tools import (LOG_COLORS, get_dict_from_file,
get_max_version,
get_remote_file,
is_file_from_content_repo)
from demisto_sdk.commands.format.format_constants import (DEFAULT_VERSION,
ERROR_RETURN_CODE,
OLD_FILE_TYPES,
SKIP_RETURN_CODE,
SUCCESS_RETURN_CODE)
from demisto_sdk.commands.validate.validate_manager import ValidateManager
yaml = YAML_Handler(allow_duplicate_keys=True)
class BaseUpdate:
"""BaseUpdate is the base class for all format commands.
Attributes:
source_file (str): the path to the file we are updating at the moment.
output_file (str): the desired file name to save the updated version of the YML to.
relative_content_path (str): Relative content path of output path.
old_file (dict): Data of old file from content repo, if exist.
schema_path (str): Schema path of file.
from_version (str): Value of Wanted fromVersion key in file.
data (dict): Dictionary of loaded file.
file_type (str): Whether the file is yml or json.
from_version_key (str): The fromVersion key in file, different between yml and json files.
verbose (bool): Whether to print a verbose log
