def validate_all_files(self):
"""Validate all files in the repo are in the right format."""
# go over packs
for root, dirs, _ in os.walk(PACKS_DIR):
for dir_in_dirs in dirs:
for directory in PACKS_DIRECTORIES:
for inner_root, inner_dirs, files in os.walk(os.path.join(root, dir_in_dirs, directory)):
for inner_dir in inner_dirs:
if inner_dir.startswith('.'):
continue
project_dir = os.path.join(inner_root, inner_dir)
_, file_path = get_yml_paths_in_dir(os.path.normpath(project_dir),
Errors.no_yml_file(project_dir))
if file_path:
print("Validating {}".format(file_path))
structure_validator = StructureValidator(file_path)
if not structure_validator.is_valid_scheme():
self._is_valid = False
# go over regular content entities
for directory in DIR_LIST_FOR_REGULAR_ENTETIES:
print_color('Validating {} directory:'.format(directory), LOG_COLORS.GREEN)
for root, dirs, files in os.walk(directory):
for file_name in files:
file_path = os.path.join(root, file_name)
# skipping hidden files
if not file_name.endswith('.yml'):
continue
print('Validating ' + file_name)
structure_validator = StructureValidator(file_path)
if not structure_validator.is_valid_scheme():
self._is_valid = False
# go over regular PACKAGE_SUPPORTING_DIRECTORIES entities
for directory in PACKAGE_SUPPORTING_DIRECTORIES:
for root, dirs, files in os.walk(directory):
for inner_dir in dirs:
if inner_dir.startswith('.'):
continue
project_dir = os.path.join(root, inner_dir)
_, file_path = get_yml_paths_in_dir(project_dir, Errors.no_yml_file(project_dir))
if file_path:
print('Validating ' + file_path)
structure_validator = StructureValidator(file_path)
if not structure_validator.is_valid_scheme():
self._is_valid = False
def test_upload_an_integration_directory(demisto_client_configure, mocker):
"""
Given
- An integration directory called UploadTest
When
- Uploading an integration
Then
- Ensure integration is uploaded successfully
- Ensure success upload message is printed as expected
"""
mocker.patch("builtins.print")
integration_dir_name = "UploadTest"
integration_path = f"{git_path()}/demisto_sdk/tests/test_files/Packs/DummyPack/Integrations/{integration_dir_name}"
uploader = Uploader(input=integration_path, insecure=False, verbose=False)
mocker.patch.object(uploader, 'client')
uploader.upload()
_, integration_yml_name = get_yml_paths_in_dir(integration_path)
uploaded_file_name = f'integration-{os.path.basename(integration_yml_name)}'
upload_success_message = u'{}{}{}'.format(
LOG_COLORS.GREEN,
f"Uploaded integration - '{uploaded_file_name}': successfully",
LOG_COLORS.NATIVE
)
assert print.call_args_list[3][0][0] == upload_success_message
def copy_packs_content_to_packs_bundle(self, packs):
"""
Copy content in packs to the bundle that gets zipped to 'content_packs.zip'. Preserves directory structure
except that packages inside the "Integrations" or "Scripts" directory inside a pack are flattened. Adds file
prefixes according to how server expects to ingest the files, e.g. 'integration-' is prepended to integration
yml filenames and 'script-' is prepended to script yml filenames and so on and so forth.
"""
for pack in packs:
pack_name = os.path.basename(pack)
if pack_name in self.packs_to_skip:
continue
pack_dst = os.path.join(self.packs_bundle, pack_name)
os.mkdir(pack_dst)
pack_dirs = get_child_directories(pack)
pack_files = get_child_files(pack)
# copy first level pack files over
for file_path in pack_files:
shutil.copy(
file_path,
os.path.join(pack_dst, os.path.basename(file_path)))
# handle content directories in the pack
for content_dir in pack_dirs:
dir_name = os.path.basename(content_dir)
dest_dir = os.path.join(pack_dst, dir_name)
os.mkdir(dest_dir)
if dir_name in DIR_TO_PREFIX:
packages_dirs = get_child_directories(content_dir)
for package_dir in packages_dirs:
ymls, _ = get_yml_paths_in_dir(package_dir,
error_msg='')
if not ymls or (len(ymls) == 1
and ymls[0].endswith('_unified.yml')):
msg = 'Skipping package: {} -'.format(package_dir)
if not ymls:
print_warning(
'{} No yml files found in the package directory'
.format(msg))
else:
print_warning(
'{} Only unified yml found in the package directory'
.format(msg))
continue
package_dir_name = os.path.basename(package_dir)
unifier = Unifier(package_dir, dir_name, dest_dir)
unifier.merge_script_package_to_yml()
# also copy CHANGELOG markdown files over (should only be one per package)
package_files = get_child_files(package_dir)
changelog_files = [
file_path for file_path in package_files
if 'CHANGELOG.md' in file_path
]
for md_file_path in changelog_files:
md_out_name = '{}-{}_CHANGELOG.md'.format(
DIR_TO_PREFIX.get(dir_name), package_dir_name)
shutil.copyfile(
md_file_path,
os.path.join(dest_dir, md_out_name))
else:
self.copy_dir_files(content_dir, dest_dir)
def copy_dir_yml(self, dir_path, bundle):
"""
Copy the yml files inside a directory to a bundle.
:param dir_path: source directory
:param bundle: destination bundle
:return: None
"""
scan_files, _ = get_yml_paths_in_dir(dir_path, error_msg='')
content_files = 0
dir_name = os.path.basename(dir_path)
copy_func = self.copy_playbook_yml if dir_name in [
'Playbooks', 'TestPlaybooks'
] else self.copy_content_yml
for path in scan_files:
if len(os.path.basename(path)) >= self.file_name_max_size:
self.long_file_names.append(path)
with open(path, 'r') as file_:
yml_info = yaml.safe_load(file_)
ver = yml_info.get('fromversion', '0')
print(f' - processing: {ver} ({path})')
copy_func(path, os.path.join(bundle, os.path.basename(path)),
yml_info)
content_files += 1
print(f' - total files: {content_files}')
def test_upload_a_script_directory(demisto_client_configure, mocker):
"""
Given
- A script directory called DummyScript
When
- Uploading an script
Then
- Ensure script is uploaded successfully
- Ensure success upload message is printed as expected
"""
mocker.patch("builtins.print")
script_dir_name = "DummyScript"
scripts_path = f"{git_path()}/demisto_sdk/tests/test_files/Packs/DummyPack/Scripts/{script_dir_name}"
uploader = Uploader(input=scripts_path, insecure=False, verbose=False)
mocker.patch.object(uploader, 'client')
uploader.upload()
_, script_yml_name = get_yml_paths_in_dir(scripts_path)
uploaded_file_name = f'script-{os.path.basename(script_yml_name)}'
upload_success_message = u'{}{}{}'.format(
LOG_COLORS.GREEN,
f"Uploaded script - '{uploaded_file_name}': successfully",
LOG_COLORS.NATIVE
)
assert print.call_args_list[3][0][0] == upload_success_message
def copy_dir_yml(self, dir_path, bundle):
"""
Copy the yml files inside a directory to a bundle.
:param dir_path: source directory
:param bundle: destination bundle
:return: None
"""
scan_files, _ = get_yml_paths_in_dir(dir_path, error_msg='')
content_files = 0
dir_name = os.path.basename(dir_path)
for path in scan_files:
if len(os.path.basename(path)) >= self.file_name_max_size:
self.long_file_names.append(path)
ryaml = YAML()
ryaml.allow_duplicate_keys = True
with io.open(path, mode='r', encoding='utf-8') as file_:
yml_info = ryaml.load(file_)
ver = yml_info.get('fromversion', '0')
print(f' - processing: {ver} ({path})')
if dir_name in ['Playbooks', 'TestPlaybooks']:
# in TestPlaybook dir we might have scripts - all should go to test_bundle
if dir_name == 'TestPlaybooks' and os.path.basename(path).startswith('script-'):
self.copy_content_yml(path, os.path.join(bundle, os.path.basename(path)), yml_info)
self.copy_playbook_yml(path, os.path.join(bundle, os.path.basename(path)))
else:
self.copy_content_yml(path, os.path.join(bundle, os.path.basename(path)), yml_info)
content_files += 1
print(f' - total files: {content_files}')
def get_main_file_details(content_entity: str,
entity_instance_path: str) -> tuple:
"""
Returns the details of the "main" file within an entity instance.
For example: In the HelloWorld integration under Packs/HelloWorld, the main file is the yml file.
It contains all relevant ids and names for all the files under the HelloWorld integration dir.
:param content_entity: The content entity, for example Integrations
:param entity_instance_path: For example: ~/.../content/Packs/TestPack/Integrations/HelloWorld
:return: The main file id & name
"""
main_file_data: dict = dict()
main_file_path: str = str()
# Entities which contain yml files
if content_entity in (INTEGRATIONS_DIR, SCRIPTS_DIR, PLAYBOOKS_DIR,
TEST_PLAYBOOKS_DIR):
if os.path.isdir(entity_instance_path):
_, main_file_path = get_yml_paths_in_dir(entity_instance_path)
elif os.path.isfile(entity_instance_path):
main_file_path = entity_instance_path
if main_file_path:
main_file_data = get_yaml(main_file_path)
# Entities which are json files (md files are ignored - changelog/readme)
else:
if os.path.isfile(entity_instance_path) and retrieve_file_ending(
entity_instance_path) == 'json':
main_file_data = get_json(entity_instance_path)
main_id = get_entity_id_by_entity_type(main_file_data, content_entity)
main_name = get_entity_name_by_entity_type(main_file_data,
content_entity)
return main_id, main_name
def test_get_yml_paths_in_dir(self, dir_path):
yml_paths, first_yml_path = tools.get_yml_paths_in_dir(dir_path, error_msg='')
yml_paths_test = glob.glob(os.path.join(dir_path, '*yml'))
assert sorted(yml_paths) == sorted(yml_paths_test)
if yml_paths_test:
assert first_yml_path == yml_paths_test[0]
else:
assert not first_yml_path
def __init__(self,
input: str,
dir_name=INTEGRATIONS_DIR,
output: str = '',
image_prefix=DEFAULT_IMAGE_PREFIX,
force: bool = False):
directory_name = ''
# Changing relative path to current abspath fixed problem with default output file name.
if input == '.':
input = os.path.abspath(input)
for optional_dir_name in DIR_TO_PREFIX:
if optional_dir_name in input:
directory_name = optional_dir_name
if not directory_name:
print_error(
'You have failed to provide a legal file path, a legal file path '
'should contain either Integrations or Scripts directories')
self.image_prefix = image_prefix
self.package_path = input
self.use_force = force
if self.package_path.endswith(os.sep):
self.package_path = self.package_path.rstrip(os.sep)
self.dest_path = output
yml_paths, self.yml_path = get_yml_paths_in_dir(
self.package_path, Errors.no_yml_file(self.package_path))
for path in yml_paths:
# The plugin creates a unified YML file for the package.
# In case this script runs locally and there is a unified YML file in the package we need to ignore it.
# Also,
# we don't take the unified file by default because
# there might be packages that were not created by the plugin.
if 'unified' not in path and os.path.basename(
os.path.dirname(path)) not in [
SCRIPTS_DIR, INTEGRATIONS_DIR
]:
self.yml_path = path
break
self.ryaml = YAML()
self.ryaml.preserve_quotes = True
self.ryaml.width = 50000 # make sure long lines will not break (relevant for code section)
if self.yml_path:
with io.open(self.yml_path, 'r', encoding='utf8') as yml_file:
self.yml_data = self.ryaml.load(yml_file)
else:
self.yml_data = {}
print_error(f'No yml found in path: {self.package_path}')
# script key for scripts is a string.
# script key for integrations is a dictionary.
self.is_script_package = isinstance(self.yml_data.get('script'), str)
self.dir_name = SCRIPTS_DIR if self.is_script_package else dir_name
def __init__(
self,
input: str,
output: Optional[str] = None,
force: bool = False,
marketplace: Optional[str] = None,
):
directory_name = ''
# Changing relative path to current abspath fixed problem with default output file name.
input = os.path.abspath(input)
if not os.path.isdir(input):
print_error(UNSUPPORTED_INPUT_ERR_MSG)
sys.exit(1)
for optional_dir_name in DIR_TO_PREFIX:
if optional_dir_name in input:
directory_name = optional_dir_name
if not directory_name:
print_error(UNSUPPORTED_INPUT_ERR_MSG)
self.package_path = input
self.package_path = self.package_path.rstrip(os.sep)
self.use_force = force
self.dest_path = output
self.dir_name = ''
self.marketplace = marketplace
if marketplace:
MARKETPLACE_TAG_PARSER.marketplace = marketplace
yml_paths, self.yml_path = get_yml_paths_in_dir(
self.package_path, Errors.no_yml_file(self.package_path))
for path in yml_paths:
# The plugin creates a unified YML file for the package.
# In case this script runs locally and there is a unified YML file in the package we need to ignore it.
# Also,
# we don't take the unified file by default because
# there might be packages that were not created by the plugin.
if 'unified' not in path and os.path.basename(
os.path.dirname(path)) not in [
SCRIPTS_DIR, INTEGRATIONS_DIR
]:
self.yml_path = path
break
self.yaml = YAML_Handler(
width=50000
) # make sure long lines will not break (relevant for code section)
if self.yml_path:
with io.open(self.yml_path, 'r', encoding='utf8') as yml_file:
self.yml_data = self.yaml.load(yml_file)
else:
self.yml_data = {}
print_error(f'No yml found in path: {self.package_path}')
def create_unifieds_and_copy(self,
package_dir,
dest_dir='',
skip_dest_dir=''):
"""
For directories that have packages, aka subdirectories for each integration/script
e.g. "Integrations", "Scripts". Creates a unified yml and writes
it to the dest_dir
Arguments:
package_dir: (str)
Path to directory in which there are package subdirectories. e.g. "Integrations", "Scripts"
dest_dir: (str)
Path to destination directory to which the unified yml for a package should be written
skip_dest_dir: (str)
Path to the directory to which the unified yml for a package should be written in the
case the package is part of the skipped list
"""
dest_dir = dest_dir if dest_dir else self.content_bundle
skip_dest_dir = skip_dest_dir if skip_dest_dir else self.test_bundle
scanned_packages = glob.glob(os.path.join(package_dir, '*/'))
package_dir_name = os.path.basename(package_dir)
for package in scanned_packages:
ymls, _ = get_yml_paths_in_dir(package, error_msg='')
if not ymls or (len(ymls) == 1
and ymls[0].endswith('_unified.yml')):
msg = 'Skipping package: {} -'.format(package)
if not ymls:
print_warning(
f'{msg} No yml files found in the package directory')
else:
print_warning(
f'{msg} Only unified yml found in the package directory'
)
continue
unification_tool = Unifier(package, package_dir_name, dest_dir)
if any(package_to_skip in package
for package_to_skip in self.packages_to_skip):
# there are some packages that we don't want to include in the content zip
# for example HelloWorld integration
unification_tool = Unifier(package, package_dir_name,
skip_dest_dir)
print('skipping {}'.format(package))
if parse_version(
unification_tool.yml_data.get(
'fromversion', '0.0.0')) <= parse_version('6.0.0'):
unified_yml_paths = unification_tool.merge_script_package_to_yml(
file_name_suffix=self.file_name_suffix)
for unified_yml_path in unified_yml_paths:
self.add_from_version_to_yml(unified_yml_path)
def copy_dir_yml(self, dir_path, bundle):
"""
Copy the yml files inside a directory to a bundle.
:param dir_path: source directory
:param bundle: destination bundle
:return: None
"""
scan_files, _ = get_yml_paths_in_dir(dir_path, error_msg='')
content_files = 0
dir_name = os.path.basename(dir_path)
if scan_files:
print(f"\nStarting process for {dir_path}")
for path in scan_files:
if not self.should_process_file_to_bundle(path, bundle):
continue
new_file_path = self.add_suffix_to_file_path(
os.path.join(bundle, os.path.basename(path)))
if len(os.path.basename(path)) >= self.file_name_max_size:
self.long_file_names.append(path)
ryaml = YAML()
ryaml.allow_duplicate_keys = True
with io.open(path, mode='r', encoding='utf-8') as file_:
yml_info = ryaml.load(file_)
ver = yml_info.get('fromversion', '0')
updated_yml_info = self.add_from_version_to_yml(
yml_content=yml_info, save_yml=False)
if updated_yml_info:
yml_info = updated_yml_info
process_message = f' - processing: {path}'
if ver != '0' and ver != '':
process_message += f' - current fromversion: {ver}'
print(process_message)
if dir_name in ['Playbooks', 'TestPlaybooks']:
# in TestPlaybook dir we might have scripts - all should go to test_bundle
if dir_name == 'TestPlaybooks' and os.path.basename(
path).startswith('script-'):
self.copy_content_yml(path, new_file_path, yml_info)
self.copy_playbook_yml(path, new_file_path)
else:
self.copy_content_yml(path, new_file_path, yml_info)
content_files += 1
if content_files > 0:
print(f'Finished process - total files: {content_files}\n')
def get_script_package_data(self):
# should be static method
_, yml_path = get_yml_paths_in_dir(self.package_path, error_msg='')
if not yml_path:
raise Exception(
f'No yml files found in package path: {self.package_path}. '
'Is this really a package dir?')
code_type = get_yaml(yml_path).get('type')
unifier = Unifier(self.package_path)
code_path = unifier.get_code_file(TYPE_TO_EXTENSION[code_type])
with open(code_path, 'r') as code_file:
code = code_file.read()
return yml_path, code
def get_script_or_integration_package_data(self):
# should be static method
_, yml_path = get_yml_paths_in_dir(self.package_path, error_msg='')
if not yml_path:
raise Exception(
f'No yml files found in package path: {self.package_path}. '
'Is this really a package dir?')
if find_type(yml_path) in (FileType.SCRIPT, FileType.TEST_SCRIPT):
code_type = get_yaml(yml_path).get('type')
else:
code_type = get_yaml(yml_path).get('script', {}).get('type')
unifier = Unifier(self.package_path)
code_path = unifier.get_code_file(TYPE_TO_EXTENSION[code_type])
with io.open(code_path, 'r', encoding='utf-8') as code_file:
code = code_file.read()
return yml_path, code
def test_upload_a_script_directory(demisto_client_configure, mocker):
"""
Given
- A script directory called DummyScript
When
- Uploading an script
Then
- Ensure script is uploaded successfully
- Ensure success upload message is printed as expected
"""
mocker.patch.object(demisto_client, 'configure', return_value="object")
script_dir_name = "DummyScript"
scripts_path = f"{git_path()}/demisto_sdk/tests/test_files/Packs/DummyPack/Scripts/{script_dir_name}"
uploader = Uploader(input=scripts_path, insecure=False, verbose=False)
mocker.patch.object(uploader, 'client')
uploader.upload()
_, script_yml_name = get_yml_paths_in_dir(scripts_path)
uploaded_file_name = script_yml_name.split('/')[-1]
assert [(uploaded_file_name, FileType.SCRIPT.value)] == uploader.successfully_uploaded_files
def test_upload_an_integration_directory(demisto_client_configure, mocker):
"""
Given
- An integration directory called UploadTest
When
- Uploading an integration
Then
- Ensure integration is uploaded successfully
- Ensure success upload message is printed as expected
"""
mocker.patch.object(demisto_client, 'configure', return_value="object")
integration_dir_name = "UploadTest"
integration_path = f"{git_path()}/demisto_sdk/tests/test_files/Packs/DummyPack/Integrations/{integration_dir_name}"
uploader = Uploader(input=integration_path, insecure=False, verbose=False)
mocker.patch.object(uploader, 'client')
uploader.upload()
_, integration_yml_name = get_yml_paths_in_dir(integration_path)
integration_yml_name = integration_yml_name.split('/')[-1]
assert [(integration_yml_name, FileType.INTEGRATION.value)] == uploader.successfully_uploaded_files
def __init__(self,
project_dir: str,
no_test: bool = False,
no_pylint: bool = False,
no_flake8: bool = False,
no_mypy: bool = False,
root: bool = False,
keep_container: bool = False,
cpu_num: int = 0,
configuration: Configuration = Configuration(),
lock: threading.Lock = threading.Lock(),
no_bandit: bool = False,
no_pslint: bool = False,
requirements_3: str = '',
requirements_2: str = '',
no_vulture: bool = False):
if no_test and no_pylint and no_flake8 and no_mypy and no_bandit and no_vulture:
raise ValueError("Nothing to run as all --no-* options specified.")
self.configuration = configuration
dev_scripts_dir = os.path.join(self.configuration.sdk_env_dir,
'common', 'dev_sh_scripts')
self.run_dev_tasks_script_name = 'run_dev_tasks.sh'
self.run_dev_tasks_script_pwsh_name = 'run_dev_tasks_pwsh.sh'
self.run_mypy_script_name = 'run_mypy.sh'
self.container_setup_script_name = 'pkg_dev_container_setup.sh'
self.container_setup_script_pwsh_name = 'pkg_dev_container_setup_pwsh.sh'
self.cert_file = os.path.join(dev_scripts_dir, 'panw-cert.crt')
self.run_dev_tasks_script = os.path.join(
dev_scripts_dir, self.run_dev_tasks_script_name)
self.run_dev_tasks_script_pwsh = os.path.join(
dev_scripts_dir, self.run_dev_tasks_script_pwsh_name)
self.container_setup_script = os.path.join(
dev_scripts_dir, self.container_setup_script_name)
self.container_setup_script_pwsh = os.path.join(
dev_scripts_dir, self.container_setup_script_pwsh_name)
self.run_mypy_script = os.path.join(dev_scripts_dir,
self.run_mypy_script_name)
self.docker_login_completed = False
self.project_dir = os.path.abspath(
os.path.join(self.configuration.env_dir, project_dir))
if self.project_dir[-1] != os.sep:
self.project_dir = os.path.join(self.project_dir, '')
self.root = root
self.keep_container = keep_container
self.cpu_num = cpu_num
self.common_server_created = False
self.run_args = {
'pylint': not no_pylint,
'flake8': not no_flake8,
'mypy': not no_mypy,
'bandit': not no_bandit,
'tests': not no_test,
'pslint': not no_pslint,
'vulture': not no_vulture,
}
self.lock = lock
self.requirements_3 = requirements_3
self.requirements_2 = requirements_2
# load yaml
_, yml_path = get_yml_paths_in_dir(
self.project_dir, Errors.no_yml_file(self.project_dir))
if not yml_path:
raise ValueError(f'yml path failed for: {self.project_dir}')
print_v('Using yaml file: {}'.format(yml_path))
with open(yml_path, 'r') as yml_file:
yml_data = yaml.safe_load(yml_file)
self.script_obj = yml_data
if isinstance(self.script_obj.get('script'), dict):
self.script_obj = self.script_obj.get('script')
self.script_type = self.script_obj.get('type')
def run_dev_packages(self) -> int:
return_code = 0
# load yaml
_, yml_path = get_yml_paths_in_dir(
self.project_dir, Errors.no_yml_file(self.project_dir))
if not yml_path:
return 1
print_v('Using yaml file: {}'.format(yml_path))
with open(yml_path, 'r') as yml_file:
yml_data = yaml.safe_load(yml_file)
script_obj = yml_data
if isinstance(script_obj.get('script'), dict):
script_obj = script_obj.get('script')
script_type = script_obj.get('type')
if script_type != 'python':
if script_type == 'powershell':
# TODO powershell linting
return 0
print(
'Script is not of type "python". Found type: {}. Nothing to do.'
.format(script_type))
return 0
dockers = get_all_docker_images(script_obj)
py_num = get_python_version(dockers[0], self.log_verbose)
self.lock.acquire()
print_color(
"============ Starting process for: {} ============\n".format(
self.project_dir), LOG_COLORS.YELLOW)
if self.lock.locked():
self.lock.release()
self._setup_dev_files(py_num)
if self.run_args['flake8']:
result_val = self.run_flake8(py_num)
if result_val:
return_code = result_val
if self.run_args['mypy']:
result_val = self.run_mypy(py_num)
if result_val:
return_code = result_val
if self.run_args['bandit']:
result_val = self.run_bandit(py_num)
if result_val:
return_code = result_val
for docker in dockers:
for try_num in (1, 2):
print_v("Using docker image: {}".format(docker))
py_num = get_python_version(docker, self.log_verbose)
try:
if self.run_args['tests'] or self.run_args['pylint']:
if py_num == 2.7:
requirements = self.requirements_2
else:
requirements = self.requirements_3
docker_image_created = self._docker_image_create(
docker, requirements)
output, status_code = self._docker_run(
docker_image_created)
self.lock.acquire()
print_color(
"\n========== Running tests/pylint for: {} ========="
.format(self.project_dir), LOG_COLORS.YELLOW)
if status_code == 1:
raise subprocess.CalledProcessError(*output)
else:
print(output)
print_color(
"============ Finished process for: {} "
"with docker: {} ============\n".format(
self.project_dir, docker),
LOG_COLORS.GREEN)
if self.lock.locked():
self.lock.release()
break # all is good no need to retry
except subprocess.CalledProcessError as ex:
if ex.output:
print_color(
"=========================== ERROR IN {}==========================="
"\n{}\n".format(self.project_dir, ex.output),
LOG_COLORS.RED)
else:
print_color(
"========= Test Failed on {}, Look at the error/s above ========\n"
.format(self.project_dir), LOG_COLORS.RED)
return_code = 1
if not self.log_verbose:
sys.stderr.write(
"Need a more detailed log? try running with the -v options as so: \n{} -v\n\n"
.format(" ".join(sys.argv[:])))
if self.lock.locked():
self.lock.release()
# circle ci docker setup sometimes fails on
if try_num > 1 or not ex.output or 'read: connection reset by peer' not in ex.output:
return 2
else:
sys.stderr.write(
"Retrying as failure seems to be docker communication related...\n"
)
finally:
sys.stdout.flush()
sys.stderr.flush()
return return_code
def copy_packs_content_to_packs_bundle(self, packs):
"""
Copy content in packs to the bundle that gets zipped to 'content_packs.zip'. Preserves directory structure
except that packages inside the "Integrations" or "Scripts" directory inside a pack are flattened. Adds file
prefixes according to how server expects to ingest the files, e.g. 'integration-' is prepended to integration
yml filenames and 'script-' is prepended to script yml filenames and so on and so forth.
"""
for pack in packs:
pack_name = os.path.basename(pack)
if pack_name in self.packs_to_skip:
continue
pack_dst = os.path.join(self.packs_bundle, pack_name)
os.mkdir(pack_dst)
pack_dirs = get_child_directories(pack)
pack_files = get_child_files(pack)
# copy first level pack files over
for file_path in pack_files:
shutil.copy(
file_path,
os.path.join(pack_dst, os.path.basename(file_path)))
# handle content directories in the pack
for content_dir in pack_dirs:
dir_name = os.path.basename(content_dir)
dest_dir = os.path.join(pack_dst, dir_name)
os.mkdir(dest_dir)
if dir_name in DIR_TO_PREFIX:
packages_dirs = get_child_directories(content_dir)
if packages_dirs: # split yml files directories
for package_dir in packages_dirs:
ymls, _ = get_yml_paths_in_dir(package_dir,
error_msg='')
if not ymls or (len(ymls) == 1 and
ymls[0].endswith('_unified.yml')):
msg = f'Skipping package: {package_dir} -'
if not ymls:
print_warning(
'{} No yml files found in the package directory'
.format(msg))
else:
print_warning(
'{} Only unified yml found in the package directory'
.format(msg))
continue
unifier = Unifier(package_dir, dir_name, dest_dir)
if parse_version(
unifier.yml_data.get(
'toversion',
'99.99.99')) >= parse_version('6.0.0'):
new_file_paths = unifier.merge_script_package_to_yml(
file_name_suffix=self.file_name_suffix)
for new_file_path in new_file_paths:
self.add_from_version_to_yml(new_file_path)
non_split_yml_files = [
f for f in os.listdir(content_dir)
if os.path.isfile(os.path.join(content_dir, f)) and (
fnmatch.fnmatch(f, 'integration-*.yml')
or fnmatch.fnmatch(f, 'script-*.yml'))
]
if non_split_yml_files: # old format non split yml files
for yml_file in non_split_yml_files:
new_file_path = self.add_suffix_to_file_path(
os.path.join(dest_dir, yml_file))
shutil.copyfile(
os.path.join(content_dir, yml_file),
new_file_path)
self.add_from_version_to_yml(new_file_path)
else:
self.copy_dir_files(content_dir,
dest_dir,
is_legacy_bundle=False)
def is_context_different_in_yml(self) -> bool:
"""
Checks if there has been a corresponding change to the integration's README
when changing the context paths of an integration.
This validation might run together with is_context_change_in_readme in Integration's validation.
Returns:
True if there has been a corresponding change to README file when context is changed in integration
"""
valid = True
# disregards scripts as the structure of the files is different:
dir_path = os.path.dirname(self.file_path)
if 'Scripts' in dir_path:
return True
# Get YML file, assuming only one yml in integration
yml_file_paths = get_yml_paths_in_dir(dir_path)
# Handles case of Pack's Readme, so no YML file is found in pack.
if not yml_file_paths[0]:
return True
yml_file_path = yml_file_paths[
1] # yml_file_paths[1] should contain the first yml file found in dir
# If get_yml_paths_in_dir does not return full path, dir_path should be added to path.
if dir_path not in yml_file_path:
yml_file_path = os.path.join(dir_path, yml_file_path)
# Getting the relevant error_code:
error, missing_from_readme_error_code = Errors.readme_missing_output_context(
'', '')
error, missing_from_yml_error_code = Errors.missing_output_context(
'', '')
# Only run validation if the validation has not run with is_context_change_in_readme on integration
# so no duplicates errors will be created:
errors, ignored_errors = self._get_error_lists()
if f'{self.file_path} - [{missing_from_readme_error_code}]' in ignored_errors \
or f'{self.file_path} - [{missing_from_readme_error_code}]' in errors \
or f'{yml_file_path} - [{missing_from_yml_error_code}]' in ignored_errors \
or f'{yml_file_path} - [{missing_from_yml_error_code}]' in errors:
return False
# get YML file's content:
yml_as_dict = get_yaml(yml_file_path)
difference_context_paths = compare_context_path_in_yml_and_readme(
yml_as_dict, self.readme_content)
# Add errors to error's list
for command_name in difference_context_paths:
if difference_context_paths[command_name].get('only in yml'):
error, code = Errors.readme_missing_output_context(
command_name,
", ".join(difference_context_paths[command_name].get(
'only in yml')))
if self.handle_error(error, code, file_path=self.file_path):
valid = False
if difference_context_paths[command_name].get('only in readme'):
error, code = Errors.missing_output_context(
command_name,
", ".join(difference_context_paths[command_name].get(
'only in readme')))
if self.handle_error(error, code, file_path=yml_file_path):
valid = False
return valid
