def __init__(self,
file_path,
check_git=True,
old_file_path=None,
old_git_branch='master'):
self._is_valid = True
self.file_path = file_path
if check_git:
self.current_integration = get_yaml(file_path)
# The replace in the end is for Windows support
if old_file_path:
git_hub_path = os.path.join(CONTENT_GITHUB_LINK,
old_git_branch,
old_file_path).replace("\\", "/")
file_content = requests.get(git_hub_path, verify=False).content
self.old_integration = yaml.safe_load(file_content)
else:
try:
file_path_from_old_branch = os.path.join(
CONTENT_GITHUB_LINK, old_git_branch,
file_path).replace("\\", "/")
res = requests.get(file_path_from_old_branch, verify=False)
res.raise_for_status()
self.old_integration = yaml.safe_load(res.content)
except Exception as e:
print_warning(
"{}\nCould not find the old integration please make sure that you did not break "
"backward compatibility".format(str(e)))
self.old_integration = None
def is_duplicate_description(self):
"""Check if the integration has a non-duplicate description ."""
is_description_in_yml = False
is_description_in_package = False
package_path = None
md_file_path = None
if not re.match(INTEGRATION_REGEX, self.file_path, re.IGNORECASE):
package_path = os.path.dirname(self.file_path)
try:
md_file_path = glob.glob(os.path.join(os.path.dirname(self.file_path), '*.md'))[0]
except IndexError:
print_error("No description file was found in the package {}."
" Consider adding one.".format(package_path))
if md_file_path:
is_description_in_package = True
data_dictionary = get_yaml(self.file_path)
if not data_dictionary:
return is_description_in_package
if data_dictionary.get('detaileddescription'):
is_description_in_yml = True
if is_description_in_package and is_description_in_yml:
self._is_valid = False
print_error("A description was found both in the package and in the yml, "
"please update the package {}.".format(package_path))
return False
return True
def get_script_data(file_path, script_code=None):
script_data = OrderedDict()
data_dictionary = get_yaml(file_path)
id = data_dictionary.get('commonfields', {}).get('id', '-')
if script_code is None:
script_code = data_dictionary.get('script', '')
name = data_dictionary.get('name', '-')
tests = data_dictionary.get('tests')
toversion = data_dictionary.get('toversion')
deprecated = data_dictionary.get('deprecated')
fromversion = data_dictionary.get('fromversion')
depends_on, command_to_integration = get_depends_on(data_dictionary)
script_executions = sorted(list(set(re.findall(r"demisto.executeCommand\(['\"](\w+)['\"].*", script_code))))
script_data['name'] = name
script_data['file_path'] = file_path
if toversion:
script_data['toversion'] = toversion
if fromversion:
script_data['fromversion'] = fromversion
if deprecated:
script_data['deprecated'] = deprecated
if depends_on:
script_data['depends_on'] = depends_on
if script_executions:
script_data['script_executions'] = script_executions
if command_to_integration:
script_data['command_to_integration'] = command_to_integration
if tests:
script_data['tests'] = tests
return {id: script_data}
def __init__(self, file_path, check_git=True, old_file_path=None):
self._is_valid = True
self.file_path = file_path
if check_git:
self.current_integration = get_yaml(file_path)
# The replace in the end is for Windows support
if old_file_path:
git_hub_path = os.path.join(CONTENT_GITHUB_MASTER_LINK,
old_file_path).replace("\\", "/")
file_content = requests.get(git_hub_path, verify=False).content
self.old_integration = yaml.safe_load(file_content)
else:
try:
file_path_from_master = os.path.join(
CONTENT_GITHUB_MASTER_LINK,
file_path).replace("\\", "/")
self.old_integration = yaml.safe_load(
requests.get(file_path_from_master,
verify=False).content)
except Exception as e:
print(str(e))
print_error(
"Could not find the old integration please make sure that you did not break "
"backward compatibility")
self.old_integration = None
def get_integration_data(file_path):
integration_data = OrderedDict()
data_dictionary = get_yaml(file_path)
id = data_dictionary.get('commonfields', {}).get('id', '-')
name = data_dictionary.get('name', '-')
deprecated = data_dictionary.get('deprecated', False)
tests = data_dictionary.get('tests')
toversion = data_dictionary.get('toversion')
fromversion = data_dictionary.get('fromversion')
commands = data_dictionary.get('script', {}).get('commands', [])
cmd_list = [command.get('name') for command in commands]
deprecated_commands = []
for command in commands:
if command.get('deprecated', False):
deprecated_commands.append(command.get('name'))
integration_data['name'] = name
integration_data['file_path'] = file_path
if toversion:
integration_data['toversion'] = toversion
if fromversion:
integration_data['fromversion'] = fromversion
if cmd_list:
integration_data['commands'] = cmd_list
if tests:
integration_data['tests'] = tests
if deprecated:
integration_data['deprecated'] = deprecated
if deprecated_commands:
integration_data['deprecated_commands'] = deprecated_commands
return {id: integration_data}
def is_existing_image(self):
"""Check if the integration has an image."""
is_image_in_yml = False
is_image_in_package = False
data_dictionary = get_yaml(self.file_path)
if not data_dictionary:
return False
if data_dictionary.get('image'):
is_image_in_yml = True
if not re.match(INTEGRATION_REGEX, self.file_path, re.IGNORECASE):
package_path = os.path.dirname(self.file_path)
if is_image_in_yml:
print_error(
"You have added an image in the yml "
"file, please update the package {}".format(package_path))
return False
image_path = glob.glob(package_path + '/*.png')
if image_path:
is_image_in_package = True
if not (is_image_in_package or is_image_in_yml):
print_error(
"You have failed to add an image in the yml/package for {}".
format(self.file_path))
self._is_valid = False
return False
return True
def oversize_image(self):
"""Check if the image if over sized, bigger than IMAGE_MAX_SIZE"""
if re.match(IMAGE_REGEX, self.file_path, re.IGNORECASE):
if os.path.getsize(
self.file_path
) > self.IMAGE_MAX_SIZE: # disable-secrets-detection
print_error(
"{} has too large logo, please update the logo to be under 10kB"
.format(self.file_path))
self._is_valid = False
else:
data_dictionary = get_yaml(self.file_path)
if not data_dictionary:
return
image = data_dictionary.get('image', '')
if ((len(image) - 22) / 4.0
) * 3 > self.IMAGE_MAX_SIZE: # disable-secrets-detection
print_error(
"{} has too large logo, please update the logo to be under 10kB"
.format(self.file_path))
self._is_valid = False
def get_script_data(file_path, script_code=None):
script_data = OrderedDict()
data_dictionary = get_yaml(file_path)
id = data_dictionary.get('commonfields', {}).get('id', '-')
if script_code is None:
script_code = data_dictionary.get('script', '')
name = data_dictionary.get('name', '-')
tests = data_dictionary.get('tests')
toversion = data_dictionary.get('toversion')
deprecated = data_dictionary.get('deprecated')
fromversion = data_dictionary.get('fromversion')
depends_on, command_to_integration = get_depends_on(data_dictionary)
script_executions = sorted(list(set(re.findall("demisto.executeCommand\\(['\"](\w+)['\"].*", script_code))))
script_data['name'] = name
if toversion:
script_data['toversion'] = toversion
if fromversion:
script_data['fromversion'] = fromversion
if deprecated:
script_data['deprecated'] = deprecated
if depends_on:
script_data['depends_on'] = depends_on
if script_executions:
script_data['script_executions'] = script_executions
if command_to_integration:
script_data['command_to_integration'] = command_to_integration
if tests:
script_data['tests'] = tests
return {id: script_data}
def get_playbook_data(file_path):
playbook_data = OrderedDict()
data_dictionary = get_yaml(file_path)
id = data_dictionary.get('id', '-')
name = data_dictionary.get('name', '-')
tests = data_dictionary.get('tests')
toversion = data_dictionary.get('toversion')
fromversion = data_dictionary.get('fromversion')
implementing_scripts = get_task_ids_from_playbook('scriptName', data_dictionary)
implementing_playbooks = get_task_ids_from_playbook('playbookName', data_dictionary)
command_to_integration = get_commmands_from_playbook(data_dictionary)
playbook_data['name'] = name
if toversion:
playbook_data['toversion'] = toversion
if fromversion:
playbook_data['fromversion'] = fromversion
if implementing_scripts:
playbook_data['implementing_scripts'] = implementing_scripts
if implementing_playbooks:
playbook_data['implementing_playbooks'] = implementing_playbooks
if command_to_integration:
playbook_data['command_to_integration'] = command_to_integration
if tests:
playbook_data['tests'] = tests
return {id: playbook_data}
def get_playbook_data(file_path):
playbook_data = OrderedDict()
data_dictionary = get_yaml(file_path)
id = data_dictionary.get('id', '-')
name = data_dictionary.get('name', '-')
tests = data_dictionary.get('tests')
toversion = data_dictionary.get('toversion')
fromversion = data_dictionary.get('fromversion')
implementing_scripts = get_task_ids_from_playbook('scriptName', data_dictionary)
implementing_playbooks = get_task_ids_from_playbook('playbookName', data_dictionary)
command_to_integration = get_commmands_from_playbook(data_dictionary)
playbook_data['name'] = name
playbook_data['file_path'] = file_path
if toversion:
playbook_data['toversion'] = toversion
if fromversion:
playbook_data['fromversion'] = fromversion
if implementing_scripts:
playbook_data['implementing_scripts'] = implementing_scripts
if implementing_playbooks:
playbook_data['implementing_playbooks'] = implementing_playbooks
if command_to_integration:
playbook_data['command_to_integration'] = command_to_integration
if tests:
playbook_data['tests'] = tests
return {id: playbook_data}
def is_valid_version(self):
"""Validate that the version of self.file_path is -1."""
file_extension = os.path.splitext(self.file_path)[1]
version_number = -1
reputations_valid = True
layouts_valid = True
if file_extension == '.yml':
yaml_dict = get_yaml(self.file_path)
version_number = yaml_dict.get('commonfields', {}).get('version')
if not version_number: # some files like playbooks do not have commonfields key
version_number = yaml_dict.get('version')
elif file_extension == '.json':
if checked_type(self.file_path, self.VERSION_SCHEMAS):
file_name = os.path.basename(self.file_path)
json_dict = get_json(self.file_path)
if file_name == "reputations.json":
reputations_valid = self.validate_reputations_file(
json_dict)
elif re.match(LAYOUT_REGEX, self.file_path, re.IGNORECASE):
layouts_valid = self.validate_layout_file(json_dict)
else:
version_number = json_dict.get('version')
if version_number != -1 or not reputations_valid or not layouts_valid:
print_error("The version for our files should always be -1, "
"please update the file {}.".format(self.file_path))
self._is_valid = False
return self._is_valid
def get_integration_commands(file_path):
cmd_list = []
data_dictionary = get_yaml(file_path)
commands = data_dictionary.get('script', {}).get('commands', [])
for command in commands:
cmd_list.append(command.get('name'))
return cmd_list
def get_integration_commands(file_path):
cmd_list = []
data_dictionary = get_yaml(file_path)
commands = data_dictionary.get('script', {}).get('commands', [])
for command in commands:
cmd_list.append(command.get('name'))
return cmd_list
def test_get_docker_image_from_yml():
from Tests.scripts.hook_validations.docker import DockerImageValidator
with patch.object(DockerImageValidator, '__init__',
lambda x, y, z, w: None):
# Test integration case
docker_validator = DockerImageValidator(None, None, None)
docker_validator.yml_file = get_yaml(
"Tests/scripts/hook_validations/tests/tests_data/fake_integration.yml"
)
docker_validator.is_integration = True
docker_image = docker_validator.get_docker_image_from_yml()
assert docker_image == "demisto/pyjwt:1.0"
# Test script case
docker_validator.yml_file = get_yaml(
"Tests/scripts/hook_validations/tests/tests_data/fake-script.yml")
docker_validator.is_integration = False
docker_image = docker_validator.get_docker_image_from_yml()
assert docker_image == "demisto/stix2:1.0.0.204"
def __init__(self, file_path, check_git=True, old_file_path=None, old_git_branch='master'):
self._is_valid = True
self.file_path = file_path
if check_git:
self.current_integration = get_yaml(file_path)
old_integration_file = old_file_path or file_path
self.old_integration = get_remote_file(old_integration_file, old_git_branch)
def get_script_package_data(package_path):
if package_path[-1] != os.sep:
package_path = os.path.join(package_path, '')
yml_path = glob.glob(package_path + '*.yml')[0]
code_type = get_yaml(yml_path).get('type')
code_path = get_code_file(package_path, TYPE_TO_EXTENSION[code_type])
with open(code_path, 'r') as code_file:
code = code_file.read()
return yml_path, code
def __init__(self, file_path, check_git=True, old_file_path=None, old_git_branch='master'):
self.file_path = file_path
self.current_script = {}
self.old_script = {}
if check_git:
self.current_script = get_yaml(file_path)
old_script_file = old_file_path or file_path
self.old_script = get_remote_file(old_script_file, old_git_branch)
def get_script_package_data(package_path):
if package_path[-1] != os.sep:
package_path = os.path.join(package_path, '')
yml_path = glob.glob(package_path + '*.yml')[0]
code_type = get_yaml(yml_path).get('type')
code_path = get_code_file(package_path, TYPE_TO_EXTENSION[code_type])
with open(code_path, 'r') as code_file:
code = code_file.read()
return yml_path, code
def __init__(self, yml_file_path, is_modified_file):
self.is_modified_file = is_modified_file
self.yml_file = get_yaml(yml_file_path)
self.from_version = self.yml_file.get('fromversion', '0')
self.docker_image_name, self.docker_image_tag = DockerImageValidator.parse_docker_image(
self.yml_file.get('dockerimage', ''))
self.is_latest_tag = True
self.docker_image_latest_tag = DockerImageValidator.get_docker_image_latest_tag(
self.docker_image_name)
self.is_valid = True
def is_py_script_or_integration(file_path):
file_yml = get_yaml(file_path)
if checked_type(file_path, [INTEGRATION_REGEX, INTEGRATION_YML_REGEX, BETA_INTEGRATION_REGEX,
BETA_INTEGRATION_YML_REGEX]):
return file_yml.get('script', {}).get('type', '') == 'python'
if checked_type(file_path, [SCRIPT_YML_REGEX, SCRIPT_REGEX, SCRIPT_PY_REGEX]):
return file_yml.get('type', '') == 'python'
return False
def filepath_to_integration_name(integration_file_path):
"""Load an integration file and return the integration name.
Args:
integration_file_path (str): The path to an integration yml file.
Returns:
(str): The name of the integration.
"""
integration_yaml = get_yaml(integration_file_path)
integration_name = integration_yaml.get('name')
return integration_name
def get_script_package_data(package_path):
if package_path[-1] != os.sep:
package_path = os.path.join(package_path, '')
yml_files = glob.glob(package_path + '*.yml')
if not yml_files:
raise Exception("No yml files found in package path: {}. "
"Is this really a package dir? If not remove it.".format(package_path))
yml_path = yml_files[0]
code_type = get_yaml(yml_path).get('type')
code_path = get_code_file(package_path, TYPE_TO_EXTENSION[code_type])
with open(code_path, 'r') as code_file:
code = code_file.read()
return yml_path, code
def validate_no_old_format(self, old_format_files):
""" Validate there are no files in the old format(unified yml file for the code and configuration).
Args:
old_format_files(set): file names which are in the old format.
"""
invalid_files = []
for f in old_format_files:
yaml_data = get_yaml(f)
if 'toversion' not in yaml_data: # we only fail on old format if no toversion (meaning it is latest)
invalid_files.append(f)
if invalid_files:
print_error("You must update the following files to the new package format. The files are:\n{}".format(
'\n'.join(list(invalid_files))))
self._is_valid = False
def is_py_script_or_integration(file_path):
file_yml = get_yaml(file_path)
if re.match(INTEGRATION_REGEX, file_path, re.IGNORECASE):
if file_yml.get('script', {}).get('type', 'javascript') != 'python':
return False
return True
if re.match(SCRIPT_REGEX, file_path, re.IGNORECASE):
if file_yml.get('type', 'javascript') != 'python':
return False
return True
return False
def validate_no_old_format(self, old_format_files):
""" Validate there are no files in the old format(unified yml file for the code and configuration).
Args:
old_format_files(set): file names which are in the old format.
"""
invalid_files = []
for f in old_format_files:
yaml_data = get_yaml(f)
if 'toversion' not in yaml_data: # we only fail on old format if no toversion (meaning it is latest)
invalid_files.append(f)
if invalid_files:
print_error('You should update the following files to the package format, for further details please visit '
'https://github.com/demisto/content/tree/master/docs/package_directory_structure. '
'The files are:\n{}'.format('\n'.join(list(invalid_files))))
self._is_valid = False
def __init__(self, file_path, check_git=True, old_file_path=None):
self._is_valid = True
self.file_path = file_path
if check_git:
self.current_integration = get_yaml(file_path)
# The replace in the end is for Windows support
if old_file_path:
git_hub_path = os.path.join(self.CONTENT_GIT_HUB_LINK, old_file_path).replace("\\", "/")
file_content = get(git_hub_path).content
self.old_integration = yaml.load(file_content)
else:
try:
file_path_from_master = os.path.join(self.CONTENT_GIT_HUB_LINK, file_path).replace("\\", "/")
self.old_integration = yaml.load(get(file_path_from_master).content, Loader=yaml.FullLoader)
except Exception:
print_error("Could not find the old integration please make sure that you did not break "
"backward compatibility")
self.old_integration = None
def is_valid_beta_description(self):
"""Check if beta disclaimer exists in detailed description"""
data_dictionary = get_yaml(self.file_path)
description_in_yml = data_dictionary.get('detaileddescription',
'') if data_dictionary else ''
if not re.match(BETA_INTEGRATION_REGEX, self.file_path, re.IGNORECASE):
package_path = os.path.dirname(self.file_path)
try:
md_file_path = glob.glob(
os.path.join(os.path.dirname(self.file_path),
'*_description.md'))[0]
except IndexError:
self._is_valid = False
print_error(
"No detailed description file was found in the package {}. Please add one,"
" and make sure it includes the beta disclaimer note."
"It should contain the string in constant"
"\"BETA_INTEGRATION_DISCLAIMER\"".format(package_path))
return False
with open(md_file_path) as description_file:
description = description_file.read()
if BETA_INTEGRATION_DISCLAIMER not in description:
self._is_valid = False
print_error(
"Detailed description in beta integration package {} "
"dose not contain the beta disclaimer note. "
"It should contain the string in constant"
" \"BETA_INTEGRATION_DISCLAIMER\".".format(package_path))
return False
else:
return True
elif BETA_INTEGRATION_DISCLAIMER not in description_in_yml:
self._is_valid = False
print_error("Detailed description field in beta integration {} "
"dose not contain the beta disclaimer note."
"It should contain the string in constant"
" \"BETA_INTEGRATION_DISCLAIMER\".".format(
self.file_path))
return False
return True
def process_test_playbook_path(file_path):
"""
Process a yml file in the testplyabook dir. Maybe either a script or playbook
Arguments:
file_path {string} -- path to yaml file
Returns:
pair -- first element is a playbook second is a script. each may be None
"""
print("adding {0} to id_set".format(file_path))
script = None
playbook = None
if checked_type(file_path, (TEST_SCRIPT_REGEX, PACKS_TEST_PLAYBOOKS_REGEX, TEST_PLAYBOOK_REGEX)):
yml_data = get_yaml(file_path)
if 'commonfields' in yml_data:
# script files contain this key
script = get_script_data(file_path)
else:
playbook = get_playbook_data(file_path)
return playbook, script
def get_integration_data(file_path):
integration_data = OrderedDict()
data_dictionary = get_yaml(file_path)
id = data_dictionary.get('commonfields', {}).get('id', '-')
name = data_dictionary.get('name', '-')
tests = data_dictionary.get('tests')
toversion = data_dictionary.get('toversion')
fromversion = data_dictionary.get('fromversion')
commands = data_dictionary.get('script', {}).get('commands', [])
cmd_list = [command.get('name') for command in commands]
integration_data['name'] = name
if toversion:
integration_data['toversion'] = toversion
if fromversion:
integration_data['fromversion'] = fromversion
if cmd_list:
integration_data['commands'] = cmd_list
if tests:
integration_data['tests'] = tests
return {id: integration_data}
def get_integration_data(file_path):
integration_data = OrderedDict()
data_dictionary = get_yaml(file_path)
id = data_dictionary.get('commonfields', {}).get('id', '-')
name = data_dictionary.get('name', '-')
tests = data_dictionary.get('tests')
toversion = data_dictionary.get('toversion')
fromversion = data_dictionary.get('fromversion')
commands = data_dictionary.get('script', {}).get('commands', [])
cmd_list = [command.get('name') for command in commands]
integration_data['name'] = name
if toversion:
integration_data['toversion'] = toversion
if fromversion:
integration_data['fromversion'] = fromversion
if cmd_list:
integration_data['commands'] = cmd_list
if tests:
integration_data['tests'] = tests
return {id: integration_data}
def load_image_from_yml(self):
data_dictionary = get_yaml(self.file_path)
if not data_dictionary:
print_error(
"{} isn't an image file or unified integration file.".format(
self.file_path))
self._is_valid = False
image = data_dictionary.get('image', '')
if not image:
print_error("{} is a yml file but has no image field.".format(
self.file_path))
self._is_valid = False
image_data = image.split('base64,')
if image_data and len(image_data) == 2:
return image_data[1]
else:
print_error("{}'s image field isn't in base64 encoding.".format(
self.file_path))
self._is_valid = False
def get_tests(file_path):
"""Collect tests mentioned in file_path"""
data_dictionary = get_yaml(file_path)
# inject no tests to whitelist so adding values to white list will not force all tests
if data_dictionary:
return data_dictionary.get('tests', [])
def get_name(file_path):
data_dictionary = get_yaml(file_path)
if data_dictionary:
return data_dictionary.get('name', '-')
def get_tests(file_path):
"""Collect tests mentioned in file_path"""
data_dictionary = get_yaml(file_path)
# inject no tests to whitelist so adding values to white list will not force all tests
if data_dictionary:
return data_dictionary.get('tests', [])
def get_name(file_path):
data_dictionary = get_yaml(file_path)
if data_dictionary:
return data_dictionary.get('name', '-')