def test_get_docker_image_from_yml():
with patch.object(DockerImageValidator, '__init__',
lambda x, y, z, w: None):
# Test integration case
docker_validator = DockerImageValidator(None, None, None)
docker_validator.yml_file = get_yaml(TEST_INTEGRATION_FILE)
docker_validator.is_integration = True
docker_image = docker_validator.get_docker_image_from_yml()
assert docker_image == "demisto/pyjwt:1.0"
# Test script case
docker_validator.yml_file = get_yaml(TEST_SCRIPT_FILE)
docker_validator.is_integration = False
docker_image = docker_validator.get_docker_image_from_yml()
assert docker_image == "demisto/stix2:1.0.0.204"
def is_existing_image(self):
"""Check if the integration has an image."""
is_image_in_yml = False
is_image_in_package = False
data_dictionary = get_yaml(self.file_path)
if not data_dictionary:
return False
if data_dictionary.get('image'):
is_image_in_yml = True
if not re.match(INTEGRATION_REGEX, self.file_path, re.IGNORECASE):
package_path = os.path.dirname(self.file_path)
if is_image_in_yml:
print_error(
"You have added an image in the yml "
"file, please update the package {}".format(package_path))
return False
image_path = glob.glob(package_path + '/*.png')
if image_path:
is_image_in_package = True
if not (is_image_in_package or is_image_in_yml):
print_error(
"You have failed to add an image in the yml/package for {}".
format(self.file_path))
self._is_valid = False
return False
return True
def get_playbook_data(file_path):
playbook_data = OrderedDict()
data_dictionary = get_yaml(file_path)
id = data_dictionary.get('id', '-')
name = data_dictionary.get('name', '-')
tests = data_dictionary.get('tests')
toversion = data_dictionary.get('toversion')
fromversion = data_dictionary.get('fromversion')
implementing_scripts = get_task_ids_from_playbook('scriptName',
data_dictionary)
implementing_playbooks = get_task_ids_from_playbook(
'playbookName', data_dictionary)
command_to_integration = get_commmands_from_playbook(data_dictionary)
playbook_data['name'] = name
playbook_data['file_path'] = file_path
if toversion:
playbook_data['toversion'] = toversion
if fromversion:
playbook_data['fromversion'] = fromversion
if implementing_scripts:
playbook_data['implementing_scripts'] = implementing_scripts
if implementing_playbooks:
playbook_data['implementing_playbooks'] = implementing_playbooks
if command_to_integration:
playbook_data['command_to_integration'] = command_to_integration
if tests:
playbook_data['tests'] = tests
return {id: playbook_data}
def oversize_image(self):
"""Check if the image if over sized, bigger than IMAGE_MAX_SIZE"""
if re.match(IMAGE_REGEX, self.file_path, re.IGNORECASE):
if os.path.getsize(
self.file_path
) > self.IMAGE_MAX_SIZE: # disable-secrets-detection
print_error(
"{} has too large logo, please update the logo to be under 10kB"
.format(self.file_path))
self._is_valid = False
else:
data_dictionary = get_yaml(self.file_path)
if not data_dictionary:
return
image = data_dictionary.get('image', '')
if ((len(image) - 22) / 4.0
) * 3 > self.IMAGE_MAX_SIZE: # disable-secrets-detection
print_error(
"{} has too large logo, please update the logo to be under 10kB"
.format(self.file_path))
self._is_valid = False
def get_integration_commands(file_path):
cmd_list = []
data_dictionary = get_yaml(file_path)
commands = data_dictionary.get('script', {}).get('commands', [])
for command in commands:
cmd_list.append(command.get('name'))
return cmd_list
def __init__(self, yml_file_path, is_modified_file, is_integration):
self.is_modified_file = is_modified_file
self.is_integration = is_integration
self.yml_file = get_yaml(yml_file_path)
self.yml_docker_image = self.get_docker_image_from_yml()
self.from_version = self.yml_file.get('fromversion', '0')
self.docker_image_name, self.docker_image_tag = DockerImageValidator.parse_docker_image(
self.yml_docker_image)
self.is_latest_tag = True
self.docker_image_latest_tag = DockerImageValidator.get_docker_image_latest_tag(
self.docker_image_name, self.yml_docker_image)
self.is_valid = True
def get_script_package_data(self):
_, yml_path = get_yml_paths_in_dir(self.package_path, error_msg='')
if not yml_path:
raise Exception("No yml files found in package path: {}. "
"Is this really a package dir? If not remove it.".format(self.package_path))
code_type = get_yaml(yml_path).get('type')
unifier = Unifier(self.package_path)
code_path = unifier.get_code_file(TYPE_TO_EXTENSION[code_type])
with open(code_path, 'r') as code_file:
code = code_file.read()
return yml_path, code
def _is_py_script_or_integration(file_path):
file_yml = get_yaml(file_path)
if re.match(INTEGRATION_REGEX, file_path, re.IGNORECASE):
if file_yml.get('script', {}).get('type',
'javascript') != 'python':
return False
return True
if re.match(SCRIPT_REGEX, file_path, re.IGNORECASE):
if file_yml.get('type', 'javascript') != 'python':
return False
return True
return False
def validate_no_old_format(self, old_format_files):
""" Validate there are no files in the old format(unified yml file for the code and configuration).
Args:
old_format_files(set): file names which are in the old format.
"""
invalid_files = []
for f in old_format_files:
yaml_data = get_yaml(f)
if 'toversion' not in yaml_data: # we only fail on old format if no toversion (meaning it is latest)
invalid_files.append(f)
if invalid_files:
print_error(
'You should update the following files to the package format, for further details please visit '
'https://github.com/demisto/content/tree/master/docs/package_directory_structure. '
'The files are:\n{}'.format('\n'.join(list(invalid_files))))
self._is_valid = False
def is_valid_beta_description(self):
"""Check if beta disclaimer exists in detailed description"""
data_dictionary = get_yaml(self.file_path)
description_in_yml = data_dictionary.get('detaileddescription',
'') if data_dictionary else ''
if not re.match(BETA_INTEGRATION_REGEX, self.file_path, re.IGNORECASE):
package_path = os.path.dirname(self.file_path)
try:
md_file_path = glob.glob(
os.path.join(os.path.dirname(self.file_path),
'*_description.md'))[0]
except IndexError:
self._is_valid = False
print_error(
"No detailed description file was found in the package {}. Please add one,"
" and make sure it includes the beta disclaimer note."
"It should contain the string in constant"
"\"BETA_INTEGRATION_DISCLAIMER\"".format(package_path))
return False
with open(md_file_path) as description_file:
description = description_file.read()
if BETA_INTEGRATION_DISCLAIMER not in description:
self._is_valid = False
print_error(
"Detailed description in beta integration package {} "
"dose not contain the beta disclaimer note. "
"It should contain the string in constant"
" \"BETA_INTEGRATION_DISCLAIMER\".".format(package_path))
return False
else:
return True
elif BETA_INTEGRATION_DISCLAIMER not in description_in_yml:
self._is_valid = False
print_error("Detailed description field in beta integration {} "
"dose not contain the beta disclaimer note."
"It should contain the string in constant"
" \"BETA_INTEGRATION_DISCLAIMER\".".format(
self.file_path))
return False
return True
def get_script_data(file_path, script_code=None):
script_data = OrderedDict()
data_dictionary = get_yaml(file_path)
id = data_dictionary.get('commonfields', {}).get('id', '-')
if script_code is None:
script_code = data_dictionary.get('script', '')
name = data_dictionary.get('name', '-')
tests = data_dictionary.get('tests')
toversion = data_dictionary.get('toversion')
deprecated = data_dictionary.get('deprecated')
fromversion = data_dictionary.get('fromversion')
depends_on, command_to_integration = get_depends_on(data_dictionary)
script_executions = sorted(
list(
set(
re.findall(r"demisto.executeCommand\(['\"](\w+)['\"].*",
script_code))))
script_data['name'] = name
script_data['file_path'] = file_path
if toversion:
script_data['toversion'] = toversion
if fromversion:
script_data['fromversion'] = fromversion
if deprecated:
script_data['deprecated'] = deprecated
if depends_on:
script_data['depends_on'] = depends_on
if script_executions:
script_data['script_executions'] = script_executions
if command_to_integration:
script_data['command_to_integration'] = command_to_integration
if tests:
script_data['tests'] = tests
return {id: script_data}
def load_image_from_yml(self):
data_dictionary = get_yaml(self.file_path)
if not data_dictionary:
print_error(
"{} isn't an image file or unified integration file.".format(
self.file_path))
self._is_valid = False
image = data_dictionary.get('image', '')
if not image:
print_error("{} is a yml file but has no image field.".format(
self.file_path))
self._is_valid = False
image_data = image.split('base64,')
if image_data and len(image_data) == 2:
return image_data[1]
else:
print_error("{}'s image field isn't in base64 encoding.".format(
self.file_path))
self._is_valid = False
def process_test_playbook_path(file_path):
"""
Process a yml file in the testplyabook dir. Maybe either a script or playbook
Arguments:
file_path {string} -- path to yaml file
Returns:
pair -- first element is a playbook second is a script. each may be None
"""
print("adding {0} to id_set".format(file_path))
script = None
playbook = None
if checked_type(
file_path,
(TEST_SCRIPT_REGEX, PACKS_TEST_PLAYBOOKS_REGEX, TEST_PLAYBOOK_REGEX)):
yml_data = get_yaml(file_path)
if 'commonfields' in yml_data:
# script files contain this key
script = get_script_data(file_path)
else:
playbook = get_playbook_data(file_path)
return playbook, script
def get_integration_data(file_path):
integration_data = OrderedDict()
data_dictionary = get_yaml(file_path)
id = data_dictionary.get('commonfields', {}).get('id', '-')
name = data_dictionary.get('name', '-')
tests = data_dictionary.get('tests')
toversion = data_dictionary.get('toversion')
fromversion = data_dictionary.get('fromversion')
commands = data_dictionary.get('script', {}).get('commands', [])
cmd_list = [command.get('name') for command in commands]
integration_data['name'] = name
integration_data['file_path'] = file_path
if toversion:
integration_data['toversion'] = toversion
if fromversion:
integration_data['fromversion'] = fromversion
if cmd_list:
integration_data['commands'] = cmd_list
if tests:
integration_data['tests'] = tests
return {id: integration_data}
def is_duplicate_description(self):
"""Check if the integration has a non-duplicate description ."""
is_description_in_yml = False
is_description_in_package = False
package_path = None
md_file_path = None
if not re.match(INTEGRATION_REGEX, self.file_path, re.IGNORECASE) \
and not re.match(BETA_INTEGRATION_REGEX, self.file_path, re.IGNORECASE):
package_path = os.path.dirname(self.file_path)
try:
md_file_path = glob.glob(
os.path.join(os.path.dirname(self.file_path),
'*_description.md'))[0]
except IndexError:
print_warning(
"No detailed description file was found in the package {}."
" Consider adding one.".format(package_path))
if md_file_path:
is_description_in_package = True
data_dictionary = get_yaml(self.file_path)
if not data_dictionary:
return is_description_in_package
if data_dictionary.get('detaileddescription'):
is_description_in_yml = True
if is_description_in_package and is_description_in_yml:
self._is_valid = False
print_error(
"A description was found both in the package and in the yml, "
"please update the package {}.".format(package_path))
return False
return True