def is_valid_default_arguments(self):
# type: () -> bool
"""Check if a reputation command (domain/email/file/ip/url)
has a default non required argument with the same name
Returns:
bool. Whether a reputation command hold a valid argument
"""
commands = self.current_file.get('script', {}).get('commands', [])
flag = True
for command in commands:
command_name = command.get('name')
if command_name in BANG_COMMAND_NAMES:
flag_found_arg = False
for arg in command.get('arguments', []):
arg_name = arg.get('name')
if arg_name == command_name:
flag_found_arg = True
if arg.get('default') is False:
self.is_valid = False
flag = False
print_error(
Errors.wrong_default_argument(
self.file_path, arg_name, command_name))
if not flag_found_arg:
print_error(
Errors.no_default_arg(self.file_path, command_name))
flag = False
if not flag:
print_error(Errors.suggest_fix(self.file_path))
return flag
def is_not_valid_display_configuration(self):
"""Validate that the display settings are not empty for non-hidden fields and for type 17 params.
Returns:
bool. Whether the display is there for non-hidden fields.
"""
configuration = self.current_file.get('configuration', [])
for configuration_param in configuration:
field_type = configuration_param['type']
is_field_hidden = configuration_param.get('hidden', False)
configuration_display = configuration_param.get('display')
# This parameter type will not use the display value.
if field_type == self.EXPIRATION_FIELD_TYPE:
if configuration_display:
print_error(
Errors.not_used_display_name(
self.file_path, configuration_param['name']))
self.is_valid = False
return True
elif not is_field_hidden and not configuration_display \
and configuration_param['name'] not in ('feedExpirationPolicy', 'feedExpirationInterval'):
print_error(
Errors.empty_display_configuration(
self.file_path, configuration_param['name']))
self.is_valid = False
return True
return False
def is_valid_param(self, param_name, param_display):
# type: (str, str) -> bool
"""Check if the given parameter has the right configuration."""
err_msgs = []
configuration = self.current_file.get('configuration', [])
for configuration_param in configuration:
configuration_param_name = configuration_param['name']
if configuration_param_name == param_name:
if configuration_param['display'] != param_display:
err_msgs.append(
Errors.wrong_display_name(param_name, param_display))
elif configuration_param.get('defaultvalue',
'') not in ('false', ''):
err_msgs.append(Errors.wrong_default_parameter(param_name))
elif configuration_param.get('required', False):
err_msgs.append(Errors.wrong_required_value(param_name))
elif configuration_param.get('type') != 8:
err_msgs.append(Errors.wrong_required_type(param_name))
if err_msgs:
print_error(
'{} Received the following error for {} validation:\n{}'.
format(self.file_path, param_name, '\n'.join(err_msgs)))
self.is_valid = False
return False
return True
def is_valid_pwsh(self) -> bool:
if self.current_file.get("type") == TYPE_PWSH:
from_version = self.current_file.get("fromversion", "0.0.0")
if not from_version or server_version_compare("5.5.0", from_version) > 0:
print_error(Errors.pwsh_wrong_version(self.file_path, from_version))
print_error(Errors.suggest_fix(self.file_path, '--from-version', '5.5.0'))
return False
return True
def _is_valid_version(self):
# type: () -> bool
"""Base is_valid_version method for files that version is their root.
Return:
True if version is valid, else False
"""
if self.current_file.get('version') != self.DEFAULT_VERSION:
print_error(
Errors.wrong_version(self.file_path, self.DEFAULT_VERSION))
print_error(Errors.suggest_fix(self.file_path))
self.is_valid = False
return False
return True
def validate_all_files(self):
"""Validate all files in the repo are in the right format."""
# go over packs
for root, dirs, _ in os.walk(PACKS_DIR):
for dir_in_dirs in dirs:
for directory in PACKS_DIRECTORIES:
for inner_root, inner_dirs, files in os.walk(os.path.join(root, dir_in_dirs, directory)):
for inner_dir in inner_dirs:
if inner_dir.startswith('.'):
continue
project_dir = os.path.join(inner_root, inner_dir)
_, file_path = get_yml_paths_in_dir(os.path.normpath(project_dir),
Errors.no_yml_file(project_dir))
if file_path:
print("Validating {}".format(file_path))
structure_validator = StructureValidator(file_path)
if not structure_validator.is_valid_scheme():
self._is_valid = False
# go over regular content entities
for directory in DIR_LIST_FOR_REGULAR_ENTETIES:
print_color('Validating {} directory:'.format(directory), LOG_COLORS.GREEN)
for root, dirs, files in os.walk(directory):
for file_name in files:
file_path = os.path.join(root, file_name)
# skipping hidden files
if not file_name.endswith('.yml'):
continue
print('Validating ' + file_name)
structure_validator = StructureValidator(file_path)
if not structure_validator.is_valid_scheme():
self._is_valid = False
# go over regular PACKAGE_SUPPORTING_DIRECTORIES entities
for directory in PACKAGE_SUPPORTING_DIRECTORIES:
for root, dirs, files in os.walk(directory):
for inner_dir in dirs:
if inner_dir.startswith('.'):
continue
project_dir = os.path.join(root, inner_dir)
_, file_path = get_yml_paths_in_dir(project_dir, Errors.no_yml_file(project_dir))
if file_path:
print('Validating ' + file_path)
structure_validator = StructureValidator(file_path)
if not structure_validator.is_valid_scheme():
self._is_valid = False
def is_valid_feed(self):
# type: () -> bool
valid_from_version = valid_feed_params = True
if self.current_file.get("script", {}).get("feed"):
from_version = self.current_file.get("fromversion", "0.0.0")
if not from_version or server_version_compare(
"5.5.0", from_version) == 1:
print_error(
Errors.feed_wrong_from_version(self.file_path,
from_version))
print_error(
Errors.suggest_fix(self.file_path, '--from-version',
'5.5.0'))
valid_from_version = False
valid_feed_params = self.all_feed_params_exist()
return valid_from_version and valid_feed_params
def get_common_server_python(self) -> bool:
"""Getting common server python in not exists changes self.common_server_created to True if needed.
Returns:
bool. True if exists/created, else False
"""
# If not CommonServerPython is dir
if not os.path.isfile(
os.path.join(self.project_dir,
self.common_server_target_path)):
# Get file from git
try:
res = requests.get(self.common_server_remote_path,
verify=False)
with open(
os.path.join(self.project_dir,
self.common_server_target_path),
"w+") as f:
f.write(res.text)
self.common_server_created = True
except requests.exceptions.RequestException:
print_error(
Errors.no_common_server_python(
self.common_server_remote_path))
return False
return True
def is_docker_image_valid(self):
if not self.is_docker_image_latest_tag():
print_error(
Errors.not_latest_docker(self.file_path, self.docker_image_tag,
self.docker_image_latest_tag))
self.is_valid = False
return self.is_valid
def is_valid_version(self):
# type: () -> bool
if self.current_file.get('commonfields',
{}).get('version') != self.DEFAULT_VERSION:
print_error(Errors.wrong_version(self.file_path))
return False
return True
def is_valid_fromversion_on_modified(self):
# type: () -> bool
"""Check that the fromversion property was not changed on existing Content files.
Returns:
(bool): Whether the files' fromversion as been modified or not.
"""
if not self.old_file:
return True
from_version_new = self.current_file.get(
"fromversion") or self.current_file.get("fromVersion")
from_version_old = self.old_file.get(
"fromversion") or self.old_file.get("fromVersion")
# if in old file there was no fromversion ,format command will add from version key with 1.0.0
if not from_version_old and from_version_new == OLD_FILE_DEFAULT_1_FROMVERSION:
return True
if from_version_old != from_version_new:
print_error(Errors.from_version_modified(self.file_path))
self.is_valid = False
return False
return True
def is_valid_scheme(self):
# type: () -> bool
"""Validate the file scheme according to the scheme we have saved in SCHEMAS_PATH.
Returns:
bool. Whether the scheme is valid on self.file_path.
"""
if self.scheme_name in [None, 'image', 'readme', 'changelog']:
return True
# ignore reputations.json
if checked_type(self.file_path,
JSON_ALL_REPUTATIONS_INDICATOR_TYPES_REGEXES):
return True
try:
# disabling massages of level INFO and beneath of pykwalify such as: INFO:pykwalify.core:validation.valid
log = logging.getLogger('pykwalify.core')
log.setLevel(logging.WARNING)
path = os.path.normpath(
os.path.join(__file__, "..", "..", self.SCHEMAS_PATH,
'{}.yml'.format(self.scheme_name)))
core = Core(source_file=self.file_path, schema_files=[path])
core.validate(raise_exception=True)
except Exception as err:
try:
print_error(self.parse_error_msg(err))
print_error(Errors.suggest_fix(self.file_path))
except Exception:
print_error('Failed: {} failed.\nin {}'.format(
self.file_path, str(err)))
self.is_valid = False
return False
return True
def is_changed_context_path(self):
# type: () -> bool
"""Check if a context path as been changed.
Returns:
bool. Whether a context path as been changed.
"""
current_command_to_context_paths = self._get_command_to_context_paths(
self.current_file)
old_command_to_context_paths = self._get_command_to_context_paths(
self.old_file)
# if old integration command has no outputs, no change of context will occur.
if not old_command_to_context_paths:
return False
# if new integration command has no outputs, and old one does, a change of context will occur.
if not current_command_to_context_paths and old_command_to_context_paths:
return True
for old_command, old_context_paths in old_command_to_context_paths.items(
):
if old_command in current_command_to_context_paths.keys():
if not self._is_sub_set(
current_command_to_context_paths[old_command],
old_context_paths):
print_error(
Errors.breaking_backwards_command(
self.file_path, old_command))
self.is_valid = False
return True
return False
def _has_beta_param(self):
# type: () -> bool
"""Checks that integration has 'beta' field with value set to true"""
beta = self.current_file.get('beta', False)
if not beta:
print_error(Errors.beta_field_not_found(self.file_path))
return False
return True
def _name_has_no_beta_substring(self):
# type: () -> bool
"""Checks that 'name' field dose not include the substring 'beta'"""
name = self.current_file.get('name', '')
if 'beta' in name.lower():
print_error(Errors.beta_in_name(self.file_path))
return False
return True
def _is_display_contains_beta(self):
# type: () -> bool
"""Checks that 'display' field includes the substring 'beta'"""
display = self.current_file.get('display', '')
if 'beta' not in display.lower():
print_error(Errors.no_beta_in_display(self.file_path))
return False
return True
def is_valid_category(self):
# type: () -> bool
"""Check that the integration category is in the schema."""
category = self.current_file.get('category', None)
if category not in INTEGRATION_CATEGORIES:
self.is_valid = False
print_error(Errors.wrong_category(self.file_path, category))
return False
return True
def _id_has_no_beta_substring(self):
# type: () -> bool
"""Checks that 'id' field dose not include the substring 'beta'"""
common_fields = self.current_file.get('commonfields', {})
integration_id = common_fields.get('id', '')
if 'beta' in integration_id.lower():
print_error(Errors.beta_in_id(self.file_path))
return False
return True
def is_context_path_changed(self):
# type: () -> bool
"""Check if the context path as been changed."""
current_context = [output['contextPath'] for output in self.current_file.get('outputs', [])]
old_context = [output['contextPath'] for output in self.old_file.get('outputs', [])]
if not self._is_sub_set(current_context, old_context):
print_error(Errors.breaking_backwards_context(self.file_path))
return True
return False
def is_arg_changed(self):
# type: () -> bool
"""Check if the argument has been changed."""
current_args = [arg['name'] for arg in self.current_file.get('args', [])]
old_args = [arg['name'] for arg in self.old_file.get('args', [])]
if not self._is_sub_set(current_args, old_args):
print_error(Errors.breaking_backwards_arg_changed(self.file_path))
return True
return False
def is_valid_subtype(self):
"""Validate that the subtype is python2 or python3."""
type_ = self.current_file.get('type')
if type_ == 'python':
subtype = self.current_file.get('subtype')
if subtype not in PYTHON_SUBTYPES:
print_error(Errors.wrong_subtype(self.file_path))
return False
return True
def is_valid_version(self):
# type: () -> bool
"""Return if version is valid.
Returns:
True if version is valid, else False.
"""
if self.current_file.get('layout', {}).get('version') != self.DEFAULT_VERSION:
print_error(Errors.wrong_version(self.file_path, self.DEFAULT_VERSION))
return False
return True
def is_changed_subtype(self):
"""Validate that the subtype was not changed."""
type_ = self.current_file.get('type')
if type_ == 'python':
subtype = self.current_file.get('subtype')
if self.old_file:
old_subtype = self.old_file.get('subtype', "")
if old_subtype and old_subtype != subtype:
print_error(Errors.breaking_backwards_subtype(self.file_path))
return True
return False
def is_valid_feed(self):
# type: () -> bool
if self.current_file.get("script", {}).get("feed"):
from_version = self.current_file.get("fromversion", "0.0.0")
if not from_version or server_version_compare(
"5.5.0", from_version) == 1:
print_error(
Errors.feed_wrong_from_version(self.file_path,
from_version))
return False
return True
def is_valid_name(self):
# type: () -> bool
if not is_v2_file(self.current_file):
return True
else:
name = self.current_file.get('name')
correct_name = "V2"
if not name.endswith(correct_name):
print_error(Errors.invalid_v2_script_name(self.file_path))
return False
return True
def is_valid_display_name(self):
# type: () -> bool
if not is_v2_file(self.current_file):
return True
else:
display_name = self.current_file.get('display')
correct_name = " v2"
if not display_name.endswith(correct_name):
print_error(Errors.invalid_v2_integration_name(self.file_path))
return False
return True
def is_valid_file_path(self):
"""Returns is valid filepath exists.
Can be only if file_type or scheme_name exists (runs from init)
Returns:
True if valid file path else False
"""
is_valid_path = bool(self.scheme_name or self.file_type)
if not is_valid_path:
print_error(Errors.invalid_file_path(self.file_path))
return is_valid_path
def is_added_required_args(self):
"""Check if required arg were added."""
current_args_to_required = self._get_arg_to_required_dict(self.current_file)
old_args_to_required = self._get_arg_to_required_dict(self.old_file)
for arg, required in current_args_to_required.items():
if required:
if (arg not in old_args_to_required) or \
(arg in old_args_to_required and required != old_args_to_required[arg]):
print_error(Errors.added_required_fields(self.file_path, arg))
return True
return False
def is_added_required_fields(self):
# type: () -> bool
"""Check if required field were added."""
current_field_to_required = self._get_field_to_required_dict(
self.current_file)
old_field_to_required = self._get_field_to_required_dict(self.old_file)
is_added_required = False
for field, required in current_field_to_required.items():
if field in old_field_to_required.keys():
# if required is True and old_field is False.
if required and required != old_field_to_required[field]:
print_error(
Errors.added_required_fields(self.file_path, field))
self.is_valid = False
is_added_required = True
# if required is True but no old field.
elif required:
print_error(Errors.added_required_fields(
self.file_path, field))
self.is_valid = False
is_added_required = True
return is_added_required
def is_valid_version(self):
# type: () -> bool
"""Validate that the reputations file as version of -1."""
is_valid = True
internal_version = self.current_file.get('version')
if internal_version != self.DEFAULT_VERSION:
object_id = self.current_file.get('id')
print_error(
Errors.wrong_version_reputations(self.file_path, object_id,
self.DEFAULT_VERSION))
is_valid = False
return is_valid