def create_new_directory(self, ) -> bool:
"""Creates a new directory for the integration/script/pack.
Returns:
bool. True if directory was successfully created, False otherwise.
"""
try:
os.mkdir(self.full_output_path)
except FileExistsError:
to_delete = str(
input(f"The directory {self.full_output_path} "
f"already exists.\nDo you want to overwrite it? Y/N ")
).lower()
while to_delete != 'y' and to_delete != 'n':
to_delete = str(
input(
f"Your response was invalid.\nDo you want to delete it? Y/N "
).lower())
if to_delete in ['y', 'yes']:
shutil.rmtree(path=self.full_output_path, ignore_errors=True)
os.mkdir(self.full_output_path)
else:
print_error(f"Pack not created in {self.full_output_path}")
return False
return True
def _run_query(self, playground_id: str):
"""Runs a query on Demisto instance and prints the output.
Args:
playground_id: The investigation ID of the playground.
Returns:
list. A list of the log IDs if debug mode is on, otherwise an empty list.
"""
update_entry = {'investigationId': playground_id, 'data': self.query}
ans = self.client.investigation_add_entries_sync(
update_entry=update_entry)
log_ids = []
for entry in ans:
# ans should have entries with `contents` - the readable output of the command
if entry.parent_content:
print_color('### Command:', LOG_COLORS.YELLOW)
print(entry.parent_content)
if entry.contents:
print_color('## Readable Output', LOG_COLORS.YELLOW)
if entry.type == ERROR_ENTRY_TYPE:
print_error(entry.contents + '\n')
else:
print(entry.contents + '\n')
# and entries with `file_id`s defined, that is the fileID of the debug log file
if entry.type == DEBUG_FILE_ENTRY_TYPE:
log_ids.append(entry.id)
return log_ids
def run_bandit(self, py_num) -> int:
"""Run bandit
Args:
py_num: The python version in use
Returns:
int. 0 on successful bandit run, 1 otherwise.
"""
lint_files = self._get_lint_files()
python_exe = 'python2' if py_num < 3 else 'python3'
output = run_command(' '.join(
[python_exe, '-m', 'bandit', '-lll', '-iii', '-q', lint_files]),
cwd=self.project_dir)
self.lock.acquire()
print("========= Running bandit on: {} ===============".format(
lint_files))
print_v('Using: {} to run bandit'.format(python_exe))
if len(output) == 0:
print_color("bandit completed for: {}\n".format(lint_files),
LOG_COLORS.GREEN)
if self.lock.locked():
self.lock.release()
return 0
else:
print_error(output)
if self.lock.locked():
self.lock.release()
return 1
def get_common_server_python(self) -> bool:
"""Getting common server python in not exists changes self.common_server_created to True if needed.
Returns:
bool. True if exists/created, else False
"""
# If not CommonServerPython is dir
if not os.path.isfile(
os.path.join(self.project_dir,
self.common_server_target_path)):
# Get file from git
try:
res = requests.get(self.common_server_remote_path,
verify=False)
with open(
os.path.join(self.project_dir,
self.common_server_target_path),
"w+") as f:
f.write(res.text)
self.common_server_created = True
except requests.exceptions.RequestException:
print_error(
Errors.no_common_server_python(
self.common_server_remote_path))
return False
return True
def run_flake8(self, py_num) -> int:
"""Runs flake8
Args:
py_num (int): The python version in use
Returns:
int. 0 if flake8 is successful, 1 otherwise.
"""
lint_files = self._get_lint_files()
python_exe = 'python2' if py_num < 3 else 'python3'
print_v('Using: {} to run flake8'.format(python_exe))
output = run_command(f'{python_exe} -m flake8 {self.project_dir}',
cwd=self.configuration.env_dir)
self.lock.acquire()
print("\n========= Running flake8 on: {}===============".format(
lint_files))
if len(output) == 0:
print_color("flake8 completed for: {}\n".format(lint_files),
LOG_COLORS.GREEN)
if self.lock.locked():
self.lock.release()
return 0
else:
print_error(output)
if self.lock.locked():
self.lock.release()
return 1
def run_mypy(self, py_num) -> int:
"""Runs mypy
Args:
py_num: The python version in use
Returns:
int. 0 on successful mypy run, 1 otherwise.
"""
self.get_common_server_python()
lint_files = self._get_lint_files()
sys.stdout.flush()
script_path = os.path.abspath(
os.path.join(self.configuration.sdk_env_dir, self.run_mypy_script))
output = run_command(' '.join(
['bash', script_path, str(py_num), lint_files]),
cwd=self.project_dir)
self.lock.acquire()
print(
"========= Running mypy on: {} ===============".format(lint_files))
if 'Success: no issues found in 1 source file' in output:
print(output)
print_color("mypy completed for: {}\n".format(lint_files),
LOG_COLORS.GREEN)
self.remove_common_server_python()
if self.lock.locked():
self.lock.release()
return 0
else:
print_error(output)
self.remove_common_server_python()
if self.lock.locked():
self.lock.release()
return 1
def is_valid_content_flag(self):
"""Validate that field is marked as content."""
is_valid_flag = self.current_file.get('content') is True
if not is_valid_flag:
print_error("The content key must be set to true, please update the file '{}'".format(self.file_path))
return is_valid_flag
def is_valid_system_flag(self):
"""Validate that field is not marked as system."""
is_valid_flag = self.current_file.get('system', False) is False
if not is_valid_flag:
print_error("The system key must be set to false, please update the file '{}'".format(self.file_path))
return is_valid_flag
def is_existing_image(self):
"""Check if the integration has an image."""
is_image_in_yml = False
is_image_in_package = False
data_dictionary = get_yaml(self.file_path)
if not data_dictionary:
return False
if data_dictionary.get('image'):
is_image_in_yml = True
if not re.match(INTEGRATION_REGEX, self.file_path, re.IGNORECASE):
package_path = os.path.dirname(self.file_path)
if is_image_in_yml:
print_error(
"You have added an image in the yml "
"file, please update the package {}".format(package_path))
return False
image_path = glob.glob(package_path + '/*.png')
if image_path:
is_image_in_package = True
if not (is_image_in_package or is_image_in_yml):
print_error(
"You have failed to add an image in the yml/package for {}".
format(self.file_path))
self._is_valid = False
return False
return True
def oversize_image(self):
"""Check if the image if over sized, bigger than IMAGE_MAX_SIZE"""
if re.match(IMAGE_REGEX, self.file_path, re.IGNORECASE):
if os.path.getsize(
self.file_path
) > self.IMAGE_MAX_SIZE: # disable-secrets-detection
print_error(
"{} has too large logo, please update the logo to be under 10kB"
.format(self.file_path))
self._is_valid = False
else:
data_dictionary = get_yaml(self.file_path)
if not data_dictionary:
return
image = data_dictionary.get('image', '')
if ((len(image) - 22) / 4.0
) * 3 > self.IMAGE_MAX_SIZE: # disable-secrets-detection
print_error(
"{} has too large logo, please update the logo to be under 10kB"
.format(self.file_path))
self._is_valid = False
def update_content_version(
content_ver: str = '',
path: str = './Scripts/CommonServerPython/CommonServerPython.py'):
regex = r'CONTENT_RELEASE_VERSION = .*'
if not content_ver:
try:
with open('content-descriptor.json') as file_:
descriptor = json.load(file_)
content_ver = descriptor['release']
except (FileNotFoundError, json.JSONDecodeError, KeyError):
print_error(
'Invalid descriptor file. make sure file content is a valid json with "release" key.'
)
return
try:
with open(path, 'r+') as file_:
content = file_.read()
content = re.sub(regex,
f"CONTENT_RELEASE_VERSION = '{content_ver}'",
content, re.M)
file_.seek(0)
file_.write(content)
except Exception as ex:
print_warning(f'Could not open CommonServerPython File - {ex}')
def get_secrets(self, branch_name, is_circle):
secrets_found = {}
# make sure not in middle of merge
if not run_command('git rev-parse -q --verify MERGE_HEAD'):
secrets_file_paths = self.get_all_diff_text_files(
branch_name, is_circle)
secrets_found = self.search_potential_secrets(secrets_file_paths)
if secrets_found:
secrets_found_string = 'Secrets were found in the following files:'
for file_name in secrets_found:
secrets_found_string += ('\n\nIn File: ' + file_name +
'\n')
secrets_found_string += '\nThe following expressions were marked as secrets: \n'
secrets_found_string += json.dumps(
secrets_found[file_name], indent=4)
if not is_circle:
secrets_found_string += '\n\nRemove or whitelist secrets in order to proceed, then re-commit\n'
else:
secrets_found_string += '\n\nThe secrets were exposed in public repository,' \
' remove the files asap and report it.\n'
secrets_found_string += 'For more information about whitelisting visit: ' \
'https://github.com/demisto/internal-content/tree/master/documentation/secrets'
print_error(secrets_found_string)
return secrets_found
def is_valid_in_id_set(self, file_path: str, obj_data: OrderedDict, obj_set: list):
"""Check if the file is represented correctly in the id_set
Args:
file_path (string): Path to the file.
obj_data (dict): Dictionary that holds the extracted details from the given file.
obj_set (set): The set in which the file should be located at.
Returns:
bool. Whether the file is represented correctly in the id_set or not.
"""
is_found = False
file_id = list(obj_data.keys())[0]
for checked_instance in obj_set:
checked_instance_id = list(checked_instance.keys())[0]
checked_instance_data = checked_instance[checked_instance_id]
checked_instance_toversion = checked_instance_data.get('toversion', '99.99.99')
checked_instance_fromversion = checked_instance_data.get('fromversion', '0.0.0')
obj_to_version = obj_data[file_id].get('toversion', '99.99.99')
obj_from_version = obj_data[file_id].get('fromversion', '0.0.0')
if checked_instance_id == file_id and checked_instance_toversion == obj_to_version and \
checked_instance_fromversion == obj_from_version:
is_found = True
if checked_instance_data != obj_data[file_id]:
print_error("You have failed to update id_set.json with the data of {} "
"please run `python Tests/scripts/update_id_set.py`".format(file_path))
return False
if not is_found:
print_error("You have failed to update id_set.json with the data of {} "
"please run `python Tests/scripts/update_id_set.py`".format(file_path))
return is_found
def is_docker_image_latest_tag(self):
if not self.docker_image_name or not self.docker_image_latest_tag:
# If the docker image isn't in the format we expect it to be or we failed fetching the tag
# We don't want to print any error msgs to user because they have already been printed
self.is_latest_tag = False
return self.is_latest_tag
server_version = LooseVersion(self.from_version)
# Case of a modified file with version >= 5.0.0
if self.is_modified_file and server_version >= '5.0.0':
if self.docker_image_latest_tag != self.docker_image_tag and not \
'demisto/python:1.3-alpine' == '{}:{}'.format(self.docker_image_name, self.docker_image_tag):
# If docker image name are different and if the docker image isn't the default one
self.is_latest_tag = False
# Case of an added file
elif not self.is_modified_file:
if self.docker_image_latest_tag != self.docker_image_tag:
self.is_latest_tag = False
if not self.is_latest_tag:
print_error(
'The docker image tag is not the latest, please update it.\n'
'The docker image tag in the yml file is: {}\n'
'The latest docker image tag in docker hub is: {}\n'
'You can check for the tags of {} here: https://hub.docker.com/r/{}/tags\n'
.format(self.docker_image_tag, self.docker_image_latest_tag,
self.docker_image_name, self.docker_image_name))
return self.is_latest_tag
def _get_command_to_context_paths(self, integration_json):
# type: (dict) -> dict
"""Get a dictionary command name to it's context paths.
Args:
integration_json (dict): Dictionary of the examined integration.
Returns:
dict. command name to a list of it's context paths.
"""
command_to_context_dict = {}
commands = integration_json.get('script', {}).get('commands', [])
for command in commands:
context_list = []
for output in command.get('outputs', []):
command_name = command['name']
try:
context_list.append(output['contextPath'])
except KeyError:
print_error(
'Invalid context output for command {}. Output is {}'.
format(command_name, output))
self.is_valid = False
command_to_context_dict[command['name']] = sorted(context_list)
return command_to_context_dict
def is_valid_version(self):
# type: () -> bool
if self.current_file.get("commonfields", {}).get('version') == self.DEFAULT_VERSION:
return True
self.is_valid = False
print_error(Errors.wrong_version(self.file_path))
return False
def parse_docker_image(docker_image):
"""Verify that the docker image is of demisto format & parse the name and tag
Args:
docker_image: String representation of the docker image name and tag
Returns:
The name and the tag of the docker image
"""
if docker_image:
tag = ''
image = ''
try:
image_regex = re.findall(r'(demisto\/.+)', docker_image,
re.IGNORECASE)
if image_regex:
image = image_regex[0]
if ':' in image:
image_split = image.split(':')
image = image_split[0]
tag = image_split[1]
else:
print_error(
'The docker image in your integration/script does not have a tag, please attach the '
'latest tag')
except IndexError:
print_error(
'The docker image: {} is not of format - demisto/image_name'
.format(docker_image))
return image, tag
else:
# If the yml file has no docker image we provide the default one 'demisto/python:1.3-alpine'
return 'demisto/python', '1.3-alpine'
def is_not_valid_display_configuration(self):
"""Validate that the display settings are not empty for non-hidden fields and for type 17 params.
Returns:
bool. Whether the display is there for non-hidden fields.
"""
configuration = self.current_file.get('configuration', [])
for configuration_param in configuration:
field_type = configuration_param['type']
is_field_hidden = configuration_param.get('hidden', False)
configuration_display = configuration_param.get('display')
# This parameter type will not use the display value.
if field_type == self.EXPIRATION_FIELD_TYPE:
if configuration_display:
print_error(Errors.not_used_display_name(self.file_path, configuration_param['name']))
self.is_valid = False
return True
elif not is_field_hidden and not configuration_display:
print_error(Errors.empty_display_configuration(self.file_path, configuration_param['name']))
self.is_valid = False
return True
return False
def __init__(self,
indir: str,
dir_name=INTEGRATIONS_DIR,
outdir='',
image_prefix=DEFAULT_IMAGE_PREFIX):
directory_name = ""
for optional_dir_name in DIR_TO_PREFIX:
if optional_dir_name in indir:
directory_name = optional_dir_name
if not directory_name:
print_error(
"You have failed to provide a legal file path, a legal file path "
"should contain either Integrations or Scripts directories")
self.image_prefix = image_prefix
self.package_path = indir
if self.package_path[-1] != os.sep:
self.package_path = os.path.join(self.package_path, '')
self.dir_name = dir_name
self.dest_path = outdir
self.is_ci = os.getenv('CI', False)
def is_valid_default_arguments(self):
# type: () -> bool
"""Check if a reputation command (domain/email/file/ip/url)
has a default non required argument with the same name
Returns:
bool. Whether a reputation command hold a valid argument
"""
commands = self.current_file.get('script', {}).get('commands', [])
flag = True
for command in commands:
command_name = command.get('name')
if command_name in BANG_COMMAND_NAMES:
flag_found_arg = False
for arg in command.get('arguments', []):
arg_name = arg.get('name')
if arg_name == command_name:
flag_found_arg = True
if arg.get('default') is False:
self.is_valid = False
flag = False
print_error(Errors.wrong_default_argument(self.file_path, arg_name, command_name))
if not flag_found_arg:
print_error(Errors.no_default_arg(self.file_path, command_name))
flag = False
return flag
def is_valid_release_notes_structure(self):
"""Validates that the release notes written in the correct manner.
Returns:
bool. True if release notes structure valid, False otherwise
"""
release_notes_comments = self.latest_release_notes.split('\n')
if not release_notes_comments[-1]:
release_notes_comments = release_notes_comments[:-1]
if len(release_notes_comments) == 1 and self.is_valid_one_line_comment(
release_notes_comments):
return True
elif len(release_notes_comments) <= 1:
print_error(
F'File {self.release_notes_path} is not formatted according to '
F'release notes standards.\nFix according to {self.LINK_TO_RELEASE_NOTES_STANDARD}'
)
return False
else:
if self.is_valid_one_line_comment(release_notes_comments):
release_notes_comments = release_notes_comments[1:]
if not self.is_valid_multi_line_comment(release_notes_comments):
print_error(
F'File {self.release_notes_path} is not formatted according to '
F'release notes standards.\nFix according to {self.LINK_TO_RELEASE_NOTES_STANDARD}'
)
return False
return True
def is_release_notes_changed(self):
"""Validates that a new comment was added to release notes.
Returns:
bool. True if comment was added, False otherwise.
"""
# there exists a difference between origin/master and current branch
if self.master_diff:
diff_releases = self.master_diff.split('##')
unreleased_section = diff_releases[1]
unreleased_section_lines = unreleased_section.split('\n')
adds_in_diff = 0
removes_in_diff = 0
for line in unreleased_section_lines:
if line.startswith('+'):
adds_in_diff += 1
elif line.startswith('-') and not re.match(r'- *$', line):
removes_in_diff += 1
# means that at least one new line was added
if adds_in_diff - removes_in_diff > 0:
return True
print_error(
F'No new comment has been added in the release notes file: {self.release_notes_path}'
)
return False
def is_valid_param(self, param_name, param_display):
# type: (str, str) -> bool
"""Check if the given parameter has the right configuration."""
err_msgs = []
configuration = self.current_file.get('configuration', [])
for configuration_param in configuration:
configuration_param_name = configuration_param['name']
if configuration_param_name == param_name:
if configuration_param['display'] != param_display:
err_msgs.append(
Errors.wrong_display_name(param_name, param_display))
elif configuration_param.get('defaultvalue',
'') not in ('false', ''):
err_msgs.append(Errors.wrong_default_parameter(param_name))
elif configuration_param.get('required', False):
err_msgs.append(Errors.wrong_required_value(param_name))
elif configuration_param.get('type') != 8:
err_msgs.append(Errors.wrong_required_type(param_name))
if err_msgs:
print_error(
'{} Received the following error for {} validation:\n{}'.
format(self.file_path, param_name, '\n'.join(err_msgs)))
self.is_valid = False
return False
return True
def is_valid_feed(self):
# type: () -> bool
if self.current_file.get("feed"):
from_version = self.current_file.get("fromversion", "0.0.0")
if not from_version or server_version_compare("5.5.0", from_version) == 1:
print_error(Errors.feed_wrong_from_version(self.file_path, from_version))
return False
return True
def _is_display_contains_beta(self):
# type: () -> bool
"""Checks that 'display' field includes the substring 'beta'"""
display = self.current_file.get('display', '')
if 'beta' not in display.lower():
print_error(Errors.no_beta_in_display(self.file_path))
return False
return True
def _has_beta_param(self):
# type: () -> bool
"""Checks that integration has 'beta' field with value set to true"""
beta = self.current_file.get('beta', False)
if not beta:
print_error(Errors.beta_field_not_found(self.file_path))
return False
return True
def _name_has_no_beta_substring(self):
# type: () -> bool
"""Checks that 'name' field dose not include the substring 'beta'"""
name = self.current_file.get('name', '')
if 'beta' in name.lower():
print_error(Errors.beta_in_name(self.file_path))
return False
return True
def validate_pack_unique_files(self, packs):
for pack in packs:
pack_unique_files_validator = PackUniqueFilesValidator(pack)
pack_errors = pack_unique_files_validator.validate_pack_unique_files(
)
if pack_errors:
print_error(pack_errors)
self._is_valid = False
def _id_has_no_beta_substring(self):
# type: () -> bool
"""Checks that 'id' field dose not include the substring 'beta'"""
common_fields = self.current_file.get('commonfields', {})
integration_id = common_fields.get('id', '')
if 'beta' in integration_id.lower():
print_error(Errors.beta_in_id(self.file_path))
return False
return True
def is_valid_category(self):
# type: () -> bool
"""Check that the integration category is in the schema."""
category = self.current_file.get('category', None)
if category not in INTEGRATION_CATEGORIES:
self.is_valid = False
print_error(Errors.wrong_category(self.file_path, category))
return False
return True