def _check_should_run_pkg(self, pkg_dir: str) -> bool:
"""Checks if there is a difference in the package before this Lint run and after it.
Args:
pkg_dir: The package directory to check.
Returns:
bool. True if there is a difference and False otherwise.
"""
# get the current branch name.
current_branch = run_command(f"git rev-parse --abbrev-ref HEAD")
# This will return a list of all files that changed up until the last commit (not including any changes
# which were made but not yet committed).
changes_from_last_commit_vs_master = run_command(
f"git diff origin/master...{current_branch}")
# This will check if any changes were made to the files in the package (pkg_dir) but are yet to be committed.
changes_since_last_commit = run_command(
f"git diff --name-only -- {pkg_dir}")
# if the package is in the list of changed files or if any files within the package were changed
# but not yet committed, return True
if pkg_dir in changes_from_last_commit_vs_master or len(
changes_since_last_commit) > 0:
return True
# if no changes were made to the package - return False.
return False
def get_modified_and_added_files(self, tag='origin/master'):
"""Get lists of the modified and added files in your branch according to the git diff output.
Args:
tag (string): String of git tag used to update modified files
Returns:
(modified_files, added_files). Tuple of sets.
"""
# Two dots is the default in git diff, it will compare with the last known commit as the base
# Three dots will compare with the last known shared commit as the base
compare_type = '.' if 'master' in tag else ''
all_changed_files_string = run_command(
'git diff --name-status {tag}..{compare_type}refs/heads/{branch}'.
format(tag=tag, branch=self.branch_name,
compare_type=compare_type))
modified_files, added_files, _, old_format_files = self.get_modified_files(
all_changed_files_string,
tag=tag,
print_ignored_files=self.print_ignored_files)
if not self.is_circle:
files_string = run_command(
'git diff --name-status --no-merges HEAD')
nc_modified_files, nc_added_files, nc_deleted_files, nc_old_format_files = self.get_modified_files(
files_string, print_ignored_files=self.print_ignored_files)
all_changed_files_string = run_command(
'git diff --name-status {}'.format(tag))
modified_files_from_tag, added_files_from_tag, _, _ = \
self.get_modified_files(all_changed_files_string,
print_ignored_files=self.print_ignored_files)
if self.file_path:
if F'M\t{self.file_path}' in files_string:
modified_files = {self.file_path}
added_files = set()
else:
modified_files = set()
added_files = {self.file_path}
return modified_files, added_files, set(), set()
old_format_files = old_format_files.union(nc_old_format_files)
modified_files = modified_files.union(
modified_files_from_tag.intersection(nc_modified_files))
added_files = added_files.union(
added_files_from_tag.intersection(nc_added_files))
modified_files = modified_files - set(nc_deleted_files)
added_files = added_files - set(nc_modified_files) - set(
nc_deleted_files)
packs = self.get_packs(modified_files, added_files)
return modified_files, added_files, old_format_files, packs
def get_all_diff_text_files(self, branch_name, is_circle):
"""
Get all new/modified text files that need to be searched for secrets
:param branch_name: current branch being worked on
:param is_circle: boolean to check if being ran from circle
:return: list: list of text files
"""
changed_files_string = run_command("git diff --name-status origin/master...{}".format(branch_name)) \
if is_circle else run_command("git diff --name-status --no-merges HEAD")
return list(self.get_diff_text_files(changed_files_string))
def update_object_in_id_set(obj_id, obj_data, file_path, instances_set):
change_string = run_command("git diff HEAD {0}".format(file_path))
is_added_from_version = True if re.search(r'\+fromversion: .*',
change_string) else False
is_added_to_version = True if re.search(r'\+toversion: .*',
change_string) else False
file_to_version = get_to_version(file_path)
file_from_version = get_from_version(file_path)
updated = False
for instance in instances_set:
instance_id = instance.keys()[0]
integration_to_version = instance[instance_id].get(
'toversion', '99.99.99')
integration_from_version = instance[instance_id].get(
'fromversion', '0.0.0')
if obj_id == instance_id:
if is_added_from_version or (not is_added_from_version
and file_from_version
== integration_from_version):
if is_added_to_version or (not is_added_to_version
and file_to_version
== integration_to_version):
instance[obj_id] = obj_data[obj_id]
updated = True
break
if not updated:
# in case we didn't found then we need to create one
add_new_object_to_id_set(obj_id, obj_data, instances_set)
def run_bandit(self, py_num) -> int:
"""Run bandit
Args:
py_num: The python version in use
Returns:
int. 0 on successful bandit run, 1 otherwise.
"""
lint_files = self._get_lint_files()
python_exe = 'python2' if py_num < 3 else 'python3'
output = run_command(' '.join(
[python_exe, '-m', 'bandit', '-lll', '-iii', '-q', lint_files]),
cwd=self.project_dir)
self.lock.acquire()
print("========= Running bandit on: {} ===============".format(
lint_files))
print_v('Using: {} to run bandit'.format(python_exe))
if len(output) == 0:
print_color("bandit completed for: {}\n".format(lint_files),
LOG_COLORS.GREEN)
if self.lock.locked():
self.lock.release()
return 0
else:
print_error(output)
if self.lock.locked():
self.lock.release()
return 1
def run_mypy(self, py_num) -> int:
"""Runs mypy
Args:
py_num: The python version in use
Returns:
int. 0 on successful mypy run, 1 otherwise.
"""
self.get_common_server_python()
lint_files = self._get_lint_files()
sys.stdout.flush()
script_path = os.path.abspath(
os.path.join(self.configuration.sdk_env_dir, self.run_mypy_script))
output = run_command(' '.join(
['bash', script_path, str(py_num), lint_files]),
cwd=self.project_dir)
self.lock.acquire()
print(
"========= Running mypy on: {} ===============".format(lint_files))
if 'Success: no issues found in 1 source file' in output:
print(output)
print_color("mypy completed for: {}\n".format(lint_files),
LOG_COLORS.GREEN)
self.remove_common_server_python()
if self.lock.locked():
self.lock.release()
return 0
else:
print_error(output)
self.remove_common_server_python()
if self.lock.locked():
self.lock.release()
return 1
def run_flake8(self, py_num) -> int:
"""Runs flake8
Args:
py_num (int): The python version in use
Returns:
int. 0 if flake8 is successful, 1 otherwise.
"""
lint_files = self._get_lint_files()
python_exe = 'python2' if py_num < 3 else 'python3'
print_v('Using: {} to run flake8'.format(python_exe))
output = run_command(f'{python_exe} -m flake8 {self.project_dir}',
cwd=self.configuration.env_dir)
self.lock.acquire()
print("\n========= Running flake8 on: {}===============".format(
lint_files))
if len(output) == 0:
print_color("flake8 completed for: {}\n".format(lint_files),
LOG_COLORS.GREEN)
if self.lock.locked():
self.lock.release()
return 0
else:
print_error(output)
if self.lock.locked():
self.lock.release()
return 1
def get_current_working_branch() -> str:
branches = run_command('git branch')
branch_name_reg = re.search(r'\* (.*)', branches)
if branch_name_reg:
return branch_name_reg.group(1)
return ''
def get_secrets(self, branch_name, is_circle):
secrets_found = {}
# make sure not in middle of merge
if not run_command('git rev-parse -q --verify MERGE_HEAD'):
secrets_file_paths = self.get_all_diff_text_files(
branch_name, is_circle)
secrets_found = self.search_potential_secrets(secrets_file_paths)
if secrets_found:
secrets_found_string = 'Secrets were found in the following files:'
for file_name in secrets_found:
secrets_found_string += ('\n\nIn File: ' + file_name +
'\n')
secrets_found_string += '\nThe following expressions were marked as secrets: \n'
secrets_found_string += json.dumps(
secrets_found[file_name], indent=4)
if not is_circle:
secrets_found_string += '\n\nRemove or whitelist secrets in order to proceed, then re-commit\n'
else:
secrets_found_string += '\n\nThe secrets were exposed in public repository,' \
' remove the files asap and report it.\n'
secrets_found_string += 'For more information about whitelisting visit: ' \
'https://github.com/demisto/internal-content/tree/master/documentation/secrets'
print_error(secrets_found_string)
return secrets_found
def _docker_run(self, docker_image):
workdir = '/devwork' # this is setup in CONTAINER_SETUP_SCRIPT
pylint_files = os.path.basename(self._get_lint_files())
run_params = [
'docker', 'create', '-w', workdir, '-e',
'PYLINT_FILES={}'.format(pylint_files)
]
if not self.root:
run_params.extend(['-u', '{}:4000'.format(os.getuid())])
if not self.run_args['tests']:
run_params.extend(['-e', 'PYTEST_SKIP=1'])
if not self.run_args['pylint']:
run_params.extend(['-e', 'PYLINT_SKIP=1'])
run_params.extend(['-e', 'CPU_NUM={}'.format(self.cpu_num)])
run_params.extend(['-e', 'CI={}'.format(os.getenv("CI", "false"))])
run_params.extend([
docker_image, 'sh', './{}'.format(self.run_dev_tasks_script_name)
])
output = run_command(' '.join(run_params))
container_id = output.strip()
try:
output = output + '\n' + run_command(' '.join([
'docker', 'cp', self.project_dir + '/.',
container_id + ':' + workdir
]),
exit_on_error=False)
output = output + '\n' + run_command(' '.join([
'docker', 'cp', self.run_dev_tasks_script,
container_id + ':' + workdir
]),
exit_on_error=False)
output = output + '\n' + subprocess.check_output(
['docker', 'start', '-a', container_id],
stderr=subprocess.STDOUT,
universal_newlines=True)
return output, 0
finally:
if not self.keep_container:
run_command(f'docker rm {container_id}')
else:
print("Test container [{}] was left available".format(
container_id))
def is_release_branch():
# type: () -> bool
"""Check if we are working on a release branch.
Returns:
(bool): is release branch
"""
diff_string_config_yml = run_command(
"git diff origin/master .circleci/config.yml")
if re.search(r'[+-][ ]+CONTENT_VERSION: ".*', diff_string_config_yml):
return True
return False
def get_master_diff(self):
"""Gets difference between current branch and origin/master
git diff with the --unified=100 option means that if there exists a
difference between origin/master and current branch, the output will have at most 100
lines of context.
Returns:
str. empty string if no changes made or no origin/master branch, otherwise full difference context.
"""
return run_command(
F'git diff --unified=100 '
F'origin/master {self.release_notes_path}') # type: str
def get_changed_files(from_branch: str = 'master',
filter_results: Callable = None):
temp_files = run_command(f'git diff --name-status {from_branch}').split(
'\n')
files: List = []
for file in temp_files:
if file:
temp_file_data = {'status': file[0]}
if file.lower().startswith('r'):
file = file.split('\t')
temp_file_data['name'] = file[2]
else:
temp_file_data['name'] = file[2:]
files.append(temp_file_data)
if filter_results:
filter(filter_results, files)
return files
def __init__(self,
is_backward_check=True,
prev_ver='origin/master',
use_git=False,
is_circle=False,
print_ignored_files=False,
validate_conf_json=True,
validate_id_set=False,
configuration=Configuration()):
self.branch_name = ''
self.use_git = use_git
if self.use_git:
print('Using git')
branches = run_command('git branch')
branch_name_reg = re.search(r'\* (.*)', branches)
self.branch_name = branch_name_reg.group(1)
print(f'Running validation on branch {self.branch_name}')
self.prev_ver = prev_ver
if not self.prev_ver:
# validate against master if no version was provided
self.prev_ver = 'origin/master'
self._is_valid = True
self.configuration = configuration
self.is_backward_check = is_backward_check
self.is_circle = is_circle
self.print_ignored_files = print_ignored_files
self.validate_conf_json = validate_conf_json
self.validate_id_set = validate_id_set
if self.validate_conf_json:
self.conf_json_validator = ConfJsonValidator()
if self.validate_id_set:
self.id_set_validator = IDSetValidator(
is_circle=self.is_circle, configuration=self.configuration)
def _docker_image_create(self, docker_base_image, requirements):
"""Create the docker image with dev dependencies. Will check if already existing.
Uses a hash of the requirements to determine the image tag
Arguments:
docker_base_image (string): docker image to use as base for installing dev deps
requirements (string): requirements doc
Returns:
string. image name to use
"""
if ':' not in docker_base_image:
docker_base_image += ':latest'
with open(self.container_setup_script, "rb") as f:
setup_script_data = f.read()
md5 = hashlib.md5(requirements.encode('utf-8') +
setup_script_data).hexdigest()
target_image = 'devtest' + docker_base_image + '-' + md5
lock_file = ".lock-" + target_image.replace("/", "-")
try:
if (time.time() - os.path.getctime(lock_file)) > (60 * 5):
print("{}: Deleting old lock file: {}".format(
datetime.now(), lock_file))
os.remove(lock_file)
except Exception as ex:
print_v(
"Failed check and delete for lock file: {}. Error: {}".format(
lock_file, ex))
wait_print = True
for x in range(60):
images_ls = run_command(' '.join([
'docker', 'image', 'ls', '--format',
'{{.Repository}}:{{.Tag}}', target_image
])).strip()
if images_ls == target_image:
print('{}: Using already existing docker image: {}'.format(
datetime.now(), target_image))
return target_image
if wait_print:
print(
"{}: Existing image: {} not found will obtain lock file or wait for image"
.format(datetime.now(), target_image))
wait_print = False
print_v("Trying to obtain lock file: " + lock_file)
try:
f = open(lock_file, "x")
f.close()
print("{}: Obtained lock file: {}".format(
datetime.now(), lock_file))
break
except Exception as ex:
print_v("Failed getting lock. Will wait {}".format(str(ex)))
time.sleep(5)
try:
# try doing a pull
try:
print("{}: Trying to pull image: {}".format(
datetime.now(), target_image))
pull_res = subprocess.check_output(
['docker', 'pull', target_image],
stderr=subprocess.STDOUT,
universal_newlines=True)
print("Pull succeeded with output: {}".format(pull_res))
return target_image
except subprocess.CalledProcessError as cpe:
print_v(
"Failed docker pull (will create image) with status: {}. Output: {}"
.format(cpe.returncode, cpe.output))
print(
"{}: Creating docker image: {} (this may take a minute or two...)"
.format(datetime.now(), target_image))
container_id = run_command(' '.join([
'docker', 'create', '-i', docker_base_image, 'sh',
'/' + self.container_setup_script_name
])).strip()
subprocess.check_call([
'docker', 'cp', self.container_setup_script,
container_id + ':/' + self.container_setup_script_name
])
print_v(
subprocess.check_output(
['docker', 'start', '-a', '-i', container_id],
input=requirements,
stderr=subprocess.STDOUT,
universal_newlines=True))
print_v(
subprocess.check_output(
['docker', 'commit', container_id, target_image],
stderr=subprocess.STDOUT,
universal_newlines=True))
print_v(
subprocess.check_output(['docker', 'rm', container_id],
stderr=subprocess.STDOUT,
universal_newlines=True))
if self._docker_login():
print("{}: Pushing image: {} to docker hub".format(
datetime.now(), target_image))
print_v(
subprocess.check_output(['docker', 'push', target_image],
stderr=subprocess.STDOUT,
universal_newlines=True))
except subprocess.CalledProcessError as err:
print(
"Failed executing command with error: {} Output: \n{}".format(
err, err.output))
raise err
finally:
try:
os.remove(lock_file)
except Exception as ex:
print("{}: Error removing file: {}".format(datetime.now(), ex))
print('{}: Done creating docker image: {}'.format(
datetime.now(), target_image))
return target_image
def update_id_set():
branches = run_command("git branch")
branch_name_reg = re.search(r"\* (.*)", branches)
branch_name = branch_name_reg.group(1)
print("Getting added files")
files_string = run_command("git diff --name-status HEAD")
second_files_string = run_command(
"git diff --name-status origin/master...{}".format(branch_name))
added_files, modified_files, added_scripts, modified_scripts = \
get_changed_files(files_string + '\n' + second_files_string)
if added_files or modified_files or added_scripts or modified_scripts:
print("Updating id_set.json")
with open('./Tests/id_set.json', 'r') as id_set_file:
try:
ids_dict = json.load(id_set_file,
object_pairs_hook=OrderedDict)
except ValueError as ex:
if "Expecting property name" in str(ex):
# if we got this error it means we have corrupted id_set.json
# usually it will happen if we merged from master and we had a conflict in id_set.json
# so we checkout the id_set.json to be exact as in master and then run update_id_set
run_command("git checkout origin/master Tests/id_set.json")
with open('./Tests/id_set.json',
'r') as id_set_file_from_master:
ids_dict = json.load(id_set_file_from_master,
object_pairs_hook=OrderedDict)
else:
raise
test_playbook_set = ids_dict['TestPlaybooks']
integration_set = ids_dict['integrations']
playbook_set = ids_dict['playbooks']
script_set = ids_dict['scripts']
if added_files:
for file_path in added_files:
if re.match(INTEGRATION_REGEX, file_path, re.IGNORECASE) or \
re.match(INTEGRATION_YML_REGEX, file_path, re.IGNORECASE):
add_new_object_to_id_set(
get_script_or_integration_id(file_path),
get_integration_data(file_path), integration_set)
print("Adding {0} to id_set".format(
get_script_or_integration_id(file_path)))
if re.match(SCRIPT_REGEX, file_path, re.IGNORECASE):
add_new_object_to_id_set(
get_script_or_integration_id(file_path),
get_script_data(file_path), script_set)
print("Adding {0} to id_set".format(
get_script_or_integration_id(file_path)))
if re.match(PLAYBOOK_REGEX, file_path, re.IGNORECASE):
add_new_object_to_id_set(collect_ids(file_path),
get_playbook_data(file_path),
playbook_set)
print("Adding {0} to id_set".format(collect_ids(file_path)))
if re.match(TEST_PLAYBOOK_REGEX, file_path, re.IGNORECASE):
add_new_object_to_id_set(collect_ids(file_path),
get_playbook_data(file_path),
test_playbook_set)
print("Adding {0} to id_set".format(collect_ids(file_path)))
if re.match(TEST_SCRIPT_REGEX, file_path, re.IGNORECASE):
add_new_object_to_id_set(
get_script_or_integration_id(file_path),
get_script_data(file_path), script_set)
print("Adding {0} to id_set".format(collect_ids(file_path)))
if modified_files:
for file_path in modified_files:
if re.match(INTEGRATION_REGEX, file_path, re.IGNORECASE) or \
re.match(INTEGRATION_YML_REGEX, file_path, re.IGNORECASE):
id = get_script_or_integration_id(file_path)
integration_data = get_integration_data(file_path)
update_object_in_id_set(id, integration_data, file_path,
integration_set)
print("updated {0} in id_set".format(id))
if re.match(SCRIPT_REGEX, file_path, re.IGNORECASE) or re.match(
TEST_SCRIPT_REGEX, file_path, re.IGNORECASE):
id = get_script_or_integration_id(file_path)
script_data = get_script_data(file_path)
update_object_in_id_set(id, script_data, file_path, script_set)
print("updated {0} in id_set".format(id))
if re.match(PLAYBOOK_REGEX, file_path, re.IGNORECASE):
id = collect_ids(file_path)
playbook_data = get_playbook_data(file_path)
update_object_in_id_set(id, playbook_data, file_path,
playbook_set)
print("updated {0} in id_set".format(id))
if re.match(TEST_PLAYBOOK_REGEX, file_path, re.IGNORECASE):
id = collect_ids(file_path)
playbook_data = get_playbook_data(file_path)
update_object_in_id_set(id, playbook_data, file_path,
test_playbook_set)
print("updated {0} in id_set".format(id))
if added_scripts:
for added_script_package in added_scripts:
unifier = Unifier(added_script_package)
yml_path, code = unifier.get_script_package_data()
add_new_object_to_id_set(
get_script_or_integration_id(yml_path),
get_script_data(yml_path, script_code=code), script_set)
print("Adding {0} to id_set".format(
get_script_or_integration_id(yml_path)))
if modified_scripts:
for modified_script_package in added_scripts:
unifier = Unifier(modified_script_package)
yml_path, code = unifier.get_script_package_data()
update_object_in_id_set(
get_script_or_integration_id(yml_path),
get_script_data(yml_path, script_code=code), yml_path,
script_set)
print("Adding {0} to id_set".format(
get_script_or_integration_id(yml_path)))
if added_files or modified_files:
new_ids_dict = OrderedDict()
# we sort each time the whole set in case someone manually changed something
# it shouldn't take too much time
new_ids_dict['scripts'] = sort(script_set)
new_ids_dict['playbooks'] = sort(playbook_set)
new_ids_dict['integrations'] = sort(integration_set)
new_ids_dict['TestPlaybooks'] = sort(test_playbook_set)
with open('./Tests/id_set.json', 'w') as id_set_file:
json.dump(new_ids_dict, id_set_file, indent=4)
print("Finished updating id_set.json")
def get_branch_name():
branches = run_command('git branch')
branch_name_reg = re.search(r'\* (.*)', branches)
branch_name = branch_name_reg.group(1)
return branch_name
