def _check_should_run_pkg(self, pkg_dir: str) -> bool: """Checks if there is a difference in the package before this Lint run and after it. Args: pkg_dir: The package directory to check. Returns: bool. True if there is a difference and False otherwise. """ # get the current branch name. current_branch = run_command(f"git rev-parse --abbrev-ref HEAD") # This will return a list of all files that changed up until the last commit (not including any changes # which were made but not yet committed). changes_from_last_commit_vs_master = run_command( f"git diff origin/master...{current_branch}") # This will check if any changes were made to the files in the package (pkg_dir) but are yet to be committed. changes_since_last_commit = run_command( f"git diff --name-only -- {pkg_dir}") # if the package is in the list of changed files or if any files within the package were changed # but not yet committed, return True if pkg_dir in changes_from_last_commit_vs_master or len( changes_since_last_commit) > 0: return True # if no changes were made to the package - return False. return False
def get_modified_and_added_files(self, tag='origin/master'): """Get lists of the modified and added files in your branch according to the git diff output. Args: tag (string): String of git tag used to update modified files Returns: (modified_files, added_files). Tuple of sets. """ # Two dots is the default in git diff, it will compare with the last known commit as the base # Three dots will compare with the last known shared commit as the base compare_type = '.' if 'master' in tag else '' all_changed_files_string = run_command( 'git diff --name-status {tag}..{compare_type}refs/heads/{branch}'. format(tag=tag, branch=self.branch_name, compare_type=compare_type)) modified_files, added_files, _, old_format_files = self.get_modified_files( all_changed_files_string, tag=tag, print_ignored_files=self.print_ignored_files) if not self.is_circle: files_string = run_command( 'git diff --name-status --no-merges HEAD') nc_modified_files, nc_added_files, nc_deleted_files, nc_old_format_files = self.get_modified_files( files_string, print_ignored_files=self.print_ignored_files) all_changed_files_string = run_command( 'git diff --name-status {}'.format(tag)) modified_files_from_tag, added_files_from_tag, _, _ = \ self.get_modified_files(all_changed_files_string, print_ignored_files=self.print_ignored_files) if self.file_path: if F'M\t{self.file_path}' in files_string: modified_files = {self.file_path} added_files = set() else: modified_files = set() added_files = {self.file_path} return modified_files, added_files, set(), set() old_format_files = old_format_files.union(nc_old_format_files) modified_files = modified_files.union( modified_files_from_tag.intersection(nc_modified_files)) added_files = added_files.union( added_files_from_tag.intersection(nc_added_files)) modified_files = modified_files - set(nc_deleted_files) added_files = added_files - set(nc_modified_files) - set( nc_deleted_files) packs = self.get_packs(modified_files, added_files) return modified_files, added_files, old_format_files, packs
def get_all_diff_text_files(self, branch_name, is_circle): """ Get all new/modified text files that need to be searched for secrets :param branch_name: current branch being worked on :param is_circle: boolean to check if being ran from circle :return: list: list of text files """ changed_files_string = run_command("git diff --name-status origin/master...{}".format(branch_name)) \ if is_circle else run_command("git diff --name-status --no-merges HEAD") return list(self.get_diff_text_files(changed_files_string))
def update_object_in_id_set(obj_id, obj_data, file_path, instances_set): change_string = run_command("git diff HEAD {0}".format(file_path)) is_added_from_version = True if re.search(r'\+fromversion: .*', change_string) else False is_added_to_version = True if re.search(r'\+toversion: .*', change_string) else False file_to_version = get_to_version(file_path) file_from_version = get_from_version(file_path) updated = False for instance in instances_set: instance_id = instance.keys()[0] integration_to_version = instance[instance_id].get( 'toversion', '99.99.99') integration_from_version = instance[instance_id].get( 'fromversion', '0.0.0') if obj_id == instance_id: if is_added_from_version or (not is_added_from_version and file_from_version == integration_from_version): if is_added_to_version or (not is_added_to_version and file_to_version == integration_to_version): instance[obj_id] = obj_data[obj_id] updated = True break if not updated: # in case we didn't found then we need to create one add_new_object_to_id_set(obj_id, obj_data, instances_set)
def run_bandit(self, py_num) -> int: """Run bandit Args: py_num: The python version in use Returns: int. 0 on successful bandit run, 1 otherwise. """ lint_files = self._get_lint_files() python_exe = 'python2' if py_num < 3 else 'python3' output = run_command(' '.join( [python_exe, '-m', 'bandit', '-lll', '-iii', '-q', lint_files]), cwd=self.project_dir) self.lock.acquire() print("========= Running bandit on: {} ===============".format( lint_files)) print_v('Using: {} to run bandit'.format(python_exe)) if len(output) == 0: print_color("bandit completed for: {}\n".format(lint_files), LOG_COLORS.GREEN) if self.lock.locked(): self.lock.release() return 0 else: print_error(output) if self.lock.locked(): self.lock.release() return 1
def run_mypy(self, py_num) -> int: """Runs mypy Args: py_num: The python version in use Returns: int. 0 on successful mypy run, 1 otherwise. """ self.get_common_server_python() lint_files = self._get_lint_files() sys.stdout.flush() script_path = os.path.abspath( os.path.join(self.configuration.sdk_env_dir, self.run_mypy_script)) output = run_command(' '.join( ['bash', script_path, str(py_num), lint_files]), cwd=self.project_dir) self.lock.acquire() print( "========= Running mypy on: {} ===============".format(lint_files)) if 'Success: no issues found in 1 source file' in output: print(output) print_color("mypy completed for: {}\n".format(lint_files), LOG_COLORS.GREEN) self.remove_common_server_python() if self.lock.locked(): self.lock.release() return 0 else: print_error(output) self.remove_common_server_python() if self.lock.locked(): self.lock.release() return 1
def run_flake8(self, py_num) -> int: """Runs flake8 Args: py_num (int): The python version in use Returns: int. 0 if flake8 is successful, 1 otherwise. """ lint_files = self._get_lint_files() python_exe = 'python2' if py_num < 3 else 'python3' print_v('Using: {} to run flake8'.format(python_exe)) output = run_command(f'{python_exe} -m flake8 {self.project_dir}', cwd=self.configuration.env_dir) self.lock.acquire() print("\n========= Running flake8 on: {}===============".format( lint_files)) if len(output) == 0: print_color("flake8 completed for: {}\n".format(lint_files), LOG_COLORS.GREEN) if self.lock.locked(): self.lock.release() return 0 else: print_error(output) if self.lock.locked(): self.lock.release() return 1
def get_current_working_branch() -> str: branches = run_command('git branch') branch_name_reg = re.search(r'\* (.*)', branches) if branch_name_reg: return branch_name_reg.group(1) return ''
def get_secrets(self, branch_name, is_circle): secrets_found = {} # make sure not in middle of merge if not run_command('git rev-parse -q --verify MERGE_HEAD'): secrets_file_paths = self.get_all_diff_text_files( branch_name, is_circle) secrets_found = self.search_potential_secrets(secrets_file_paths) if secrets_found: secrets_found_string = 'Secrets were found in the following files:' for file_name in secrets_found: secrets_found_string += ('\n\nIn File: ' + file_name + '\n') secrets_found_string += '\nThe following expressions were marked as secrets: \n' secrets_found_string += json.dumps( secrets_found[file_name], indent=4) if not is_circle: secrets_found_string += '\n\nRemove or whitelist secrets in order to proceed, then re-commit\n' else: secrets_found_string += '\n\nThe secrets were exposed in public repository,' \ ' remove the files asap and report it.\n' secrets_found_string += 'For more information about whitelisting visit: ' \ 'https://github.com/demisto/internal-content/tree/master/documentation/secrets' print_error(secrets_found_string) return secrets_found
def _docker_run(self, docker_image): workdir = '/devwork' # this is setup in CONTAINER_SETUP_SCRIPT pylint_files = os.path.basename(self._get_lint_files()) run_params = [ 'docker', 'create', '-w', workdir, '-e', 'PYLINT_FILES={}'.format(pylint_files) ] if not self.root: run_params.extend(['-u', '{}:4000'.format(os.getuid())]) if not self.run_args['tests']: run_params.extend(['-e', 'PYTEST_SKIP=1']) if not self.run_args['pylint']: run_params.extend(['-e', 'PYLINT_SKIP=1']) run_params.extend(['-e', 'CPU_NUM={}'.format(self.cpu_num)]) run_params.extend(['-e', 'CI={}'.format(os.getenv("CI", "false"))]) run_params.extend([ docker_image, 'sh', './{}'.format(self.run_dev_tasks_script_name) ]) output = run_command(' '.join(run_params)) container_id = output.strip() try: output = output + '\n' + run_command(' '.join([ 'docker', 'cp', self.project_dir + '/.', container_id + ':' + workdir ]), exit_on_error=False) output = output + '\n' + run_command(' '.join([ 'docker', 'cp', self.run_dev_tasks_script, container_id + ':' + workdir ]), exit_on_error=False) output = output + '\n' + subprocess.check_output( ['docker', 'start', '-a', container_id], stderr=subprocess.STDOUT, universal_newlines=True) return output, 0 finally: if not self.keep_container: run_command(f'docker rm {container_id}') else: print("Test container [{}] was left available".format( container_id))
def is_release_branch(): # type: () -> bool """Check if we are working on a release branch. Returns: (bool): is release branch """ diff_string_config_yml = run_command( "git diff origin/master .circleci/config.yml") if re.search(r'[+-][ ]+CONTENT_VERSION: ".*', diff_string_config_yml): return True return False
def get_master_diff(self): """Gets difference between current branch and origin/master git diff with the --unified=100 option means that if there exists a difference between origin/master and current branch, the output will have at most 100 lines of context. Returns: str. empty string if no changes made or no origin/master branch, otherwise full difference context. """ return run_command( F'git diff --unified=100 ' F'origin/master {self.release_notes_path}') # type: str
def get_changed_files(from_branch: str = 'master', filter_results: Callable = None): temp_files = run_command(f'git diff --name-status {from_branch}').split( '\n') files: List = [] for file in temp_files: if file: temp_file_data = {'status': file[0]} if file.lower().startswith('r'): file = file.split('\t') temp_file_data['name'] = file[2] else: temp_file_data['name'] = file[2:] files.append(temp_file_data) if filter_results: filter(filter_results, files) return files
def __init__(self, is_backward_check=True, prev_ver='origin/master', use_git=False, is_circle=False, print_ignored_files=False, validate_conf_json=True, validate_id_set=False, configuration=Configuration()): self.branch_name = '' self.use_git = use_git if self.use_git: print('Using git') branches = run_command('git branch') branch_name_reg = re.search(r'\* (.*)', branches) self.branch_name = branch_name_reg.group(1) print(f'Running validation on branch {self.branch_name}') self.prev_ver = prev_ver if not self.prev_ver: # validate against master if no version was provided self.prev_ver = 'origin/master' self._is_valid = True self.configuration = configuration self.is_backward_check = is_backward_check self.is_circle = is_circle self.print_ignored_files = print_ignored_files self.validate_conf_json = validate_conf_json self.validate_id_set = validate_id_set if self.validate_conf_json: self.conf_json_validator = ConfJsonValidator() if self.validate_id_set: self.id_set_validator = IDSetValidator( is_circle=self.is_circle, configuration=self.configuration)
def _docker_image_create(self, docker_base_image, requirements): """Create the docker image with dev dependencies. Will check if already existing. Uses a hash of the requirements to determine the image tag Arguments: docker_base_image (string): docker image to use as base for installing dev deps requirements (string): requirements doc Returns: string. image name to use """ if ':' not in docker_base_image: docker_base_image += ':latest' with open(self.container_setup_script, "rb") as f: setup_script_data = f.read() md5 = hashlib.md5(requirements.encode('utf-8') + setup_script_data).hexdigest() target_image = 'devtest' + docker_base_image + '-' + md5 lock_file = ".lock-" + target_image.replace("/", "-") try: if (time.time() - os.path.getctime(lock_file)) > (60 * 5): print("{}: Deleting old lock file: {}".format( datetime.now(), lock_file)) os.remove(lock_file) except Exception as ex: print_v( "Failed check and delete for lock file: {}. Error: {}".format( lock_file, ex)) wait_print = True for x in range(60): images_ls = run_command(' '.join([ 'docker', 'image', 'ls', '--format', '{{.Repository}}:{{.Tag}}', target_image ])).strip() if images_ls == target_image: print('{}: Using already existing docker image: {}'.format( datetime.now(), target_image)) return target_image if wait_print: print( "{}: Existing image: {} not found will obtain lock file or wait for image" .format(datetime.now(), target_image)) wait_print = False print_v("Trying to obtain lock file: " + lock_file) try: f = open(lock_file, "x") f.close() print("{}: Obtained lock file: {}".format( datetime.now(), lock_file)) break except Exception as ex: print_v("Failed getting lock. Will wait {}".format(str(ex))) time.sleep(5) try: # try doing a pull try: print("{}: Trying to pull image: {}".format( datetime.now(), target_image)) pull_res = subprocess.check_output( ['docker', 'pull', target_image], stderr=subprocess.STDOUT, universal_newlines=True) print("Pull succeeded with output: {}".format(pull_res)) return target_image except subprocess.CalledProcessError as cpe: print_v( "Failed docker pull (will create image) with status: {}. Output: {}" .format(cpe.returncode, cpe.output)) print( "{}: Creating docker image: {} (this may take a minute or two...)" .format(datetime.now(), target_image)) container_id = run_command(' '.join([ 'docker', 'create', '-i', docker_base_image, 'sh', '/' + self.container_setup_script_name ])).strip() subprocess.check_call([ 'docker', 'cp', self.container_setup_script, container_id + ':/' + self.container_setup_script_name ]) print_v( subprocess.check_output( ['docker', 'start', '-a', '-i', container_id], input=requirements, stderr=subprocess.STDOUT, universal_newlines=True)) print_v( subprocess.check_output( ['docker', 'commit', container_id, target_image], stderr=subprocess.STDOUT, universal_newlines=True)) print_v( subprocess.check_output(['docker', 'rm', container_id], stderr=subprocess.STDOUT, universal_newlines=True)) if self._docker_login(): print("{}: Pushing image: {} to docker hub".format( datetime.now(), target_image)) print_v( subprocess.check_output(['docker', 'push', target_image], stderr=subprocess.STDOUT, universal_newlines=True)) except subprocess.CalledProcessError as err: print( "Failed executing command with error: {} Output: \n{}".format( err, err.output)) raise err finally: try: os.remove(lock_file) except Exception as ex: print("{}: Error removing file: {}".format(datetime.now(), ex)) print('{}: Done creating docker image: {}'.format( datetime.now(), target_image)) return target_image
def update_id_set(): branches = run_command("git branch") branch_name_reg = re.search(r"\* (.*)", branches) branch_name = branch_name_reg.group(1) print("Getting added files") files_string = run_command("git diff --name-status HEAD") second_files_string = run_command( "git diff --name-status origin/master...{}".format(branch_name)) added_files, modified_files, added_scripts, modified_scripts = \ get_changed_files(files_string + '\n' + second_files_string) if added_files or modified_files or added_scripts or modified_scripts: print("Updating id_set.json") with open('./Tests/id_set.json', 'r') as id_set_file: try: ids_dict = json.load(id_set_file, object_pairs_hook=OrderedDict) except ValueError as ex: if "Expecting property name" in str(ex): # if we got this error it means we have corrupted id_set.json # usually it will happen if we merged from master and we had a conflict in id_set.json # so we checkout the id_set.json to be exact as in master and then run update_id_set run_command("git checkout origin/master Tests/id_set.json") with open('./Tests/id_set.json', 'r') as id_set_file_from_master: ids_dict = json.load(id_set_file_from_master, object_pairs_hook=OrderedDict) else: raise test_playbook_set = ids_dict['TestPlaybooks'] integration_set = ids_dict['integrations'] playbook_set = ids_dict['playbooks'] script_set = ids_dict['scripts'] if added_files: for file_path in added_files: if re.match(INTEGRATION_REGEX, file_path, re.IGNORECASE) or \ re.match(INTEGRATION_YML_REGEX, file_path, re.IGNORECASE): add_new_object_to_id_set( get_script_or_integration_id(file_path), get_integration_data(file_path), integration_set) print("Adding {0} to id_set".format( get_script_or_integration_id(file_path))) if re.match(SCRIPT_REGEX, file_path, re.IGNORECASE): add_new_object_to_id_set( get_script_or_integration_id(file_path), get_script_data(file_path), script_set) print("Adding {0} to id_set".format( get_script_or_integration_id(file_path))) if re.match(PLAYBOOK_REGEX, file_path, re.IGNORECASE): add_new_object_to_id_set(collect_ids(file_path), get_playbook_data(file_path), playbook_set) print("Adding {0} to id_set".format(collect_ids(file_path))) if re.match(TEST_PLAYBOOK_REGEX, file_path, re.IGNORECASE): add_new_object_to_id_set(collect_ids(file_path), get_playbook_data(file_path), test_playbook_set) print("Adding {0} to id_set".format(collect_ids(file_path))) if re.match(TEST_SCRIPT_REGEX, file_path, re.IGNORECASE): add_new_object_to_id_set( get_script_or_integration_id(file_path), get_script_data(file_path), script_set) print("Adding {0} to id_set".format(collect_ids(file_path))) if modified_files: for file_path in modified_files: if re.match(INTEGRATION_REGEX, file_path, re.IGNORECASE) or \ re.match(INTEGRATION_YML_REGEX, file_path, re.IGNORECASE): id = get_script_or_integration_id(file_path) integration_data = get_integration_data(file_path) update_object_in_id_set(id, integration_data, file_path, integration_set) print("updated {0} in id_set".format(id)) if re.match(SCRIPT_REGEX, file_path, re.IGNORECASE) or re.match( TEST_SCRIPT_REGEX, file_path, re.IGNORECASE): id = get_script_or_integration_id(file_path) script_data = get_script_data(file_path) update_object_in_id_set(id, script_data, file_path, script_set) print("updated {0} in id_set".format(id)) if re.match(PLAYBOOK_REGEX, file_path, re.IGNORECASE): id = collect_ids(file_path) playbook_data = get_playbook_data(file_path) update_object_in_id_set(id, playbook_data, file_path, playbook_set) print("updated {0} in id_set".format(id)) if re.match(TEST_PLAYBOOK_REGEX, file_path, re.IGNORECASE): id = collect_ids(file_path) playbook_data = get_playbook_data(file_path) update_object_in_id_set(id, playbook_data, file_path, test_playbook_set) print("updated {0} in id_set".format(id)) if added_scripts: for added_script_package in added_scripts: unifier = Unifier(added_script_package) yml_path, code = unifier.get_script_package_data() add_new_object_to_id_set( get_script_or_integration_id(yml_path), get_script_data(yml_path, script_code=code), script_set) print("Adding {0} to id_set".format( get_script_or_integration_id(yml_path))) if modified_scripts: for modified_script_package in added_scripts: unifier = Unifier(modified_script_package) yml_path, code = unifier.get_script_package_data() update_object_in_id_set( get_script_or_integration_id(yml_path), get_script_data(yml_path, script_code=code), yml_path, script_set) print("Adding {0} to id_set".format( get_script_or_integration_id(yml_path))) if added_files or modified_files: new_ids_dict = OrderedDict() # we sort each time the whole set in case someone manually changed something # it shouldn't take too much time new_ids_dict['scripts'] = sort(script_set) new_ids_dict['playbooks'] = sort(playbook_set) new_ids_dict['integrations'] = sort(integration_set) new_ids_dict['TestPlaybooks'] = sort(test_playbook_set) with open('./Tests/id_set.json', 'w') as id_set_file: json.dump(new_ids_dict, id_set_file, indent=4) print("Finished updating id_set.json")
def get_branch_name(): branches = run_command('git branch') branch_name_reg = re.search(r'\* (.*)', branches) branch_name = branch_name_reg.group(1) return branch_name