def get_zipped_packs_names(zip_path): """ Creates a list of dictionaries containing a pack name as key and the latest zip file path of the pack as value. Args: zip_path: path containing all the packs copied from the storage bucket Returns: A dictionary containing each pack name and it's zip path. {'Slack': 'content/packs/slack/1.3.19/slack.zip', 'qualys': 'content/packs/qualys/2.0/qualys.zip'} """ zipped_packs = {} zip_path = os.path.join(zip_path, 'packs') # directory of the packs dir_entries = os.listdir(zip_path) packs_list = [pack.name for pack in os.scandir(PACKS_FULL_PATH)] # list of all packs from repo for entry in dir_entries: entry_path = os.path.join(zip_path, entry) if entry not in IGNORED_FILES and entry in packs_list and os.path.isdir(entry_path): # This is a pack directory, should keep only most recent release zip pack_files = get_files_in_dir(entry_path, ['zip']) latest_zip = get_latest_pack_zip_from_pack_files(entry, pack_files) if not latest_zip: logging.warning(f'Failed to get the zip of the pack {entry} from GCP') continue logging.info(f"Found latest zip of {entry}, which is {latest_zip}") zipped_packs[Path(latest_zip).stem] = latest_zip if not zipped_packs: raise Exception('No zip files were found') return zipped_packs
def build_pack_content_object(self, content_entity: str, entity_instance_path: str) -> dict: """ Build the pack content object the represents an entity instance. For example: HelloWorld Integration in Packs/HelloWorld. :param content_entity: The content entity, for example Integrations :param entity_instance_path: For example, for integration: ~/.../content/Packs/TestPack/Integrations/HelloWorld and for layout: ~/.../content/Packs/TestPack/Layout/layout-HelloWorldLayout.json :return: A pack content object. For example, INTEGRATION_PACK_OBJECT / LAYOUT_PACK_OBJECT variables in downloader_test.py """ # If the entity_instance_path is a file then get_files_in_dir will return the list: [entity_instance_path] file_paths: list = get_files_in_dir(entity_instance_path, CONTENT_FILE_ENDINGS, recursive=False) # If it's integration/script, all files under it should have the main details of the yml file, # otherwise we'll use the file's details. content_object: dict = dict() main_id, main_name = self.get_main_file_details(content_entity, entity_instance_path) # if main file doesn't exist/no entity instance path exist the content object won't be added to the pack content if all((main_id, main_name, file_paths)): content_object = {main_name: list()} # For example take a look at INTEGRATION_CUSTOM_CONTENT_OBJECT variable in downloader_test.py for file_path in file_paths: content_object[main_name].append({ 'name': main_name, 'id': main_id, 'path': file_path, 'file_ending': retrieve_file_ending(file_path) }) return content_object
def test_recursive_pack(self): pack_dir = 'demisto_sdk/commands/download/tests/tests_env/content/Packs/TestPack' files = [ f'{pack_dir}/Integrations/TestIntegration/TestIntegration.py', f'{pack_dir}/Integrations/TestIntegration/TestIntegration_testt.py', f'{pack_dir}/Scripts/TestScript/TestScript.py' ] assert sorted(get_files_in_dir(pack_dir, ['py'])) == sorted(files)
def test_not_recursive(self): project_dir = 'demisto_sdk/commands/download' files = [ f'{project_dir}/__init__.py', f'{project_dir}/downloader.py', f'{project_dir}/README.md' ] assert sorted(get_files_in_dir(project_dir, ['py', 'md'], False)) == sorted(files)
def test_recursive(self): integrations_dir = 'demisto_sdk/commands/download/tests/tests_env/content/Packs/TestPack/Integrations' integration_instance_dir = f'{integrations_dir}/TestIntegration' files = [ f'{integration_instance_dir}/TestIntegration.py', f'{integration_instance_dir}/TestIntegration_testt.py' ] assert sorted(get_files_in_dir(integrations_dir, ['py'])) == sorted(files)
def format_manager(input: str = None, output: str = None, from_version: str = '', no_validate: bool = None): """ Format_manager is a function that activated format command on different type of files. Args: input: (str) The path of the specific file. from_version: (str) in case of specific value for from_version that needs to be updated. output: (str) The path to save the formatted file to. no_validate (flag): Whether the user specifies not to run validate after format. Returns: int 0 in case of success 1 otherwise """ if input: files = get_files_in_dir(input, ['json', 'yml', 'py']) else: files = [file['name'] for file in get_changed_files(filter_results=lambda _file: not _file.pop('status') == 'D')] if output and not output.endswith(('yml', 'json', 'py')): raise Exception("The given output path is not a specific file path.\n" "Only file path can be a output path. Please specify a correct output.") log_list = [] error_list = [] if files: for file in files: file_path = file.replace('\\', '/') file_type = find_type(file_path) if file_type: file_type = file_type.value info_res, err_res, skip_res = run_format_on_file(input=file_path, file_type=file_type, from_version=from_version, output=output, no_validate=no_validate) if err_res: error_list.append("err_res") if err_res: log_list.extend([(err_res, print_error)]) if info_res: log_list.extend([(info_res, print_success)]) if skip_res: log_list.extend([(skip_res, print_warning)]) else: log_list.append(([f'Failed format file {input}.' + "No such file or directory"], print_error)) if error_list: for string, print_func in log_list: print_func('\n'.join(string)) return 1 for string, print_func in log_list: print_func('\n'.join(string)) return 0
def check_separators_in_files(self): """ Check if there are separators in the script files names. Returns: true if the files names are valid and there is no separators, and false if not. """ # Gets the all script files that may have the script name as base name files_to_check = get_files_in_dir(os.path.dirname(self.file_path), ['yml', 'py'], False) valid_files = [] invalid_files = [] for file_path in files_to_check: file_name = os.path.basename(file_path) if file_name.endswith('_test.py') or file_name.endswith( '_unified.yml'): base_name = file_name.rsplit('_', 1)[0] else: base_name = file_name.rsplit('.', 1)[0] valid_base_name = self.remove_separators_from_name(base_name) if valid_base_name != base_name: valid_files.append( valid_base_name.join(file_name.rsplit(base_name, 1))) invalid_files.append(file_name) if invalid_files: error_message, error_code = Errors.file_name_has_separators( 'script', invalid_files, valid_files) if self.handle_error(error_message, error_code, file_path=self.file_path): self.is_valid = False return False return True
def test_project_dir_is_file(self): project_dir = 'demisto_sdk/commands/download/downloader.py' assert get_files_in_dir(project_dir, ['py']) == [project_dir]
def format_manager(input: str = None, output: str = None, from_version: str = '', no_validate: bool = False, verbose: bool = False, update_docker: bool = False, assume_yes: bool = False, deprecate: bool = False, use_git: bool = False, prev_ver: str = None, include_untracked: bool = False, add_tests: bool = None, interactive: bool = True, id_set_path: str = None, clear_cache: bool = False): """ Format_manager is a function that activated format command on different type of files. Args: input: (str) The path of the specific file. from_version: (str) in case of specific value for from_version that needs to be updated. output: (str) The path to save the formatted file to. no_validate (flag): Whether the user specifies not to run validate after format. verbose (bool): Whether to print verbose logs or not update_docker (flag): Whether to update the docker image. assume_yes (bool): Whether to assume "yes" as answer to all prompts and run non-interactively deprecate (bool): Whether to deprecate the entity use_git (bool): Use git to automatically recognize which files changed and run format on them prev_ver (str): Against which branch should the difference be recognized include_untracked (bool): Whether to include untracked files when checking against git interactive (bool): Whether to run the format interactively or not (usually for contribution management) add_tests (bool): Whether to exclude tests automatically. id_set_path (str): The path of the id_set.json file. clear_cache (bool): wether to clear the cache Returns: int 0 in case of success 1 otherwise """ prev_ver = prev_ver if prev_ver else 'demisto/master' supported_file_types = ['json', 'yml', 'py', 'md'] use_git = use_git or not input if input: files = get_files_in_dir(input, supported_file_types) elif use_git: files = get_files_to_format_from_git(supported_file_types, prev_ver, include_untracked) if output and not output.endswith(('yml', 'json', 'py')): raise Exception( "The given output path is not a specific file path.\n" "Only file path can be a output path. Please specify a correct output." ) log_list = [] error_list: List[Tuple[int, int]] = [] if files: format_excluded_file = excluded_files + ['pack_metadata.json'] for file in files: file_path = file.replace('\\', '/') file_type = find_type(file_path, clear_cache=clear_cache) current_excluded_files = format_excluded_file[:] dirname = os.path.dirname(file_path) if dirname.endswith('CommonServerPython'): current_excluded_files.remove('CommonServerPython.py') if os.path.basename(file_path) in current_excluded_files: continue if any(test_dir in str(dirname) for test_dir in TESTS_AND_DOC_DIRECTORIES): continue if file_type and file_type.value not in UNFORMATTED_FILES: file_type = file_type.value info_res, err_res, skip_res = run_format_on_file( input=file_path, file_type=file_type, from_version=from_version, interactive=interactive, output=output, no_validate=no_validate, verbose=verbose, update_docker=update_docker, assume_yes=assume_yes, deprecate=deprecate, add_tests=add_tests, id_set_path=id_set_path) if err_res: log_list.extend([(err_res, print_error)]) if info_res: log_list.extend([(info_res, print_success)]) if skip_res: log_list.extend([(skip_res, print_warning)]) elif file_type: log_list.append(([ f"Ignoring format for {file_path} as {file_type.value} is currently not " f"supported by format command" ], print_warning)) else: log_list.append(([ f"Was unable to identify the file type for the following file: {file_path}" ], print_error)) update_content_entity_ids(files, verbose) else: if not use_git: log_list.append(([ f'Failed format file {input}.' + "No such file or directory" ], print_error)) return 1 print('') # Just adding a new line before summary for string, print_func in log_list: print_func('\n'.join(string)) if error_list: return 1 return 0
def format_manager(input: str = None, output: str = None, from_version: str = '', no_validate: bool = False, verbose: bool = False, update_docker: bool = False, assume_yes: bool = False): """ Format_manager is a function that activated format command on different type of files. Args: input: (str) The path of the specific file. from_version: (str) in case of specific value for from_version that needs to be updated. output: (str) The path to save the formatted file to. no_validate (flag): Whether the user specifies not to run validate after format. verbose (bool): Whether to print verbose logs or not update_docker (flag): Whether to update the docker image. assume_yes (bool): Whether to assume "yes" as answer to all prompts and run non-interactively Returns: int 0 in case of success 1 otherwise """ if input: files = get_files_in_dir(input, ['json', 'yml', 'py']) else: files = [file['name'] for file in get_changed_files(filter_results=lambda _file: not _file.pop('status') == 'D')] if output and not output.endswith(('yml', 'json', 'py')): raise Exception("The given output path is not a specific file path.\n" "Only file path can be a output path. Please specify a correct output.") log_list = [] error_list = [] if files: format_excluded_file = excluded_files + ['pack_metadata.json'] for file in files: file_path = file.replace('\\', '/') file_type = find_type(file_path) current_excluded_files = format_excluded_file[:] dirname = os.path.dirname(file_path) if dirname.endswith('CommonServerPython'): current_excluded_files.remove('CommonServerPython.py') if os.path.basename(file_path) in current_excluded_files: continue if dirname.endswith('test_data') or dirname.endswith('doc_imgs'): continue if file_type and file_type.value not in UNFORMATTED_FILES: file_type = file_type.value info_res, err_res, skip_res = run_format_on_file(input=file_path, file_type=file_type, from_version=from_version, output=output, no_validate=no_validate, verbose=verbose, update_docker=update_docker, assume_yes=assume_yes) if err_res: error_list.append("err_res") if err_res: log_list.extend([(err_res, print_error)]) if info_res: log_list.extend([(info_res, print_success)]) if skip_res: log_list.extend([(skip_res, print_warning)]) elif file_type: log_list.append(([f"Ignoring format for {file_path} as {file_type.value} is currently not " f"supported by format command"], print_warning)) else: log_list.append(([f"Was unable to identify the file type for the following file: {file_path}"], print_error)) else: log_list.append(([f'Failed format file {input}.' + "No such file or directory"], print_error)) print('') # Just adding a new line before summary for string, print_func in log_list: print_func('\n'.join(string)) if error_list: return 1 return 0