def copy_packs_content_to_packs_bundle(self, packs): """ Copy content in packs to the bundle that gets zipped to 'content_packs.zip'. Preserves directory structure except that packages inside the "Integrations" or "Scripts" directory inside a pack are flattened. Adds file prefixes according to how server expects to ingest the files, e.g. 'integration-' is prepended to integration yml filenames and 'script-' is prepended to script yml filenames and so on and so forth. """ for pack in packs: pack_name = os.path.basename(pack) if pack_name in self.packs_to_skip: continue pack_dst = os.path.join(self.packs_bundle, pack_name) os.mkdir(pack_dst) pack_dirs = get_child_directories(pack) pack_files = get_child_files(pack) # copy first level pack files over for file_path in pack_files: shutil.copy( file_path, os.path.join(pack_dst, os.path.basename(file_path))) # handle content directories in the pack for content_dir in pack_dirs: dir_name = os.path.basename(content_dir) dest_dir = os.path.join(pack_dst, dir_name) os.mkdir(dest_dir) if dir_name in DIR_TO_PREFIX: packages_dirs = get_child_directories(content_dir) for package_dir in packages_dirs: ymls, _ = get_yml_paths_in_dir(package_dir, error_msg='') if not ymls or (len(ymls) == 1 and ymls[0].endswith('_unified.yml')): msg = 'Skipping package: {} -'.format(package_dir) if not ymls: print_warning( '{} No yml files found in the package directory' .format(msg)) else: print_warning( '{} Only unified yml found in the package directory' .format(msg)) continue package_dir_name = os.path.basename(package_dir) unifier = Unifier(package_dir, dir_name, dest_dir) unifier.merge_script_package_to_yml() # also copy CHANGELOG markdown files over (should only be one per package) package_files = get_child_files(package_dir) changelog_files = [ file_path for file_path in package_files if 'CHANGELOG.md' in file_path ] for md_file_path in changelog_files: md_out_name = '{}-{}_CHANGELOG.md'.format( DIR_TO_PREFIX.get(dir_name), package_dir_name) shutil.copyfile( md_file_path, os.path.join(dest_dir, md_out_name)) else: self.copy_dir_files(content_dir, dest_dir)
def generate_readmes_for_new_content_pack(self, is_contribution=False): """ Generate the readme files for a new content pack. """ for pack_subdir in get_child_directories(self.pack_dir_path): basename = os.path.basename(pack_subdir) if basename in {SCRIPTS_DIR, INTEGRATIONS_DIR}: directories = get_child_directories(pack_subdir) for directory in directories: files = get_child_files(directory) for file in files: file_name = os.path.basename(file) if file_name.startswith('integration-') \ or file_name.startswith('script-') \ or file_name.startswith('automation-'): unified_file = file self.generate_readme_for_pack_content_item( unified_file, is_contribution) os.remove(unified_file) elif basename == 'Playbooks': files = get_child_files(pack_subdir) for file in files: file_name = os.path.basename(file) if file_name.startswith('playbook') and file_name.endswith( '.yml'): self.generate_readme_for_pack_content_item(file)
def test_merge_and_extract_new_file(self, tmp_path): env = Environment(tmp_path) parameters = [{ 'custom_content_object': env.INTEGRATION_CUSTOM_CONTENT_OBJECT, 'raw_files': [ 'odp/bn.py', 'odp/bn.yml', 'odp/bn_image.png', 'odp/bn_description.md', 'odp/README.md' ] }, { 'custom_content_object': env.SCRIPT_CUSTOM_CONTENT_OBJECT, 'raw_files': ['odp/bn.py', 'odp/bn.yml', 'odp/README.md'] }] for param in parameters: temp_dir = env.tmp_path / f'temp_dir_{parameters.index(param)}' os.mkdir(temp_dir) entity = param['custom_content_object']['entity'] downloader = Downloader(output=str(temp_dir), input='', regex='') basename = downloader.create_dir_name( param['custom_content_object']['name']) output_entity_dir_path = f'{temp_dir}/{entity}' os.mkdir(output_entity_dir_path) output_dir_path = f'{output_entity_dir_path}/{basename}' os.mkdir(output_dir_path) files = [ file.replace('odp', output_dir_path).replace('bn', basename) for file in param['raw_files'] ] downloader.merge_and_extract_new_file( param['custom_content_object']) output_files = get_child_files(output_dir_path) assert sorted(output_files) == sorted(files)
def merge_and_extract_new_file(self, custom_content_object: dict) -> None: """ Merges new files of type integration/script (not existing in the output pack) :param custom_content_object: The custom content object to merge into the pack :return: None """ file_entity: str = custom_content_object['entity'] file_path: str = custom_content_object['path'] file_type: str = custom_content_object['type'] file_name: str = custom_content_object['name'] file_code_language: str = custom_content_object.get('code_lang', '') if not self.verify_code_lang(file_code_language, file_type, file_name): return dir_output_path: str = os.path.join(self.output_pack_path, file_entity) # dir name should be the same as file name without separators mentioned in constants.py dir_name: str = self.create_dir_name(file_name) dir_output_path = os.path.join(dir_output_path, dir_name) extractor = Extractor(input=file_path, output=dir_output_path, file_type=file_type, base_name=dir_name, no_auto_create_dir=True, no_logging=not self.log_verbose, no_pipenv=True) extractor.extract_to_package_format() for file_path in get_child_files(dir_output_path): self.format_file(file_path, retrieve_file_ending(file_path)) self.num_added_files += 1 self.log_finished_file('Added', file_name, file_entity[:-1])
def content_item_to_package_format(self, content_item_dir: str, del_unified: bool = True): """ Iterate over the YAML files in a directory and create packages (a containing directory and component files) from the YAMLs of integrations and scripts Args: content_item_dir (str): Path to the directory containing the content item YAML file(s) del_unified (bool): Whether to delete the unified yaml the package was extracted from """ child_files = get_child_files(content_item_dir) content_item_file_path = '' for child_file in child_files: cf_name_lower = os.path.basename(child_file).lower() if cf_name_lower.startswith( (SCRIPT, AUTOMATION, INTEGRATION)) and cf_name_lower.endswith('yml'): content_item_file_path = child_file file_type = find_type(content_item_file_path) file_type = file_type.value if file_type else file_type try: extractor = Extractor(input=content_item_file_path, file_type=file_type, output=content_item_dir) extractor.extract_to_package_format() except Exception as e: err_msg = f'Error occurred while trying to split the unified YAML "{content_item_file_path}" ' \ f'into its component parts.\nError: "{e}"' self.contrib_conversion_errs.append(err_msg) if del_unified: os.remove(content_item_file_path)
def merge_and_extract_existing_file(self, custom_content_object: dict) -> None: """ "Smart" merges old files of type integration/script (existing in the output pack) :param custom_content_object: The custom content object to merge into the pack :return: None """ file_path: str = custom_content_object['path'] file_name: str = custom_content_object['name'] file_type: str = custom_content_object['type'] file_entity: str = custom_content_object['entity'] file_code_language: str = custom_content_object.get('code_lang', '') if not self.verify_code_lang(file_code_language, file_type, file_name): return base_name: str = self.create_dir_name(file_name) temp_dir = mkdtemp() extractor = Extractor(input=file_path, output=temp_dir, file_type=file_type, base_name=base_name, no_logging=not self.log_verbose, no_pipenv=True, no_readme=True, no_auto_create_dir=True) extractor.extract_to_package_format() extracted_file_paths: list = get_child_files(temp_dir) corresponding_pack_object: dict = self.get_corresponding_pack_content_object(custom_content_object) for ex_file_path in extracted_file_paths: ex_file_ending: str = retrieve_file_ending(ex_file_path) ex_file_detail: str = self.get_extracted_file_detail(ex_file_ending) # Get the file name to search for in the pack object (integration/script contains several files of the # same type. For example: integration's py code and integration's unit tests code) searched_basename: str = self.get_searched_basename(file_name, ex_file_ending, ex_file_detail) corresponding_pack_file_object: dict = self.get_corresponding_pack_file_object(searched_basename, corresponding_pack_object) if not corresponding_pack_file_object: corresponding_pack_file_path: str = os.path.join(self.output_pack_path, file_entity, self.create_dir_name(file_name), searched_basename) else: corresponding_pack_file_path = corresponding_pack_file_object['path'] # We use "smart" merge only for yml files (py, png & md files to be moved regularly) if ex_file_ending == 'yml': # adding the deleted fields (by Demisto) of the old yml/json file to the custom content file. self.update_data(ex_file_path, corresponding_pack_file_path, ex_file_ending) try: shutil.move(src=ex_file_path, dst=corresponding_pack_file_path) except shutil.Error as e: print_color(e, LOG_COLORS.RED) raise self.format_file(corresponding_pack_file_path, ex_file_ending) try: shutil.rmtree(temp_dir, ignore_errors=True) except shutil.Error as e: print_color(e, LOG_COLORS.RED) raise self.num_merged_files += 1 self.log_finished_file('Merged', file_name, file_entity[:-1])
def directory_uploader(self, path: str): """Uploads directories by path Args: path (str): Path for directory to upload. """ if is_path_of_integration_directory(path): list_integrations = get_child_directories(path) for integration in list_integrations: self.integration_uploader(integration) elif is_path_of_script_directory(path): list_script = get_child_directories(path) for script in list_script: self.script_uploader(script) elif is_path_of_playbook_directory( path) or is_path_of_test_playbook_directory(path): list_playbooks = get_child_files(path) for playbook in list_playbooks: if playbook.endswith('.yml'): self.playbook_uploader(playbook) elif is_path_of_incident_field_directory(path): list_incident_fields = get_child_files(path) for incident_field in list_incident_fields: if incident_field.endswith('.json'): self.incident_field_uploader(incident_field) elif is_path_of_widget_directory(path): list_widgets = get_child_files(path) for widget in list_widgets: if widget.endswith('.json'): self.widget_uploader(widget) elif is_path_of_dashboard_directory(path): list_dashboards = get_child_files(path) for dashboard in list_dashboards: if dashboard.endswith('.json'): self.dashboard_uploader(dashboard) elif is_path_of_layout_directory(path): list_layouts = get_child_files(path) for layout in list_layouts: if layout.endswith('.json'): self.layout_uploader(layout) elif is_path_of_incident_type_directory(path): list_incident_types = get_child_files(path) for incident_type in list_incident_types: if incident_type.endswith('.json'): self.incident_type_uploader(incident_type) elif is_path_of_classifier_directory(path): list_classifiers = get_child_files(path) for classifiers in list_classifiers: if classifiers.endswith('.json'): self.classifier_uploader(classifiers)
def get_custom_content_objects(self) -> List[dict]: """ Creates a list of all custom content objects :return: The list of all custom content objects """ custom_content_file_paths: list = get_child_files(self.custom_content_temp_dir) custom_content_objects: List = list() for file_path in custom_content_file_paths: custom_content_object: Dict = self.build_custom_content_object(file_path) if custom_content_object['type']: # If custom content object's type is empty it means the file isn't of support content entity custom_content_objects.append(custom_content_object) return custom_content_objects
def build_pack_content(self) -> None: """ Build a data structure called custom content that holds basic data for each content entity within the given output pack. For example check out the PACK_CONTENT variable in downloader_test.py """ for content_entity_path in get_child_directories(self.output_pack_path): raw_content_entity: str = os.path.basename(os.path.normpath(content_entity_path)) content_entity: str = raw_content_entity if content_entity in (INTEGRATIONS_DIR, SCRIPTS_DIR): # If entity is of type integration/script it will have dirs, otherwise files entity_instances_paths: list = get_child_directories(content_entity_path) else: entity_instances_paths = get_child_files(content_entity_path) for entity_instance_path in entity_instances_paths: content_object: dict = self.build_pack_content_object(content_entity, entity_instance_path) if content_object: self.pack_content[content_entity].append(content_object)
def test_merge_and_extract_new_file(self, custom_content_object, raw_files): env_guard = EnvironmentGuardian() temp_dir = mkdtemp() entity = custom_content_object['entity'] downloader = Downloader(output=temp_dir, input='') basename = downloader.create_dir_name(custom_content_object['name']) output_entity_dir_path = f'{temp_dir}/{entity}' os.mkdir(output_entity_dir_path) output_dir_path = f'{output_entity_dir_path}/{basename}' os.mkdir(output_dir_path) files = [file.replace('odp', output_dir_path).replace('bn', basename) for file in raw_files] downloader.merge_and_extract_new_file(custom_content_object) output_files = get_child_files(output_dir_path) test_answer = sorted(output_files) == sorted(files) env_guard.restore_environment('test_merge_and_extract_new_file', temp_dir) assert test_answer
def get_system_content_objects(self) -> List[dict]: """ Creates a list of all custom content objects :return: The list of all custom content objects """ system_content_file_paths: list = get_child_files( self.system_content_temp_dir) system_content_objects: List = list() for file_path in system_content_file_paths: try: system_content_object: Dict = self.build_custom_content_object( file_path) system_content_objects.append(system_content_object) # Do not add file to custom_content_objects if it has an invalid format except ValueError as e: print_color(f"Error when loading {file_path}, skipping", LOG_COLORS.RED) print_color(f"{e}", LOG_COLORS.RED) return system_content_objects
def get_custom_content_objects(self) -> List[dict]: """ Creates a list of all custom content objects :return: The list of all custom content objects """ custom_content_file_paths: list = get_child_files( self.custom_content_temp_dir) custom_content_objects: List = list() for file_path in custom_content_file_paths: try: custom_content_object: Dict = self.build_custom_content_object( file_path) if custom_content_object['type']: # If custom content object's type is empty it means the file isn't of support content entity custom_content_objects.append(custom_content_object) # Do not add file to custom_content_objects if it has an invalid format except ValueError as e: print_color(f"Error when loading {file_path}, skipping", LOG_COLORS.RED) print_color(f"{e}", LOG_COLORS.RED) return custom_content_objects
def directory_uploader(self, path: str): """Uploads directories by path Args: path (str): Path for directory to upload. """ if is_path_of_integration_directory(path): # Upload unified integration files list_unified_integrations = get_child_files(path) for unified_integration in list_unified_integrations: file_type = find_type(unified_integration) if file_type == FileType.INTEGRATION: self.integration_uploader(unified_integration) # Upload spliced integration files list_integrations = get_child_directories(path) for integration in list_integrations: self.integration_uploader(integration) elif is_path_of_script_directory(path): # Upload unified scripts files list_unified_scripts = get_child_files(path) for unified_script in list_unified_scripts: file_type = find_type(unified_script) if file_type in (FileType.SCRIPT, FileType.TEST_SCRIPT): self.script_uploader(unified_script) # Upload spliced scripts list_script = get_child_directories(path) for script in list_script: self.script_uploader(script) elif is_path_of_playbook_directory( path) or is_path_of_test_playbook_directory(path): list_playbooks = get_child_files(path) for playbook in list_playbooks: if playbook.endswith('.yml'): self.playbook_uploader(playbook) elif is_path_of_incident_field_directory(path): list_incident_fields = get_child_files(path) for incident_field in list_incident_fields: if incident_field.endswith('.json'): self.incident_field_uploader(incident_field) elif is_path_of_widget_directory(path): list_widgets = get_child_files(path) for widget in list_widgets: if widget.endswith('.json'): self.widget_uploader(widget) elif is_path_of_dashboard_directory(path): list_dashboards = get_child_files(path) for dashboard in list_dashboards: if dashboard.endswith('.json'): self.dashboard_uploader(dashboard) elif is_path_of_layout_directory(path): list_layouts = get_child_files(path) for layout in list_layouts: if layout.endswith('.json'): self.layout_uploader(layout) elif is_path_of_incident_type_directory(path): list_incident_types = get_child_files(path) for incident_type in list_incident_types: if incident_type.endswith('.json'): self.incident_type_uploader(incident_type) elif is_path_of_classifier_directory(path): list_classifiers = get_child_files(path) for classifiers in list_classifiers: if classifiers.endswith('.json'): self.classifier_uploader(classifiers)
def content_item_to_package_format( self, content_item_dir: str, del_unified: bool = True, source_mapping: Union[Dict[str, Dict[str, str]]] = None): """ Iterate over the YAML files in a directory and create packages (a containing directory and component files) from the YAMLs of integrations and scripts Args: content_item_dir (str): Path to the directory containing the content item YAML file(s) del_unified (bool): Whether to delete the unified yaml the package was extracted from source_mapping (Union[Dict], optional): Can be used when updating an existing pack and the package directory of a content item is not what would ordinarily be set by the `demisto-sdk` `split` command. Sample value would be, `{'integration-AbuseIPDB.yml': {'containing_dir_name': 'AbuseDB', 'base_name': 'AbuseDB'}}` - the split command would create a containing directory of `AbuseIPDB` for the file `integration-AbuseIPDB.yml` and we need the containing directory of the package to match what already exists in the repo. """ child_files = get_child_files(content_item_dir) for child_file in child_files: cf_name_lower = os.path.basename(child_file).lower() if cf_name_lower.startswith( (SCRIPT, AUTOMATION, INTEGRATION)) and cf_name_lower.endswith('yml'): content_item_file_path = child_file file_type = find_type(content_item_file_path) file_type = file_type.value if file_type else file_type try: child_file_name = os.path.basename(child_file) if source_mapping and child_file_name in source_mapping.keys( ): child_file_mapping = source_mapping.get( child_file_name, {}) base_name = child_file_mapping.get('base_name', '') containing_dir_name = child_file_mapping.get( 'containing_dir_name', '') # for legacy unified yamls in the repo, their containing directory will be that of their # entity type directly instead of the typical package format. For those cases, we need the # extractor to auto create the containing directory. An example would be - # 'content/Packs/AbuseDB/Scripts/script-AbuseIPDBPopulateIndicators.yml' autocreate_dir = containing_dir_name == ENTITY_TYPE_TO_DIR.get( file_type, '') output_dir = os.path.join( self.pack_dir_path, ENTITY_TYPE_TO_DIR.get(file_type, '')) if not autocreate_dir: output_dir = os.path.join(output_dir, containing_dir_name) os.makedirs(output_dir, exist_ok=True) extractor = YmlSplitter( input=content_item_file_path, file_type=file_type, output=output_dir, no_readme=True, base_name=base_name, no_auto_create_dir=(not autocreate_dir), no_pipenv=self.no_pipenv) else: extractor = YmlSplitter(input=content_item_file_path, file_type=file_type, output=content_item_dir, no_pipenv=self.no_pipenv) extractor.extract_to_package_format() except Exception as e: err_msg = f'Error occurred while trying to split the unified YAML "{content_item_file_path}" ' \ f'into its component parts.\nError: "{e}"' self.contrib_conversion_errs.append(err_msg) finally: output_path = extractor.get_output_path() if self.create_new: # Moving the unified file to its package. shutil.move(content_item_file_path, output_path) if del_unified: if os.path.exists(content_item_file_path): os.remove(content_item_file_path) moved_unified_dst = os.path.join( output_path, child_file_name) if os.path.exists(moved_unified_dst): os.remove(moved_unified_dst)
def copy_packs_content_to_packs_bundle(self, packs): """ Copy content in packs to the bundle that gets zipped to 'content_packs.zip'. Preserves directory structure except that packages inside the "Integrations" or "Scripts" directory inside a pack are flattened. Adds file prefixes according to how server expects to ingest the files, e.g. 'integration-' is prepended to integration yml filenames and 'script-' is prepended to script yml filenames and so on and so forth. """ for pack in packs: pack_name = os.path.basename(pack) if pack_name in self.packs_to_skip: continue pack_dst = os.path.join(self.packs_bundle, pack_name) os.mkdir(pack_dst) pack_dirs = get_child_directories(pack) pack_files = get_child_files(pack) # copy first level pack files over for file_path in pack_files: shutil.copy( file_path, os.path.join(pack_dst, os.path.basename(file_path))) # handle content directories in the pack for content_dir in pack_dirs: dir_name = os.path.basename(content_dir) dest_dir = os.path.join(pack_dst, dir_name) os.mkdir(dest_dir) if dir_name in DIR_TO_PREFIX: packages_dirs = get_child_directories(content_dir) if packages_dirs: # split yml files directories for package_dir in packages_dirs: ymls, _ = get_yml_paths_in_dir(package_dir, error_msg='') if not ymls or (len(ymls) == 1 and ymls[0].endswith('_unified.yml')): msg = f'Skipping package: {package_dir} -' if not ymls: print_warning( '{} No yml files found in the package directory' .format(msg)) else: print_warning( '{} Only unified yml found in the package directory' .format(msg)) continue unifier = Unifier(package_dir, dir_name, dest_dir) if parse_version( unifier.yml_data.get( 'toversion', '99.99.99')) >= parse_version('6.0.0'): new_file_paths = unifier.merge_script_package_to_yml( file_name_suffix=self.file_name_suffix) for new_file_path in new_file_paths: self.add_from_version_to_yml(new_file_path) non_split_yml_files = [ f for f in os.listdir(content_dir) if os.path.isfile(os.path.join(content_dir, f)) and ( fnmatch.fnmatch(f, 'integration-*.yml') or fnmatch.fnmatch(f, 'script-*.yml')) ] if non_split_yml_files: # old format non split yml files for yml_file in non_split_yml_files: new_file_path = self.add_suffix_to_file_path( os.path.join(dest_dir, yml_file)) shutil.copyfile( os.path.join(content_dir, yml_file), new_file_path) self.add_from_version_to_yml(new_file_path) else: self.copy_dir_files(content_dir, dest_dir, is_legacy_bundle=False)