def merge_and_extract_new_file(self, custom_content_object: dict) -> None: """ Merges new files of type integration/script (not existing in the output pack) :param custom_content_object: The custom content object to merge into the pack :return: None """ file_entity: str = custom_content_object['entity'] file_path: str = custom_content_object['path'] file_type: str = custom_content_object['type'] file_name: str = custom_content_object['name'] file_code_language: str = custom_content_object.get('code_lang', '') if not self.verify_code_lang(file_code_language, file_type, file_name): return dir_output_path: str = os.path.join(self.output_pack_path, file_entity) # dir name should be the same as file name without separators mentioned in constants.py dir_name: str = self.create_dir_name(file_name) dir_output_path = os.path.join(dir_output_path, dir_name) extractor = Extractor(input=file_path, output=dir_output_path, file_type=file_type, base_name=dir_name, no_auto_create_dir=True, no_logging=not self.log_verbose, no_pipenv=True) extractor.extract_to_package_format() for file_path in get_child_files(dir_output_path): self.format_file(file_path, retrieve_file_ending(file_path)) self.num_added_files += 1 self.log_finished_file('Added', file_name, file_entity[:-1])
def get_main_file_details(content_entity: str, entity_instance_path: str) -> tuple: """ Returns the details of the "main" file within an entity instance. For example: In the HelloWorld integration under Packs/HelloWorld, the main file is the yml file. It contains all relevant ids and names for all the files under the HelloWorld integration dir. :param content_entity: The content entity, for example Integrations :param entity_instance_path: For example: ~/.../content/Packs/TestPack/Integrations/HelloWorld :return: The main file id & name """ main_file_data: dict = dict() main_file_path: str = str() # Entities which contain yml files if content_entity in (INTEGRATIONS_DIR, SCRIPTS_DIR, PLAYBOOKS_DIR, TEST_PLAYBOOKS_DIR): if os.path.isdir(entity_instance_path): _, main_file_path = get_yml_paths_in_dir(entity_instance_path) elif os.path.isfile(entity_instance_path): main_file_path = entity_instance_path if main_file_path: main_file_data = get_yaml(main_file_path) # Entities which are json files (md files are ignored - changelog/readme) else: if os.path.isfile(entity_instance_path) and retrieve_file_ending( entity_instance_path) == 'json': main_file_data = get_json(entity_instance_path) main_id = get_entity_id_by_entity_type(main_file_data, content_entity) main_name = get_entity_name_by_entity_type(main_file_data, content_entity) return main_id, main_name
def build_pack_content_object(self, content_entity: str, entity_instance_path: str) -> dict: """ Build the pack content object the represents an entity instance. For example: HelloWorld Integration in Packs/HelloWorld. :param content_entity: The content entity, for example Integrations :param entity_instance_path: For example, for integration: ~/.../content/Packs/TestPack/Integrations/HelloWorld and for layout: ~/.../content/Packs/TestPack/Layout/layout-HelloWorldLayout.json :return: A pack content object. For example, INTEGRATION_PACK_OBJECT / LAYOUT_PACK_OBJECT variables in downloader_test.py """ # If the entity_instance_path is a file then get_files_in_dir will return the list: [entity_instance_path] file_paths: list = get_files_in_dir(entity_instance_path, CONTENT_FILE_ENDINGS, recursive=False) # If it's integration/script, all files under it should have the main details of the yml file, # otherwise we'll use the file's details. content_object: dict = dict() main_id, main_name = self.get_main_file_details(content_entity, entity_instance_path) # if main file doesn't exist/no entity instance path exist the content object won't be added to the pack content if all((main_id, main_name, file_paths)): content_object = {main_name: list()} # For example take a look at INTEGRATION_CUSTOM_CONTENT_OBJECT variable in downloader_test.py for file_path in file_paths: content_object[main_name].append({ 'name': main_name, 'id': main_id, 'path': file_path, 'file_ending': retrieve_file_ending(file_path) }) return content_object
def merge_and_extract_existing_file(self, custom_content_object: dict) -> None: """ "Smart" merges old files of type integration/script (existing in the output pack) :param custom_content_object: The custom content object to merge into the pack :return: None """ file_path: str = custom_content_object['path'] file_name: str = custom_content_object['name'] file_type: str = custom_content_object['type'] file_entity: str = custom_content_object['entity'] file_code_language: str = custom_content_object.get('code_lang', '') if not self.verify_code_lang(file_code_language, file_type, file_name): return base_name: str = self.create_dir_name(file_name) temp_dir = mkdtemp() extractor = Extractor(input=file_path, output=temp_dir, file_type=file_type, base_name=base_name, no_logging=not self.log_verbose, no_pipenv=True, no_readme=True, no_auto_create_dir=True) extractor.extract_to_package_format() extracted_file_paths: list = get_child_files(temp_dir) corresponding_pack_object: dict = self.get_corresponding_pack_content_object(custom_content_object) for ex_file_path in extracted_file_paths: ex_file_ending: str = retrieve_file_ending(ex_file_path) ex_file_detail: str = self.get_extracted_file_detail(ex_file_ending) # Get the file name to search for in the pack object (integration/script contains several files of the # same type. For example: integration's py code and integration's unit tests code) searched_basename: str = self.get_searched_basename(file_name, ex_file_ending, ex_file_detail) corresponding_pack_file_object: dict = self.get_corresponding_pack_file_object(searched_basename, corresponding_pack_object) if not corresponding_pack_file_object: corresponding_pack_file_path: str = os.path.join(self.output_pack_path, file_entity, self.create_dir_name(file_name), searched_basename) else: corresponding_pack_file_path = corresponding_pack_file_object['path'] # We use "smart" merge only for yml files (py, png & md files to be moved regularly) if ex_file_ending == 'yml': # adding the deleted fields (by Demisto) of the old yml/json file to the custom content file. self.update_data(ex_file_path, corresponding_pack_file_path, ex_file_ending) try: shutil.move(src=ex_file_path, dst=corresponding_pack_file_path) except shutil.Error as e: print_color(e, LOG_COLORS.RED) raise self.format_file(corresponding_pack_file_path, ex_file_ending) try: shutil.rmtree(temp_dir, ignore_errors=True) except shutil.Error as e: print_color(e, LOG_COLORS.RED) raise self.num_merged_files += 1 self.log_finished_file('Merged', file_name, file_entity[:-1])
def get_full_file_path_for_vulture(file_name: str, content_path: str) -> str: """Get the full file path to a file with a given name name from the content path Args: file_name (str): The file name of the file to find content_path (str): The content file path Returns: str. The path to the file """ file_ending = retrieve_file_ending(file_name) if not file_ending: file_name = f'{file_name}.py' elif file_ending != 'py': file_name = file_name.replace(file_ending, 'py') return find_file(content_path, file_name)
def test_retrieve_file_ending(self, path, output): assert retrieve_file_ending(path) == output