def merge_and_extract_new_file(self, custom_content_object: dict) -> None:
"""
Merges new files of type integration/script (not existing in the output pack)
:param custom_content_object: The custom content object to merge into the pack
:return: None
"""
file_entity: str = custom_content_object['entity']
file_path: str = custom_content_object['path']
file_type: str = custom_content_object['type']
file_name: str = custom_content_object['name']
file_code_language: str = custom_content_object.get('code_lang', '')
if not self.verify_code_lang(file_code_language, file_type, file_name):
return
dir_output_path: str = os.path.join(self.output_pack_path, file_entity)
# dir name should be the same as file name without separators mentioned in constants.py
dir_name: str = self.create_dir_name(file_name)
dir_output_path = os.path.join(dir_output_path, dir_name)
extractor = Extractor(input=file_path,
output=dir_output_path,
file_type=file_type,
base_name=dir_name,
no_auto_create_dir=True,
no_logging=not self.log_verbose,
no_pipenv=True)
extractor.extract_to_package_format()
for file_path in get_child_files(dir_output_path):
self.format_file(file_path, retrieve_file_ending(file_path))
self.num_added_files += 1
self.log_finished_file('Added', file_name, file_entity[:-1])
def get_main_file_details(content_entity: str,
entity_instance_path: str) -> tuple:
"""
Returns the details of the "main" file within an entity instance.
For example: In the HelloWorld integration under Packs/HelloWorld, the main file is the yml file.
It contains all relevant ids and names for all the files under the HelloWorld integration dir.
:param content_entity: The content entity, for example Integrations
:param entity_instance_path: For example: ~/.../content/Packs/TestPack/Integrations/HelloWorld
:return: The main file id & name
"""
main_file_data: dict = dict()
main_file_path: str = str()
# Entities which contain yml files
if content_entity in (INTEGRATIONS_DIR, SCRIPTS_DIR, PLAYBOOKS_DIR,
TEST_PLAYBOOKS_DIR):
if os.path.isdir(entity_instance_path):
_, main_file_path = get_yml_paths_in_dir(entity_instance_path)
elif os.path.isfile(entity_instance_path):
main_file_path = entity_instance_path
if main_file_path:
main_file_data = get_yaml(main_file_path)
# Entities which are json files (md files are ignored - changelog/readme)
else:
if os.path.isfile(entity_instance_path) and retrieve_file_ending(
entity_instance_path) == 'json':
main_file_data = get_json(entity_instance_path)
main_id = get_entity_id_by_entity_type(main_file_data, content_entity)
main_name = get_entity_name_by_entity_type(main_file_data,
content_entity)
return main_id, main_name
def build_pack_content_object(self, content_entity: str, entity_instance_path: str) -> dict:
"""
Build the pack content object the represents an entity instance.
For example: HelloWorld Integration in Packs/HelloWorld.
:param content_entity: The content entity, for example Integrations
:param entity_instance_path: For example, for integration: ~/.../content/Packs/TestPack/Integrations/HelloWorld
and for layout: ~/.../content/Packs/TestPack/Layout/layout-HelloWorldLayout.json
:return: A pack content object. For example, INTEGRATION_PACK_OBJECT / LAYOUT_PACK_OBJECT variables
in downloader_test.py
"""
# If the entity_instance_path is a file then get_files_in_dir will return the list: [entity_instance_path]
file_paths: list = get_files_in_dir(entity_instance_path, CONTENT_FILE_ENDINGS, recursive=False)
# If it's integration/script, all files under it should have the main details of the yml file,
# otherwise we'll use the file's details.
content_object: dict = dict()
main_id, main_name = self.get_main_file_details(content_entity, entity_instance_path)
# if main file doesn't exist/no entity instance path exist the content object won't be added to the pack content
if all((main_id, main_name, file_paths)):
content_object = {main_name: list()}
# For example take a look at INTEGRATION_CUSTOM_CONTENT_OBJECT variable in downloader_test.py
for file_path in file_paths:
content_object[main_name].append({
'name': main_name,
'id': main_id,
'path': file_path,
'file_ending': retrieve_file_ending(file_path)
})
return content_object
def merge_and_extract_existing_file(self, custom_content_object: dict) -> None:
"""
"Smart" merges old files of type integration/script (existing in the output pack)
:param custom_content_object: The custom content object to merge into the pack
:return: None
"""
file_path: str = custom_content_object['path']
file_name: str = custom_content_object['name']
file_type: str = custom_content_object['type']
file_entity: str = custom_content_object['entity']
file_code_language: str = custom_content_object.get('code_lang', '')
if not self.verify_code_lang(file_code_language, file_type, file_name):
return
base_name: str = self.create_dir_name(file_name)
temp_dir = mkdtemp()
extractor = Extractor(input=file_path, output=temp_dir, file_type=file_type, base_name=base_name,
no_logging=not self.log_verbose, no_pipenv=True, no_readme=True,
no_auto_create_dir=True)
extractor.extract_to_package_format()
extracted_file_paths: list = get_child_files(temp_dir)
corresponding_pack_object: dict = self.get_corresponding_pack_content_object(custom_content_object)
for ex_file_path in extracted_file_paths:
ex_file_ending: str = retrieve_file_ending(ex_file_path)
ex_file_detail: str = self.get_extracted_file_detail(ex_file_ending)
# Get the file name to search for in the pack object (integration/script contains several files of the
# same type. For example: integration's py code and integration's unit tests code)
searched_basename: str = self.get_searched_basename(file_name, ex_file_ending, ex_file_detail)
corresponding_pack_file_object: dict = self.get_corresponding_pack_file_object(searched_basename,
corresponding_pack_object)
if not corresponding_pack_file_object:
corresponding_pack_file_path: str = os.path.join(self.output_pack_path, file_entity,
self.create_dir_name(file_name), searched_basename)
else:
corresponding_pack_file_path = corresponding_pack_file_object['path']
# We use "smart" merge only for yml files (py, png & md files to be moved regularly)
if ex_file_ending == 'yml':
# adding the deleted fields (by Demisto) of the old yml/json file to the custom content file.
self.update_data(ex_file_path, corresponding_pack_file_path, ex_file_ending)
try:
shutil.move(src=ex_file_path, dst=corresponding_pack_file_path)
except shutil.Error as e:
print_color(e, LOG_COLORS.RED)
raise
self.format_file(corresponding_pack_file_path, ex_file_ending)
try:
shutil.rmtree(temp_dir, ignore_errors=True)
except shutil.Error as e:
print_color(e, LOG_COLORS.RED)
raise
self.num_merged_files += 1
self.log_finished_file('Merged', file_name, file_entity[:-1])
def get_full_file_path_for_vulture(file_name: str,
content_path: str) -> str:
"""Get the full file path to a file with a given name name from the content path
Args:
file_name (str): The file name of the file to find
content_path (str): The content file path
Returns:
str. The path to the file
"""
file_ending = retrieve_file_ending(file_name)
if not file_ending:
file_name = f'{file_name}.py'
elif file_ending != 'py':
file_name = file_name.replace(file_ending, 'py')
return find_file(content_path, file_name)
def test_retrieve_file_ending(self, path, output):
assert retrieve_file_ending(path) == output
