def prepare_single_content_item_for_validation(filename: str, data: bytes, tmp_directory: str) -> str:
content = Content(tmp_directory)
pack_name = 'TmpPack'
pack_dir = content.path / 'Packs' / pack_name
# create pack_metadata.json file in TmpPack
contrib_converter = ContributionConverter(name=pack_name, base_dir=tmp_directory, pack_dir_name=pack_name)
contrib_converter.create_metadata_file({'description': 'Temporary Pack', 'author': 'xsoar'})
prefix = '-'.join(filename.split('-')[:-1])
containing_dir = pack_dir / ENTITY_TYPE_TO_DIR.get(prefix, 'Integrations')
containing_dir.mkdir(exist_ok=True)
data_as_string = data.decode()
loaded_data = yaml.load(data_as_string)
buff = io.StringIO()
yaml.dump(loaded_data, buff)
data_as_string = buff.getvalue()
# write yaml integration file to file system
file_path = containing_dir / filename
file_path.write_text(data_as_string)
file_type = find_type(str(file_path))
file_type = file_type.value if file_type else file_type
extractor = Extractor(
input=str(file_path), file_type=file_type, output=containing_dir, no_logging=True, no_pipenv=True)
# validate the resulting package files, ergo set path_to_validate to the package directory that results
# from extracting the unified yaml to a package format
extractor.extract_to_package_format()
return extractor.get_output_path()
def convert_contribution_dir_to_pack_contents(
self, unpacked_contribution_dir: str) -> None:
"""Converts a directory and its contents unpacked from the contribution zip file to the appropriate structure
Example:
The pack directory after `unpack_contribution_to_dst_pack_directory` has been executed:
ExamplePack
├── automation
│ └── automation-ExampleAutomation.yml
├── integration
│ └── integration-ExampleIntegration.yml
├── playbook
│ └── playbook-ExamplePlaybook.yml
├── report
│ └── report-ExampleReport.json
└── reputation
└── reputation-ExampleReputation.json
The updated pack directory structure after `convert_contribution_dir_to_pack_contents` has been
executed, passing the path of .../ExamplePack/integration as the argument, would appear as so:
ExamplePack
├── automation
│ └── automation-ExampleAutomation.yml
├── Integrations
│ └── integration-ExampleIntegration.yml
├── playbook
│ └── playbook-ExamplePlaybook.yml
├── report
│ └── report-ExampleReport.json
└── reputation
└── reputation-ExampleReputation.json
Args:
unpacked_contribution_dir (str): The directory to convert
"""
basename = os.path.basename(unpacked_contribution_dir)
if basename in ENTITY_TYPE_TO_DIR:
dst_name = ENTITY_TYPE_TO_DIR.get(basename, '')
src_path = os.path.join(self.pack_dir_path, basename)
dst_path = os.path.join(self.pack_dir_path, dst_name)
if os.path.exists(dst_path):
# move src folder files to dst folder
for _, _, files in os.walk(src_path, topdown=False):
for name in files:
src_file_path = os.path.join(src_path, name)
dst_file_path = os.path.join(dst_path, name)
shutil.move(src_file_path, dst_file_path)
shutil.rmtree(src_path, ignore_errors=True)
else:
# replace dst folder with src folder
shutil.move(src_path, dst_path)
def file_type_to_entity(file_data: dict, file_type: str) -> str:
"""
Given the file type returns the file entity
:param file_data: The file data
:param file_type: The file type, for example: integration
:return: The file entity, for example: Integrations
"""
if file_type and file_type == 'playbook':
name: str = get_entity_name_by_entity_type(file_data, PLAYBOOKS_DIR)
if name and 'test' in name.lower():
return TEST_PLAYBOOKS_DIR
return ENTITY_TYPE_TO_DIR.get(file_type, '')
def prepare_single_content_item_for_validation(
filename: str, data: bytes, tmp_directory: str) -> Tuple[str, Dict]:
content = Content(tmp_directory)
pack_name = 'TmpPack'
pack_dir = content.path / 'Packs' / pack_name
# create pack_metadata.json file in TmpPack
contrib_converter = ContributionConverter(name=pack_name,
base_dir=tmp_directory,
pack_dir_name=pack_name)
contrib_converter.create_metadata_file({
'description': 'Temporary Pack',
'author': 'xsoar'
})
prefix = '-'.join(filename.split('-')[:-1])
containing_dir = pack_dir / ENTITY_TYPE_TO_DIR.get(prefix, 'Integrations')
containing_dir.mkdir(exist_ok=True)
is_json = filename.casefold().endswith('.json')
data_as_string = data.decode()
loaded_data = json.loads(data_as_string) if is_json else yaml.load(
data_as_string)
if is_json:
data_as_string = json.dumps(loaded_data)
else:
buff = io.StringIO()
yaml.dump(loaded_data, buff)
data_as_string = buff.getvalue()
# write content item file to file system
file_path = containing_dir / filename
file_path.write_text(data_as_string)
file_type = find_type(str(file_path))
file_type = file_type.value if file_type else file_type
if is_json or file_type in (FileType.PLAYBOOK.value,
FileType.TEST_PLAYBOOK.value):
return str(file_path), {}
extractor = Extractor(input=str(file_path),
file_type=file_type,
output=containing_dir,
no_logging=True,
no_pipenv=True,
no_basic_fmt=True)
# validate the resulting package files, ergo set path_to_validate to the package directory that results
# from extracting the unified yaml to a package format
extractor.extract_to_package_format()
code_fp_to_row_offset = {
get_extracted_code_filepath(extractor):
extractor.lines_inserted_at_code_start
}
return extractor.get_output_path(), code_fp_to_row_offset
def content_item_to_package_format(
self,
content_item_dir: str,
del_unified: bool = True,
source_mapping: Union[Dict[str, Dict[str, str]]] = None):
"""
Iterate over the YAML files in a directory and create packages (a containing directory and
component files) from the YAMLs of integrations and scripts
Args:
content_item_dir (str): Path to the directory containing the content item YAML file(s)
del_unified (bool): Whether to delete the unified yaml the package was extracted from
source_mapping (Union[Dict], optional): Can be used when updating an existing pack and
the package directory of a content item is not what would ordinarily be set by the
`demisto-sdk` `split` command. Sample value would be,
`{'integration-AbuseIPDB.yml': {'containing_dir_name': 'AbuseDB', 'base_name': 'AbuseDB'}}`
- the split command would create a containing directory of `AbuseIPDB` for the file
`integration-AbuseIPDB.yml` and we need the containing directory of the package to match
what already exists in the repo.
"""
child_files = get_child_files(content_item_dir)
for child_file in child_files:
cf_name_lower = os.path.basename(child_file).lower()
if cf_name_lower.startswith(
(SCRIPT, AUTOMATION,
INTEGRATION)) and cf_name_lower.endswith('yml'):
content_item_file_path = child_file
file_type = find_type(content_item_file_path)
file_type = file_type.value if file_type else file_type
try:
child_file_name = os.path.basename(child_file)
if source_mapping and child_file_name in source_mapping.keys(
):
child_file_mapping = source_mapping.get(
child_file_name, {})
base_name = child_file_mapping.get('base_name', '')
containing_dir_name = child_file_mapping.get(
'containing_dir_name', '')
# for legacy unified yamls in the repo, their containing directory will be that of their
# entity type directly instead of the typical package format. For those cases, we need the
# extractor to auto create the containing directory. An example would be -
# 'content/Packs/AbuseDB/Scripts/script-AbuseIPDBPopulateIndicators.yml'
autocreate_dir = containing_dir_name == ENTITY_TYPE_TO_DIR.get(
file_type, '')
output_dir = os.path.join(
self.pack_dir_path,
ENTITY_TYPE_TO_DIR.get(file_type, ''))
if not autocreate_dir:
output_dir = os.path.join(output_dir,
containing_dir_name)
os.makedirs(output_dir, exist_ok=True)
extractor = YmlSplitter(
input=content_item_file_path,
file_type=file_type,
output=output_dir,
no_readme=True,
base_name=base_name,
no_auto_create_dir=(not autocreate_dir),
no_pipenv=self.no_pipenv)
else:
extractor = YmlSplitter(input=content_item_file_path,
file_type=file_type,
output=content_item_dir,
no_pipenv=self.no_pipenv)
extractor.extract_to_package_format()
except Exception as e:
err_msg = f'Error occurred while trying to split the unified YAML "{content_item_file_path}" ' \
f'into its component parts.\nError: "{e}"'
self.contrib_conversion_errs.append(err_msg)
finally:
output_path = extractor.get_output_path()
if self.create_new:
# Moving the unified file to its package.
shutil.move(content_item_file_path, output_path)
if del_unified:
if os.path.exists(content_item_file_path):
os.remove(content_item_file_path)
moved_unified_dst = os.path.join(
output_path, child_file_name)
if os.path.exists(moved_unified_dst):
os.remove(moved_unified_dst)
def convert_contribution_to_pack(self):
"""Create a Pack in the content repo from the contents of a contribution zipfile"""
try:
packs_dir = os.path.join(get_content_path(), 'Packs')
metadata_dict = {}
with zipfile.ZipFile(self.contribution) as zipped_contrib:
with zipped_contrib.open('metadata.json') as metadata_file:
click.echo(
f'Pulling relevant information from {metadata_file.name}',
color=LOG_COLORS.NATIVE)
metadata = json.loads(metadata_file.read())
# a name passed on the cmd line should take precedence over one pulled
# from contribution metadata
pack_name = self.name or self.format_pack_dir_name(
metadata.get('name', 'ContributionPack'))
# a description passed on the cmd line should take precedence over one pulled
# from contribution metadata
metadata_dict[
'description'] = self.description or metadata.get(
'description')
metadata_dict['name'] = pack_name
metadata_dict['author'] = metadata.get('author', '')
metadata_dict['support'] = metadata.get('support', '')
metadata_dict['url'] = metadata.get('supportDetails',
{}).get('url', '')
metadata_dict['email'] = metadata.get(
'supportDetails', {}).get('email', '')
metadata_dict['categories'] = metadata.get(
'categories') if metadata.get('categories') else []
metadata_dict['tags'] = metadata.get(
'tags') if metadata.get('tags') else []
metadata_dict['useCases'] = metadata.get(
'useCases') if metadata.get('useCases') else []
metadata_dict['keywords'] = metadata.get(
'keywords') if metadata.get('keywords') else []
while os.path.exists(os.path.join(packs_dir, pack_name)):
click.echo(
f'Modifying pack name because pack {pack_name} already exists in the content repo',
color=LOG_COLORS.NATIVE)
if len(pack_name) >= 2 and pack_name[-2].lower(
) == 'v' and pack_name[-1].isdigit():
# increment by one
pack_name = pack_name[:-1] + str(int(pack_name[-1]) + 1)
else:
pack_name += 'V2'
click.echo(f'New pack name is "{pack_name}"',
color=LOG_COLORS.NATIVE)
pack_dir = os.path.join(packs_dir, pack_name)
os.mkdir(pack_dir)
shutil.unpack_archive(filename=self.contribution,
extract_dir=pack_dir)
pack_subdirectories = get_child_directories(pack_dir)
for pack_subdir in pack_subdirectories:
basename = os.path.basename(pack_subdir)
if basename in ENTITY_TYPE_TO_DIR:
dst_name = ENTITY_TYPE_TO_DIR.get(basename)
src_path = os.path.join(pack_dir, basename)
dst_path = os.path.join(pack_dir, dst_name)
content_item_dir = shutil.move(src_path, dst_path)
if basename in {SCRIPT, AUTOMATION, INTEGRATION}:
self.content_item_to_package_format(content_item_dir,
del_unified=True)
# create pack's base files
self.full_output_path = pack_dir
self.create_pack_base_files()
metadata_dict = Initiator.create_metadata(fill_manually=False,
data=metadata_dict)
metadata_path = os.path.join(self.full_output_path,
'pack_metadata.json')
with open(metadata_path, 'w') as pack_metadata_file:
json.dump(metadata_dict, pack_metadata_file, indent=4)
# remove metadata.json file
os.remove(os.path.join(pack_dir, 'metadata.json'))
click.echo(
f'Executing \'format\' on the restructured contribution zip files at "{pack_dir}"'
)
format_manager(input=pack_dir)
except Exception as e:
click.echo(
f'Creating a Pack from the contribution zip failed with error: {e}\n {traceback.format_exc()}',
color=LOG_COLORS.RED)
finally:
if self.contrib_conversion_errs:
click.echo(
'The following errors occurred while converting unified content YAMLs to package structure:'
)
click.echo(
textwrap.indent('\n'.join(self.contrib_conversion_errs),
'\t'))
def update_tests(self) -> None:
"""
If there are no tests configured: Prompts a question to the cli that asks the user whether he wants to add
'No tests' under 'tests' key or not and format the file according to the answer
"""
if not self.data.get('tests', ''):
# try to get the test playbook files from the TestPlaybooks dir in the pack
pack_path = os.path.dirname(
os.path.dirname(os.path.abspath(self.source_file)))
test_playbook_dir_path = os.path.join(pack_path,
TEST_PLAYBOOKS_DIR)
test_playbook_ids = []
file_entity_type = find_type(self.source_file,
_dict=self.data,
file_type='yml')
file_id = get_entity_id_by_entity_type(
self.data, ENTITY_TYPE_TO_DIR.get(file_entity_type.value, ""))
commands, scripts = get_scripts_and_commands_from_yml_data(
self.data, file_entity_type)
commands_names = [command.get('id') for command in commands]
try:
# Collecting the test playbooks
test_playbooks_files = [
tpb_file
for tpb_file in listdir_fullpath(test_playbook_dir_path)
if find_type(tpb_file) == FileType.TEST_PLAYBOOK
]
for tpb_file_path in test_playbooks_files: # iterate over the test playbooks in the dir
test_playbook_data = get_yaml(tpb_file_path)
test_playbook_id = get_entity_id_by_entity_type(
test_playbook_data, content_entity='')
if not scripts and not commands: # Better safe than sorry
test_playbook_ids.append(test_playbook_id)
else:
added = False
tpb_commands, tpb_scripts = get_scripts_and_commands_from_yml_data(
test_playbook_data, FileType.TEST_PLAYBOOK)
for tpb_command in tpb_commands:
tpb_command_name = tpb_command.get('id')
tpb_command_source = tpb_command.get('source', '')
if tpb_command_source and file_id and file_id != tpb_command_source:
continue
if not added and tpb_command_name in commands_names:
command_source = commands[commands_names.index(
tpb_command_name)].get('source', '')
if command_source == tpb_command_source or command_source == '':
test_playbook_ids.append(test_playbook_id)
added = True
break
if not added:
for tpb_script in tpb_scripts:
if tpb_script in scripts:
test_playbook_ids.append(test_playbook_id)
break
self.data['tests'] = test_playbook_ids
except FileNotFoundError:
pass
if not test_playbook_ids:
# In case no_interactive flag was given - modify the tests without confirmation
if self.assume_yes or not self.add_tests:
should_modify_yml_tests = True
else:
should_modify_yml_tests = click.confirm(
f'The file {self.source_file} has no test playbooks '
f'configured. Do you want to configure it with "No tests"?'
)
if should_modify_yml_tests:
click.echo(
f'Formatting {self.output_file} with "No tests"')
self.data['tests'] = ['No tests (auto formatted)']
