class Pack:
def __init__(self, path: Union[str, Path]):
self._path = Path(path)
self._metadata = PackMetaData(self._path.joinpath('metadata.json'))
def _content_files_list_generator_factory(self, dir_name: str,
suffix: str) -> Iterator[Any]:
"""Generic content objcets iterable generator
Args:
dir_name: Directory name, for example: Integrations, Documentations etc.
suffix: file suffix to search for, if not supplied then any suffix.
Returns:
object: Any valid content object found in the given directory.
"""
objects_path = (self._path / dir_name).glob(
patterns=[f"*.{suffix}", f"*/*.{suffix}"])
for object_path in objects_path:
yield path_to_pack_object(object_path)
def _content_dirs_list_generator_factory(self, dir_name) -> Iterator[Any]:
"""Generic content objects iterable generator
Args:
dir_name: Directory name, for example: Tools.
Returns:
object: Any valid content object found in the given directory.
"""
objects_path = (self._path / dir_name).glob(patterns=["*/"])
for object_path in objects_path:
yield path_to_pack_object(object_path)
@property
def id(self) -> str:
return self._path.parts[-1]
@property
def path(self) -> Path:
return self._path
@property
def integrations(self) -> Iterator[Integration]:
return self._content_files_list_generator_factory(
dir_name=INTEGRATIONS_DIR, suffix="yml")
@property
def scripts(self) -> Iterator[Script]:
return self._content_files_list_generator_factory(dir_name=SCRIPTS_DIR,
suffix="yml")
@property
def playbooks(self) -> Iterator[Playbook]:
return self._content_files_list_generator_factory(
dir_name=PLAYBOOKS_DIR, suffix="yml")
@property
def reports(self) -> Iterator[Report]:
return self._content_files_list_generator_factory(dir_name=REPORTS_DIR,
suffix="json")
@property
def dashboards(self) -> Iterator[Dashboard]:
return self._content_files_list_generator_factory(
dir_name=DASHBOARDS_DIR, suffix="json")
@property
def incident_types(self) -> Iterator[IncidentType]:
return self._content_files_list_generator_factory(
dir_name=INCIDENT_TYPES_DIR, suffix="json")
@property
def incident_fields(self) -> Iterator[IncidentField]:
return self._content_files_list_generator_factory(
dir_name=INCIDENT_FIELDS_DIR, suffix="json")
@property
def layouts(self) -> Iterator[Layout]:
return self._content_files_list_generator_factory(dir_name=LAYOUTS_DIR,
suffix="json")
@property
def classifiers(
self
) -> Iterator[Union[Classifier, OldClassifier, ClassifierMapper]]:
return self._content_files_list_generator_factory(
dir_name=CLASSIFIERS_DIR, suffix="json")
@property
def indicator_types(self) -> Iterator[IndicatorType]:
return self._content_files_list_generator_factory(
dir_name=INDICATOR_TYPES_DIR, suffix="json")
@property
def indicator_fields(self) -> Iterator[IndicatorField]:
return self._content_files_list_generator_factory(
dir_name=INDICATOR_FIELDS_DIR, suffix="json")
@property
def connections(self) -> Iterator[Connection]:
return self._content_files_list_generator_factory(
dir_name=CONNECTIONS_DIR, suffix="json")
@property
def test_playbooks(self) -> Iterator[Union[Playbook, Script]]:
return self._content_files_list_generator_factory(
dir_name=TEST_PLAYBOOKS_DIR, suffix="yml")
@property
def widgets(self) -> Iterator[Widget]:
return self._content_files_list_generator_factory(dir_name=WIDGETS_DIR,
suffix="json")
@property
def release_notes(self) -> Iterator[ReleaseNote]:
return self._content_files_list_generator_factory(
dir_name=RELEASE_NOTES_DIR, suffix="md")
@property
def tools(self) -> Iterator[AgentTool]:
return self._content_dirs_list_generator_factory(dir_name=TOOLS_DIR)
@property
def doc_files(self) -> Iterator[DocFile]:
return self._content_files_list_generator_factory(
dir_name=DOC_FILES_DIR, suffix="*")
@property
def pack_metadata(self) -> Optional[PackMetaData]:
obj = None
file = self._path / "pack_metadata.json"
if file.exists():
obj = PackMetaData(file)
return obj
@property
def metadata(self) -> PackMetaData:
return self._metadata
@metadata.setter
def metadata(self, metadata: Optional[PackMetaData]):
self._metadata = metadata
@property
def secrets_ignore(self) -> Optional[SecretIgnore]:
obj = None
file = self._path / ".secrets-ignore"
if file.exists():
obj = SecretIgnore(file)
return obj
@property
def pack_ignore(self) -> Optional[PackIgnore]:
obj = None
file = self._path / ".pack-ignore"
if file.exists():
obj = PackIgnore(file)
return obj
@property
def readme(self) -> Optional[Readme]:
obj = None
file = self._path / "README.md"
if file.exists():
obj = Readme(file)
return obj
@property
def author_image(self) -> Optional[AuthorImage]:
obj = None
file = self._path / "Author_image.png"
if file.exists():
obj = AuthorImage(file)
return obj
def sign_pack(self, logger: logging.Logger, dumped_pack_dir: Path,
sign_directory: Path):
""" Signs pack folder and creates signature file.
Args:
logger (logging.Logger): System logger already initialized.
dumped_pack_dir (Path): Path to the updated pack to sign.
sign_directory (Path): Path to the signDirectory executable file.
"""
try:
full_command = f'{sign_directory} {dumped_pack_dir} keyfile base64'
signing_process = subprocess.Popen(full_command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
output, err = signing_process.communicate()
signing_process.wait()
if err:
logger.error(
f'Failed to sign pack for {self.path.name} - {str(err)}')
return
logger.info(f'Signed {self.path.name} pack successfully')
except Exception as error:
logger.error(
f'Error while trying to sign pack {self.path.name}.\n {error}')
class Pack:
def __init__(self, path: Union[str, Path]):
self._path = Path(path)
# in case the given path are a Pack and not zipped pack - we init the metadata from the pack
if not str(path).endswith('.zip'):
self._metadata = PackMetaData(self._path.joinpath('metadata.json'))
self._filter_items_by_id_set = False
self._pack_info_from_id_set: Dict[Any, Any] = {}
def _content_files_list_generator_factory(self, dir_name: str, suffix: str) -> Iterator[Any]:
"""Generic content objects iterable generator
Args:
dir_name: Directory name, for example: Integrations, Documentations etc.
suffix: file suffix to search for, if not supplied then any suffix.
Returns:
object: Any valid content object found in the given directory.
"""
objects_path = (self._path / dir_name).glob(patterns=[f"*.{suffix}", f"*/*.{suffix}"])
for object_path in objects_path:
content_object = path_to_pack_object(object_path)
# skip content items that are not displayed in the id set, if the corresponding flag is used,
# We excluding ReleaseNotes and TestPlaybooks, because they missing from the id set
# but are needed in the pack's zip.
if self._filter_items_by_id_set and content_object.type().value not in [FileType.RELEASE_NOTES.value,
FileType.RELEASE_NOTES_CONFIG.value,
FileType.TEST_PLAYBOOK.value]:
object_id = content_object.get_id()
if is_object_in_id_set(object_id, self._pack_info_from_id_set):
yield content_object
else:
logging.warning(f'Skipping object {object_path} with id "{object_id}" since it\'s missing from '
f'the given id set')
else:
yield content_object
def _content_dirs_list_generator_factory(self, dir_name) -> Iterator[Any]:
"""Generic content objects iterable generator
Args:
dir_name: Directory name, for example: Tools.
Returns:
object: Any valid content object found in the given directory.
"""
objects_path = (self._path / dir_name).glob(patterns=["*/"])
for object_path in objects_path:
yield path_to_pack_object(object_path)
@property
def id(self) -> str:
return self._path.parts[-1]
@property
def path(self) -> Path:
return self._path
@property
def integrations(self) -> Iterator[Integration]:
return self._content_files_list_generator_factory(dir_name=INTEGRATIONS_DIR,
suffix="yml")
@property
def scripts(self) -> Iterator[Script]:
return self._content_files_list_generator_factory(dir_name=SCRIPTS_DIR,
suffix="yml")
@property
def playbooks(self) -> Iterator[Playbook]:
return self._content_files_list_generator_factory(dir_name=PLAYBOOKS_DIR,
suffix="yml")
@property
def reports(self) -> Iterator[Report]:
return self._content_files_list_generator_factory(dir_name=REPORTS_DIR,
suffix="json")
@property
def dashboards(self) -> Iterator[Dashboard]:
return self._content_files_list_generator_factory(dir_name=DASHBOARDS_DIR,
suffix="json")
@property
def incident_types(self) -> Iterator[IncidentType]:
return self._content_files_list_generator_factory(dir_name=INCIDENT_TYPES_DIR,
suffix="json")
@property
def incident_fields(self) -> Iterator[IncidentField]:
return self._content_files_list_generator_factory(dir_name=INCIDENT_FIELDS_DIR,
suffix="json")
@property
def layouts(self) -> Iterator[LayoutObject]:
return self._content_files_list_generator_factory(dir_name=LAYOUTS_DIR,
suffix="json")
@property
def pre_process_rules(self) -> Iterator[PreProcessRule]:
return self._content_files_list_generator_factory(dir_name=PRE_PROCESS_RULES_DIR,
suffix="json")
@property
def lists(self) -> Iterator[Lists]:
return self._content_files_list_generator_factory(dir_name=LISTS_DIR,
suffix="json")
@property
def classifiers(self) -> Iterator[Union[Classifier, OldClassifier, ClassifierMapper]]:
return self._content_files_list_generator_factory(dir_name=CLASSIFIERS_DIR,
suffix="json")
@property
def indicator_types(self) -> Iterator[IndicatorType]:
return self._content_files_list_generator_factory(dir_name=INDICATOR_TYPES_DIR,
suffix="json")
@property
def indicator_fields(self) -> Iterator[IndicatorField]:
return self._content_files_list_generator_factory(dir_name=INDICATOR_FIELDS_DIR,
suffix="json")
@property
def connections(self) -> Iterator[Connection]:
return self._content_files_list_generator_factory(dir_name=CONNECTIONS_DIR,
suffix="json")
@property
def test_playbooks(self) -> Iterator[Union[Playbook, Script]]:
return self._content_files_list_generator_factory(dir_name=TEST_PLAYBOOKS_DIR,
suffix="yml")
@property
def widgets(self) -> Iterator[Widget]:
return self._content_files_list_generator_factory(dir_name=WIDGETS_DIR,
suffix="json")
@property
def release_notes(self) -> Iterator[ReleaseNote]:
return self._content_files_list_generator_factory(dir_name=RELEASE_NOTES_DIR,
suffix="md")
@property
def release_notes_config(self) -> Iterator[ReleaseNoteConfig]:
return self._content_files_list_generator_factory(dir_name=RELEASE_NOTES_DIR,
suffix="json")
@property
def generic_definitions(self) -> Iterator[GenericDefinition]:
return self._content_files_list_generator_factory(dir_name=GENERIC_DEFINITIONS_DIR,
suffix="json")
@property
def generic_modules(self) -> Iterator[GenericModule]:
return self._content_files_list_generator_factory(dir_name=GENERIC_MODULES_DIR,
suffix="json")
@property
def generic_types(self) -> Iterator[GenericType]:
return self._content_files_list_generator_factory(dir_name=GENERIC_TYPES_DIR,
suffix="json")
@property
def generic_fields(self) -> Iterator[GenericField]:
return self._content_files_list_generator_factory(dir_name=GENERIC_FIELDS_DIR,
suffix="json")
@property
def tools(self) -> Iterator[AgentTool]:
return self._content_dirs_list_generator_factory(dir_name=TOOLS_DIR)
@property
def doc_files(self) -> Iterator[DocFile]:
return self._content_files_list_generator_factory(dir_name=DOC_FILES_DIR,
suffix="*")
@property
def jobs(self) -> Iterator[Job]:
return self._content_files_list_generator_factory(JOBS_DIR,
suffix="json")
@property
def parsing_rules(self) -> Iterator[ParsingRule]:
return self._content_files_list_generator_factory(dir_name=PARSING_RULES_DIR,
suffix="yml")
@property
def modeling_rules(self) -> Iterator[ModelingRule]:
return self._content_files_list_generator_factory(dir_name=MODELING_RULES_DIR,
suffix="yml")
@property
def correlation_rules(self) -> Iterator[CorrelationRule]:
return self._content_files_list_generator_factory(dir_name=CORRELATION_RULES_DIR,
suffix="yml")
@property
def xsiam_dashboards(self) -> Iterator[XSIAMDashboard]:
return self._content_files_list_generator_factory(dir_name=XSIAM_DASHBOARDS_DIR,
suffix="json")
@property
def xsiam_reports(self) -> Iterator[XSIAMReport]:
return self._content_files_list_generator_factory(dir_name=XSIAM_REPORTS_DIR,
suffix="json")
@property
def triggers(self) -> Iterator[Trigger]:
return self._content_files_list_generator_factory(dir_name=TRIGGER_DIR,
suffix="json")
@property
def pack_metadata(self) -> Optional[PackMetaData]:
obj = None
file = self._path / "pack_metadata.json"
if file.exists():
obj = PackMetaData(file)
return obj
@property
def metadata(self) -> PackMetaData:
return self._metadata
@metadata.setter
def metadata(self, metadata: Optional[PackMetaData]):
self._metadata = metadata
@property
def secrets_ignore(self) -> Optional[SecretIgnore]:
obj = None
file = self._path / ".secrets-ignore"
if file.exists():
obj = SecretIgnore(file)
return obj
@property
def pack_ignore(self) -> Optional[PackIgnore]:
obj = None
file = self._path / ".pack-ignore"
if file.exists():
obj = PackIgnore(file)
return obj
@property
def readme(self) -> Optional[Readme]:
obj = None
file = self._path / "README.md"
if file.exists():
obj = Readme(path=file)
return obj
@property
def author_image(self) -> Optional[AuthorImage]:
obj = None
file = self._path / "Author_image.png"
if file.exists():
obj = AuthorImage(file)
return obj
@property
def contributors(self) -> Optional[Contributors]:
obj = None
file = self._path / "CONTRIBUTORS.md"
if file.exists():
obj = Contributors(path=file)
return obj
@property
def filter_items_by_id_set(self) -> bool:
return self._filter_items_by_id_set
@filter_items_by_id_set.setter
def filter_items_by_id_set(self, filter_by_id_set: bool):
self._filter_items_by_id_set = filter_by_id_set
@property
def pack_info_from_id_set(self) -> dict:
return self._pack_info_from_id_set
@pack_info_from_id_set.setter
def pack_info_from_id_set(self, pack_section_from_id_set: dict):
self._pack_info_from_id_set = pack_section_from_id_set.get(self.id, {}) if pack_section_from_id_set else {}
def sign_pack(self, logger: logging.Logger, dumped_pack_dir: Path, sign_directory: Path):
""" Signs pack folder and creates signature file.
Args:
logger (logging.Logger): System logger already initialized.
dumped_pack_dir (Path): Path to the updated pack to sign.
sign_directory (Path): Path to the signDirectory executable file.
"""
try:
full_command = f'{sign_directory} {dumped_pack_dir} keyfile base64'
signing_process = subprocess.Popen(full_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
output, err = signing_process.communicate()
signing_process.wait()
if err:
logger.error(f'Failed to sign pack for {self.path.name} - {str(err)}')
return
logger.info(f'Signed {self.path.name} pack successfully')
except Exception as error:
logger.error(f'Error while trying to sign pack {self.path.name}.\n {error}')
def is_server_version_ge(self, client, server_version_to_check):
server_version = get_demisto_version(client)
return LooseVersion(server_version.base_version) >= LooseVersion(server_version_to_check) # type: ignore
def upload(self, logger: logging.Logger, client: demisto_client, skip_validation: bool):
"""
Upload the pack zip to demisto_client,
from 6.5 server version we have the option to use skip_verify arg instead of server configuration.
Args:
logger (logging.Logger): System logger already initialized.
client: The demisto_client object of the desired XSOAR machine to upload to.
skip_validation: if true will skip upload packs validation.
Returns:
The result of the upload command from demisto_client
"""
if self.is_server_version_ge(client, '6.6.0') and skip_validation:
try:
logger.info('Uploading...')
return client.upload_content_packs(
file=self.path, skip_verify='true', skip_validation='true') # type: ignore
except Exception as err:
raise Exception(f'Failed to upload pack, error: {err}')
if self.is_server_version_ge(client, '6.5.0'):
try:
logger.info('Uploading...')
return client.upload_content_packs(file=self.path, skip_verify='true') # type: ignore
except Exception as err:
raise Exception(f'Failed to upload pack, error: {err}')
# the flow are - turn off the sign check -> upload -> turn back the check to be as previously
logger.info('Turn off the server verification for signed packs')
_, _, prev_conf = tools.update_server_configuration(client=client,
server_configuration={PACK_VERIFY_KEY: 'false'},
error_msg='Can not turn off the pack verification')
try:
logger.info('Uploading...')
return client.upload_content_packs(file=self.path) # type: ignore
finally:
config_keys_to_update = None
config_keys_to_delete = None
try:
prev_key_val = prev_conf.get(PACK_VERIFY_KEY, None)
if prev_key_val is not None:
config_keys_to_update = {PACK_VERIFY_KEY: prev_key_val}
else:
config_keys_to_delete = {PACK_VERIFY_KEY}
logger.info('Setting the server verification to be as previously')
tools.update_server_configuration(client=client,
server_configuration=config_keys_to_update,
config_keys_to_delete=config_keys_to_delete,
error_msg='Can not turn on the pack verification')
except (Exception, KeyboardInterrupt):
action = DELETE_VERIFY_KEY_ACTION if prev_key_val is None \
else SET_VERIFY_KEY_ACTION.format(prev_key_val)
raise Exception(TURN_VERIFICATION_ERROR_MSG.format(action=action))
