def zipped_pack_uploader(self, path: str, skip_validation: bool) -> int:
zipped_pack = Pack(path)
try:
logger = logging.getLogger('demisto-sdk')
if not self.pack_names:
self.pack_names = [zipped_pack.path.stem]
if self.notify_user_should_override_packs():
zipped_pack.upload(logger, self.client, skip_validation)
self.successfully_uploaded_files.extend([
(pack_name, FileType.PACK.value)
for pack_name in self.pack_names
])
return SUCCESS_RETURN_CODE
return ABORTED_RETURN_CODE
except (Exception, KeyboardInterrupt) as err:
file_name = zipped_pack.path.name # type: ignore
message = parse_error_response(err, FileType.PACK.value, file_name,
self.log_verbose)
self.failed_uploaded_files.append(
(file_name, FileType.PACK.value, message))
return ERROR_RETURN_CODE
def test_convert_dir(self, tmpdir):
"""
Given:
- Pack.
When:
- Converting every layout of version 6.0.0 and above to version 5.9.9 and below.
Then:
- Ensure expected layouts are created with expected values.
"""
fake_pack_name = 'FakeTestPack'
repo = Repo(tmpdir)
repo_path = Path(repo.path)
fake_pack = MockPack(repo_path / 'Packs', fake_pack_name, repo)
fake_pack.create_layout('close-ExtraHop_Detection', util_load_json(self.LAYOUT_CLOSE_PATH))
fake_pack.create_layout('details-ExtraHop_Detection', util_load_json(self.LAYOUT_DETAILS_PATH))
fake_pack.create_layout('edit-ExtraHop_Detection', util_load_json(self.LAYOUT_EDIT_PATH))
fake_pack.create_layout('quickView-ExtraHop_Detection',
util_load_json(self.LAYOUT_QUICK_VIEW_PATH))
fake_pack.create_layout('mobile-ExtraHop_Detection', util_load_json(self.LAYOUT_MOBILE_PATH))
fake_pack_path = fake_pack.path
layout_converter = LayoutSixConverter(Pack(fake_pack_path))
layout_converter.convert_dir()
expected_new_layout_path = f'{str(layout_converter.pack.path)}/Layouts/layoutscontainer-ExtraHop_Detection.json'
assert os.path.exists(expected_new_layout_path)
with open(expected_new_layout_path, 'r') as f:
layout_data = json.loads(f.read())
test_data_json = util_load_json(os.path.join(__file__,
f'{git_path()}/demisto_sdk/commands/convert/converters/layout/'
'tests/test_data'
'/layoutscontainer-ExtraHop_Detection.json'))
assert layout_data == test_data_json
os.remove(expected_new_layout_path)
def test_create_mapper_from_old_classifier(self, tmpdir):
"""
Given:
- 'old_classifier': Old classifier to convert into 6_0_0 classifier.
When:
- Creating a 6_0_0 mapper convention from 5_9_9 and below classifier
Then:
- Ensure expected mapper is created in the expected path with the expected data.
"""
fake_pack_name = 'FakeTestPack'
repo = Repo(tmpdir)
repo_path = Path(repo.path)
fake_pack = MockPack(repo_path / 'Packs', fake_pack_name, repo)
fake_pack.create_classifier('Cymulate_5_9_9',
util_load_json(self.OLD_CLASSIFIER_PATH))
fake_pack_path = fake_pack.path
converter = ClassifierSixConverter(Pack(fake_pack_path))
old_classifier = Classifier(self.OLD_CLASSIFIER_PATH)
converter.create_mapper_from_old_classifier(old_classifier)
mapper_path = f'{tmpdir}/Packs/FakeTestPack/Classifiers/classifier-mapper-incoming-Cymulate.json'
self.assert_expected_file_output(
mapper_path, 'classifier-mapper-incoming-Cymulate')
def test_load_user_metadata_invalid_price(repo, capsys):
"""
When:
- Dumping a pack with invalid price in pack_metadata file.
Given:
- Pack object.
Then:
- Verify that exceptions are written to the logger.
"""
import demisto_sdk.commands.common.content.objects.pack_objects.pack_metadata.pack_metadata as metadata_class
metadata_class.logger = logging_setup(3)
pack_1 = repo.setup_one_pack('Pack1')
pack_1.pack_metadata.write_json({
'name': 'Pack Number 1',
'price': 'price',
'tags': ['tag1'],
'useCases': ['usecase1'],
'vendorId': 'vendorId',
'vendorName': 'vendorName'
})
content_object_pack = Pack(pack_1.path)
pack_1_metadata = content_object_pack.metadata
pack_1_metadata.load_user_metadata('Pack1', 'Pack Number 1', pack_1.path,
logging_setup(3))
captured = capsys.readouterr()
assert 'Pack Number 1 pack price is not valid. The price was set to 0.' in captured.out
def test_group_layouts_needing_conversion_by_layout_id(self, tmpdir):
"""
Given:
-
When:
- Grouping layouts needing conversion by their layout ID.
Then:
- Ensure expected dict object is returned.
"""
fake_pack_name = 'FakeTestPack'
repo = Repo(tmpdir)
repo_path = Path(repo.path)
fake_pack = MockPack(repo_path / 'Packs', fake_pack_name, repo)
fake_pack.create_layout('close-ExtraHop_Detection', util_load_json(self.LAYOUT_CLOSE_PATH))
fake_pack.create_layout('details-ExtraHop_Detection', util_load_json(self.LAYOUT_DETAILS_PATH))
fake_pack.create_layout('edit-ExtraHop_Detection', util_load_json(self.LAYOUT_EDIT_PATH))
fake_pack.create_layout('quickView-ExtraHop_Detection',
util_load_json(self.LAYOUT_QUICK_VIEW_PATH))
fake_pack.create_layout('mobile-ExtraHop_Detection', util_load_json(self.LAYOUT_MOBILE_PATH))
fake_pack_path = fake_pack.path
layout_converter = LayoutSixConverter(Pack(fake_pack_path))
result = layout_converter.group_layouts_needing_conversion_by_layout_id()
assert len(result) == 1 and 'ExtraHop Detection' in result
layout_kinds = {layout['kind'] for layout in result['ExtraHop Detection']}
assert all(layout.layout_id() == 'ExtraHop Detection' for layout in result['ExtraHop Detection'])
assert layout_kinds == {'close', 'details', 'edit', 'mobile', 'quickView'}
def test_load_user_metadata_bad_pack_metadata_file(repo, capsys):
"""
When:
- Dumping a pack with invalid pack_metadata file.
Given:
- Pack object.
Then:
- Verify that exceptions are written to the logger.
"""
import demisto_sdk.commands.common.content.objects.pack_objects.pack_metadata.pack_metadata as metadata_class
metadata_class.logger = logging_setup(3)
pack_1 = repo.setup_one_pack('Pack1')
pack_1.pack_metadata.write_as_text('Invalid of course {')
content_object_pack = Pack(pack_1.path)
pack_1_metadata = content_object_pack.metadata
pack_1_metadata.load_user_metadata('Pack1', 'Pack Number 1', pack_1.path,
logging_setup(3))
captured = capsys.readouterr()
assert 'Failed loading Pack Number 1 user metadata.' in captured.out
def test_convert_dir(self, tmpdir):
"""
Given:
- Pack.
When:
- Converting every layout of version 6.0.0 and above to version 5.9.9 and below.
Then:
- Ensure expected layouts are created with expected values.
"""
fake_pack_name = 'FakeTestPack'
repo = Repo(tmpdir)
repo_path = Path(repo.path)
fake_pack = MockPack(repo_path / 'Packs', fake_pack_name, repo)
fake_pack.create_incident_type('ExtraHop_Detection',
util_load_json(self.INCIDENT_TYPE_ONE))
fake_pack.create_layoutcontainer('ExtraHop Detection',
util_load_json(self.LAYOUT_CONTAINER))
fake_pack_path = fake_pack.path
layout_converter = LayoutBelowSixConverter(Pack(fake_pack_path))
layout_converter.convert_dir()
test_data_json = util_load_json(
os.path.join(
__file__,
f'{git_path()}/demisto_sdk/commands/convert/converters/layout/'
'tests/test_data'
'/layout_up_to_5_9_9_expected_convert_dir_test_file_output.json'
))
for layout_field_name, layout_data in test_data_json.items():
expected_new_layout_path = f'{str(layout_converter.pack.path)}/Layouts/layout-{layout_field_name}-' \
'ExtraHop_Detection.json'
assert os.path.exists(expected_new_layout_path)
assert util_load_json(expected_new_layout_path) == layout_data
os.remove(expected_new_layout_path)
def test_get_layouts_by_layout_type(self, tmpdir):
"""
Given:
- Layout FileType.
When:
- Wanting to retrieve all layout below 6.0.0 version from the current pack.
Then:
- Ensure only layouts below 6.0.0 version in the pack are returned.
"""
fake_pack_name = 'FakeTestPack'
repo = Repo(tmpdir)
repo_path = Path(repo.path)
fake_pack = MockPack(repo_path / 'Packs', fake_pack_name, repo)
fake_pack.create_layout('close-ExtraHop_Detection',
util_load_json(self.LAYOUT_CLOSE_PATH))
fake_pack.create_layout('details-ExtraHop_Detection',
util_load_json(self.LAYOUT_DETAILS_PATH))
fake_pack.create_layout('edit-ExtraHop_Detection',
util_load_json(self.LAYOUT_EDIT_PATH))
fake_pack.create_layout('quickView-ExtraHop_Detection',
util_load_json(self.LAYOUT_QUICK_VIEW_PATH))
fake_pack.create_layout('mobile-ExtraHop_Detection',
util_load_json(self.LAYOUT_MOBILE_PATH))
fake_pack_path = fake_pack.path
layouts = BaseConverter.get_entities_by_entity_type(
Pack(fake_pack_path).layouts, FileType.LAYOUT)
assert len(layouts) == 5
assert all(layout.type() == FileType.LAYOUT for layout in layouts)
assert {layout.get('kind')
for layout in layouts
} == {'close', 'details', 'edit', 'mobile', 'quickView'}
def test_create_classifier_from_old_classifier(self, tmpdir):
"""
Given:
- 'old_classifier': Old classifier to convert into 6_0_0 classifier.
- 'intersection_fields': List of the intersecting fields of classifiers 5_9_9 and below to classifiers 6_0_0.
When:
- Creating a 6_0_0 classifier convention from 5_9_9 and below classifier
Then:
- Ensure expected classifier is created in the expected path with the expected data.
"""
fake_pack_name = 'FakeTestPack'
repo = Repo(tmpdir)
repo_path = Path(repo.path)
fake_pack = MockPack(repo_path / 'Packs', fake_pack_name, repo)
fake_pack.create_classifier('Cymulate_5_9_9',
util_load_json(self.OLD_CLASSIFIER_PATH))
fake_pack_path = fake_pack.path
converter = ClassifierSixConverter(Pack(fake_pack_path))
old_classifier = Classifier(self.OLD_CLASSIFIER_PATH)
intersecting_fields = converter.get_classifiers_schema_intersection_fields(
)
converter.create_classifier_from_old_classifier(
old_classifier, intersecting_fields)
classifier_expected_path = f'{tmpdir}/Packs/FakeTestPack/Classifiers/classifier-Cymulate.json'
self.assert_expected_file_output(classifier_expected_path,
'classifier-Cymulate')
def test_layout_to_incidents_dict(self, tmpdir):
"""
Given:
- Incident types of the pack.
When:
- Creating a dict of key as layout ID and value as list of incident type IDs whom layout field equals to layout
ID.
Then:
- Ensure expected dict is returned.
"""
fake_pack_name = 'FakeTestPack'
repo = Repo(tmpdir)
repo_path = Path(repo.path)
fake_pack = MockPack(repo_path / 'Packs', fake_pack_name, repo)
fake_pack.create_incident_type('ExtraHop_Detection',
util_load_json(self.INCIDENT_TYPE_ONE))
fake_pack.create_incident_type('ExtraHop_Detection_2',
util_load_json(self.INCIDENT_TYPE_TWO))
fake_pack_path = fake_pack.path
layout_converter = LayoutBelowSixConverter(Pack(fake_pack_path))
result = LayoutBelowSixConverter.layout_to_indicators_or_incidents_dict(
layout_converter.pack.incident_types)
assert result == {
'ExtraHop Detection':
['ExtraHop Detection', 'ExtraHop Detection 2']
}
def packs(self) -> dict[str, Pack]:
"""Packs dictionary as follow:
1. Key - Name.
2. Value - Pack object.
Notes:
1. Could be accessed by Pack id.
2. Itrate over all packs.
3. Only Pack stripped object created not on demend, In order to allow access to pack by id.
"""
return {path.name: Pack(path) for path in (self._path / PACKS_DIR).glob("*/")}
def create_pack_object(self) -> Pack:
"""
Uses self.input_path, returns a Pack object corresponding to the pack given in the path.
Examples:
- self.input_path = 'Packs/BitcoinAbuse/Layouts'
Returns: Pack('Packs/BitcoinAbuse')
- self.input_path = 'Packs/BitcoinAbuse'
Returns: Pack('Packs/BitcoinAbuse')
Returns:
(Pack): Pack object of the pack the conversion was requested for.
"""
pack_path = self.input_path if is_pack_path(
self.input_path) else os.path.dirname(self.input_path)
return Pack(pack_path)
def test_get_layout_indicator_fields(self, tmpdir):
"""
Given:
- Schema path of the layouts-container.
When:
- Wanting to retrieve all indicator dynamic fields in the schema.
Then:
- Ensure indicator dynamic fields are returned.
"""
layout_converter = LayoutSixConverter(Pack(tmpdir))
dynamic_fields = set(layout_converter.get_layout_indicator_fields())
assert dynamic_fields == {'indicatorsDetails', 'indicatorsQuickView'}
def test_get_layout_dynamic_fields(self, tmpdir):
"""
Given:
- Schema path of the layouts-container.
When:
- Wanting to retrieve all dynamic fields in the schema.
Then:
- Ensure dynamic fields are returned.
"""
dynamic_fields = set(LayoutBaseConverter(Pack(tmpdir)).get_layout_dynamic_fields().keys())
assert dynamic_fields == {'close', 'details', 'detailsV2', 'edit', 'indicatorsDetails', 'indicatorsQuickView',
'mobile', 'quickView'}
def test_entity_separators_to_underscore(self, tmpdir, name: str,
expected: str):
"""
Given:
- string, possibly containing one of the chars in 'ENTITY_NAME_SEPARATORS'.
When:
- Wanting to transform every char in 'ENTITY_NAME_SEPARATORS' to '_'.
Then:
- Ensure expected string is returned.
"""
layout_converter = LayoutSixConverter(Pack(tmpdir))
assert layout_converter.entity_separators_to_underscore(
name) == expected
def test_calculate_new_layout_relative_path(self, tmpdir, layout_id: str, expected_suffix: str):
"""
Given:
- 'layout_id': Layout ID of the new layout created.
When:
- Calculating the path to the newly created layout.
Then:
- Ensure the expected path is returned.
"""
layout_converter = LayoutSixConverter(Pack(tmpdir))
expected = f'{layout_converter.pack.path}/Layouts/{expected_suffix}'
assert layout_converter.calculate_new_layout_relative_path(layout_id) == expected
def test_convert_dir(self, tmpdir):
fake_pack_name = 'FakeTestPack'
repo = Repo(tmpdir)
repo_path = Path(repo.path)
fake_pack = MockPack(repo_path / 'Packs', fake_pack_name, repo)
fake_pack.create_classifier('Cymulate_5_9_9',
util_load_json(self.OLD_CLASSIFIER_PATH))
fake_pack_path = fake_pack.path
classifier_converter = ClassifierSixConverter(Pack(fake_pack_path))
assert classifier_converter.convert_dir() == 0
expected_new_classifier_path = f'{tmpdir}/Packs/FakeTestPack/Classifiers/classifier-Cymulate.json'
expected_new_mapper_path = f'{tmpdir}/Packs/FakeTestPack/Classifiers/classifier-mapper-incoming-Cymulate.json'
self.assert_expected_file_output(expected_new_classifier_path,
'classifier-Cymulate')
self.assert_expected_file_output(
expected_new_mapper_path, 'classifier-mapper-incoming-Cymulate')
def test_calculate_new_path(self, tmpdir, old_classifier_brand: str,
is_mapper: bool, expected_suffix: str):
"""
Given:
- 'old_classifier_brand': Old classifier brand name.
- 'is_mapper': Whether file created is mapper or classifier of 6_0_0 convention.
When:
- Creating the path to the newly classifier/mapper of 6_0_0 convention.
Then:
- Ensure the expected path is returned.
"""
classifier_six_converter = ClassifierSixConverter(Pack(tmpdir))
assert classifier_six_converter.calculate_new_path(
old_classifier_brand,
is_mapper) == f'{tmpdir}/Classifiers/{expected_suffix}'
def test_calculate_new_layout_group(self, tmpdir, old_layout_path: str, expected: str):
"""
Given:
- 'old_layouts': List of old layout objects.
When:
- Calculating the group field value for the layout above 6.0.0 version to be created.
Then:
- Ensure the expected group value is returned.
"""
fake_pack_name = 'FakeTestPack'
repo = Repo(tmpdir)
repo_path = Path(repo.path)
fake_pack = MockPack(repo_path / 'Packs', fake_pack_name, repo)
fake_pack.create_layout('test', util_load_json(old_layout_path))
fake_pack_path = fake_pack.path
layout_converter = LayoutSixConverter(Pack(fake_pack_path))
assert layout_converter.calculate_new_layout_group(
[layout for layout in layout_converter.pack.layouts]) == expected
def test_calculate_from_version(self, tmpdir, layout_id: str,
layout_kind: str, expected: str):
"""
Given:
- 'layout_id' Layout ID.
- 'layout_kind' Layout kind.
- 'current_old_layouts': Old layouts existing in the Layouts directory before the conversion.
When:
- Calculating the from version for the new layout to be created.
Then:
- Ensure that from version from existing layout is returned if corresponding layout exists with from version,
else 'MINIMAL_FROM_VERSION' is returned.
"""
full_layout_path = os.path.join(
__file__,
f'{git_path()}/demisto_sdk/commands/convert/converters/layout/tests/'
'test_data/layout-close-ExtraHop_Detection.json')
assert LayoutBelowSixConverter(Pack(tmpdir)).calculate_from_version(
layout_id, layout_kind, [Layout(full_layout_path)]) == expected
def test_calculate_new_layout_relative_path(self, tmpdir,
dynamic_field_key: str,
type_id: str,
expected_suffix: str):
"""
Given:
- 'layout_id': Layout ID of the new layout created.
- 'dynamic_key_field': The dynamic key field related to the new layout created until version 5.9.9.
- 'type_id': Type ID of the corresponding incident/indicator
When:
- Calculating the path to the newly created layout.
Then:
- Ensure the expected path is returned.
"""
layout_converter = LayoutBelowSixConverter(Pack(tmpdir))
expected = f'{layout_converter.pack.path}/Layouts/{expected_suffix}'
assert layout_converter.calculate_new_layout_relative_path(
type_id, dynamic_field_key) == expected
def init_packs(self, pack_paths):
"""Init dict that map pack name to Pack object and init also the pack_names property.
Args:
pack_paths (str): CSV str with the pack paths.
"""
self.packs = {}
for path_str in arg_to_list(pack_paths):
path = Path(path_str)
if len(path.parts) == 2 and path.parts[
0] == PACKS_DIR: # relative path from Packs/...
path = self.content.path / path
if not os.path.exists(path):
click.secho(
f'Error: Given input path: {path} does not exist, ignored',
fg='bright_red')
continue
self.packs.update({path.name: Pack(path)})
self.pack_names = [*self.packs]
def test_build_old_layout(self, tmpdir, layout_id: str,
incident_type_id: str, dynamic_field_key: str,
dynamic_field_value: str, expected: Dict):
"""
Given:
- 'layout_id': Old layout ID to be created.
- 'incident_type_id': Incident type ID corresponding to the newly created old layout.
- 'dynamic_field_key': The dynamic field key from whom the layout will be created.
- 'dynamic_field_value': The dynamic field value from whom the layout will be created.
When:
- Creating layout of the format below 6.0.0 version.
Then:
- Ensure expected dict is returned.
"""
assert LayoutBelowSixConverter(Pack(tmpdir)).build_old_layout(
layout_id,
incident_type_id,
dynamic_field_key,
dynamic_field_value,
from_version='4.1.0') == expected
def test_get_layouts_by_layout_container_type(self, tmpdir):
"""
Given:
- Layout container FileType.
When:
- Wanting to retrieve all layout-containers from the current pack.
Then:
- Ensure only layout-containers in the pack are returned.
"""
fake_pack_name = 'FakeTestPack'
repo = Repo(tmpdir)
repo_path = Path(repo.path)
fake_pack = MockPack(repo_path / 'Packs', fake_pack_name, repo)
fake_pack.create_layoutcontainer('ExtraHop Detection',
util_load_json(self.LAYOUT_CONTAINER))
fake_pack_path = fake_pack.path
layouts = BaseConverter.get_entities_by_entity_type(
Pack(fake_pack_path).layouts, FileType.LAYOUTS_CONTAINER)
assert all(layout.type() == FileType.LAYOUTS_CONTAINER
for layout in layouts)
assert [layout.layout_id()
for layout in layouts] == ['ExtraHop Detection']