def upload(self): """Upload the integration specified in self.infile to the remote Demisto instance. """ try: if self.unify: # Create a temporary unified yml file try: unifier = Unifier(self.path, outdir=self.path) self.path = unifier.merge_script_package_to_yml()[0][0] except IndexError: print_color( 'Error: Path input is not a valid package directory.', LOG_COLORS.RED) return 1 # Upload the file to Demisto result = self.client.integration_upload(file=self.path) # Print results print_v(f'Result:\n{result.to_str()}', self.log_verbose) print_color(f'Uploaded \'{result.name}\' successfully', LOG_COLORS.GREEN) except Exception as ex: raise ex finally: if self.unify and os.path.exists( self.path): # Remove the temporary file os.remove(self.path) return 0
def check_api_module_imports(self, py_num): """ Checks if the integration imports an API module and if so pastes the module in the package. :param py_num: The python version - api modules are in python 3 """ if py_num > 3: unifier = Unifier(self.project_dir) code_file_path = unifier.get_code_file('.py') try: # Look for an import to an API module in the code. If there is such import, we need to copy the correct # module file to the package directory. with io.open(code_file_path, mode='r', encoding='utf-8') as script_file: _, module_name = unifier.check_api_module_imports( script_file.read()) if module_name: module_path = os.path.join(self.configuration.env_dir, 'Packs', 'ApiModules', 'Scripts', module_name, module_name + '.py') print_v('Copying ' + os.path.join( self.configuration.env_dir, 'Scripts', module_path)) if not os.path.exists(module_path): raise ValueError( 'API Module {} not found, you might be outside of the content repository' ' or this API module does not exist'.format( module_name)) shutil.copy(os.path.join(module_path), self.project_dir) except Exception as e: print_v('Unable to retrieve the module file {}: {}'.format( module_name, str(e)))
def copy_packs_content_to_packs_bundle(self, packs): ''' Copy content in packs to the bundle that gets zipped to 'content_packs.zip'. Preserves directory structure except that packages inside the "Integrations" or "Scripts" directory inside a pack are flattened. Adds file prefixes according to how server expects to ingest the files, e.g. 'integration-' is prepended to integration yml filenames and 'script-' is prepended to script yml filenames and so on and so forth. ''' for pack in packs: pack_name = os.path.basename(pack) if pack_name in self.packs_to_skip: continue pack_dst = os.path.join(self.packs_bundle, pack_name) os.mkdir(pack_dst) pack_dirs = get_child_directories(pack) pack_files = get_child_files(pack) # copy first level pack files over for file_path in pack_files: shutil.copy( file_path, os.path.join(pack_dst, os.path.basename(file_path))) # handle content directories in the pack for content_dir in pack_dirs: dir_name = os.path.basename(content_dir) dest_dir = os.path.join(pack_dst, dir_name) os.mkdir(dest_dir) if dir_name in DIR_TO_PREFIX: packages_dirs = get_child_directories(content_dir) for package_dir in packages_dirs: ymls, _ = get_yml_paths_in_dir(package_dir, error_msg='') if not ymls or (len(ymls) == 1 and ymls[0].endswith('_unified.yml')): msg = 'Skipping package: {} -'.format(package_dir) if not ymls: print_warning( '{} No yml files found in the package directory' .format(msg)) else: print_warning( '{} Only unified yml found in the package directory' .format(msg)) continue package_dir_name = os.path.basename(package_dir) unifier = Unifier(package_dir, dir_name, dest_dir) unifier.merge_script_package_to_yml() # also copy CHANGELOG markdown files over (should only be one per package) package_files = get_child_files(package_dir) changelog_files = [ file_path for file_path in package_files if 'CHANGELOG.md' in file_path ] for md_file_path in changelog_files: md_out_name = '{}-{}_CHANGELOG.md'.format( DIR_TO_PREFIX.get(dir_name), package_dir_name) shutil.copyfile( md_file_path, os.path.join(dest_dir, md_out_name)) else: self.copy_dir_files(content_dir, dest_dir)
def is_file_has_used_id(self, file_path): """Check if the ID of the given file already exist in the system. Args: file_path (string): Path to the file. Returns: bool. Whether the ID of the given file already exist in the system or not. """ is_used = False is_json_file = False if self.is_circle: if re.match(TEST_PLAYBOOK_REGEX, file_path, re.IGNORECASE): obj_type = self.TEST_PLAYBOOK_SECTION obj_id = collect_ids(file_path) obj_data = get_playbook_data(file_path) elif re.match(SCRIPT_REGEX, file_path, re.IGNORECASE) or \ re.match(TEST_SCRIPT_REGEX, file_path, re.IGNORECASE): obj_type = self.SCRIPTS_SECTION obj_id = get_script_or_integration_id(file_path) obj_data = get_script_data(file_path) elif re.match(INTEGRATION_REGEX, file_path, re.IGNORECASE) or \ re.match(INTEGRATION_YML_REGEX, file_path, re.IGNORECASE): obj_type = self.INTEGRATION_SECTION obj_id = get_script_or_integration_id(file_path) obj_data = get_integration_data(file_path) elif re.match(PLAYBOOK_REGEX, file_path, re.IGNORECASE): obj_type = self.PLAYBOOK_SECTION obj_id = collect_ids(file_path) obj_data = get_playbook_data(file_path) elif re.match(SCRIPT_YML_REGEX, file_path, re.IGNORECASE) or \ re.match(SCRIPT_PY_REGEX, file_path, re.IGNORECASE) or \ re.match(SCRIPT_JS_REGEX, file_path, re.IGNORECASE): unifier = Unifier(os.path.dirname(os.path.dirname(file_path))) yml_path, code = unifier.get_script_package_data() obj_data = get_script_data(yml_path, script_code=code) obj_type = self.SCRIPTS_SECTION obj_id = get_script_or_integration_id(yml_path) else: # In case of a json file is_json_file = True if not is_json_file: is_used = self.is_id_duplicated(obj_id, obj_data, obj_type) return is_used
def test_clean_python_code(): from demisto_sdk.yaml_tools.unifier import Unifier unifier = Unifier("path") script_code = "import demistomock as demistofrom CommonServerPython import *" \ "from CommonServerUserPython import *from __future__ import print_function" # Test remove_print_future is False script_code = unifier.clean_python_code(script_code, remove_print_future=False) assert script_code == "from __future__ import print_function" # Test remove_print_future is True script_code = unifier.clean_python_code(script_code) assert script_code == ""
def test_get_code_file(): from demisto_sdk.yaml_tools.unifier import Unifier # Test integration case unifier = Unifier("tests/test_files/VulnDB/") assert unifier.get_code_file(".py") == "tests/test_files/VulnDB/VulnDB.py" unifier = Unifier("tests/test_files") with pytest.raises(Exception): unifier.get_code_file(".py") # Test script case unifier = Unifier("tests/test_files/CalculateGeoDistance/") assert unifier.get_code_file( ".py" ) == "tests/test_files/CalculateGeoDistance/CalculateGeoDistance.py"
def test_insert_description_to_yml(): from demisto_sdk.yaml_tools.unifier import Unifier with patch.object(Unifier, "__init__", lambda a, b, c, d, e: None): unifier = Unifier('', None, None, None) unifier.package_path = "tests/test_files/VulnDB/" unifier.dir_name = "Integrations" with open("tests/test_files/VulnDB/VulnDB_description.md", "rb") as desc_file: desc_data = desc_file.read().decode("utf-8") desc_data = '|\n ' + desc_data.replace('\n', '\n ') yml_text, found_data_path = unifier.insert_description_to_yml({}, "") assert found_data_path == "tests/test_files/VulnDB/VulnDB_description.md" assert desc_data in yml_text
def process_script(file_path): res = [] if os.path.isfile(file_path): if checked_type(file_path, (SCRIPT_REGEX, PACKS_SCRIPT_YML_REGEX)): print("adding {0} to id_set".format(file_path)) res.append(get_script_data(file_path)) else: # package script unifier = Unifier(file_path) yml_path, code = unifier.get_script_package_data() print("adding {0} to id_set".format(file_path)) res.append(get_script_data(yml_path, script_code=code)) return res
def copy_content_yml(self, path, out_path, yml_info): '''Copy content ymls (except for playbooks) to the out_path (presumably a bundle)''' parent_dir_name = os.path.basename(os.path.dirname(path)) if parent_dir_name in DIR_TO_PREFIX and not os.path.basename( path).startswith('playbook-'): script_obj = yml_info if parent_dir_name != SCRIPTS_DIR: script_obj = yml_info['script'] with io.open(path, mode='r', encoding='utf-8') as file_: yml_text = file_.read() unifier = Unifier(os.path.dirname(path), parent_dir_name, out_path) out_map = unifier.write_yaml_with_docker(yml_text, yml_info, script_obj) if len(out_map.keys()) > 1: print(" - yaml generated multiple files: {}".format( out_map.keys())) return # not a script or integration file. Simply copy shutil.copyfile(path, out_path)
def is_file_valid_in_set(self, file_path): """Check if the file is represented correctly in the id_set Args: file_path (string): Path to the file. Returns: bool. Whether the file is represented correctly in the id_set or not. """ is_valid = True if self.is_circle: # No need to check on local env because the id_set will contain this info after the commit if re.match(PLAYBOOK_REGEX, file_path, re.IGNORECASE): playbook_data = get_playbook_data(file_path) is_valid = self.is_valid_in_id_set(file_path, playbook_data, self.playbook_set) elif re.match(TEST_PLAYBOOK_REGEX, file_path, re.IGNORECASE): playbook_data = get_playbook_data(file_path) is_valid = self.is_valid_in_id_set(file_path, playbook_data, self.test_playbook_set) elif re.match(TEST_SCRIPT_REGEX, file_path, re.IGNORECASE) or \ re.match(SCRIPT_REGEX, file_path, re.IGNORECASE): script_data = get_script_data(file_path) is_valid = self.is_valid_in_id_set(file_path, script_data, self.script_set) elif re.match(INTEGRATION_REGEX, file_path, re.IGNORECASE) or \ re.match(INTEGRATION_YML_REGEX, file_path, re.IGNORECASE): integration_data = get_integration_data(file_path) is_valid = self.is_valid_in_id_set(file_path, integration_data, self.integration_set) elif re.match(SCRIPT_YML_REGEX, file_path, re.IGNORECASE) or \ re.match(SCRIPT_PY_REGEX, file_path, re.IGNORECASE) or \ re.match(SCRIPT_JS_REGEX, file_path, re.IGNORECASE): unifier = Unifier(os.path.dirname(file_path)) yml_path, code = unifier.get_script_package_data() script_data = get_script_data(yml_path, script_code=code) is_valid = self.is_valid_in_id_set(yml_path, script_data, self.script_set) return is_valid
def test_insert_script_to_yml(package_path, dir_name, file_path): from demisto_sdk.yaml_tools.unifier import Unifier with patch.object(Unifier, "__init__", lambda a, b, c, d, e: None): unifier = Unifier("", None, None, None) unifier.package_path = package_path unifier.dir_name = dir_name with open(file_path + ".yml", mode="r", encoding="utf-8") as yml_file: test_yml_text = yml_file.read() with open(file_path + ".yml", "r") as yml: test_yml_data = yaml.safe_load(yml) yml_text, script_path = unifier.insert_script_to_yml( ".py", test_yml_text, test_yml_data) with open(file_path + ".py", mode="r", encoding="utf-8") as script_file: script_code = script_file.read() clean_code = unifier.clean_python_code(script_code) lines = ['|-'] lines.extend(u' {}'.format(line) for line in clean_code.split('\n')) script_code = u'\n'.join(lines) test_yml_text = test_yml_text.replace("script: ''", "script: " + script_code) test_yml_text = test_yml_text.replace("script: '-'", "script: " + script_code) assert yml_text == test_yml_text assert script_path == file_path + ".py"
def test_get_script_package_data(): from demisto_sdk.yaml_tools.unifier import Unifier unifier = Unifier("tests/") with pytest.raises(Exception): unifier.get_script_package_data() unifier = Unifier("tests/test_files/CalculateGeoDistance") with open("tests/test_files/CalculateGeoDistance/CalculateGeoDistance.py", "r") as code_file: code = code_file.read() yml_path, code_data = unifier.get_script_package_data() assert yml_path == "tests/test_files/CalculateGeoDistance/CalculateGeoDistance.yml" assert code_data == code
def test_insert_module_code(mocker, import_name): from demisto_sdk.yaml_tools.unifier import Unifier mocker.patch.object(Unifier, '_get_api_module_code', return_value=DUMMY_MODULE) module_name = 'MicrosoftApiModule' new_code = DUMMY_SCRIPT.replace( import_name, '\n### GENERATED CODE ###\n# This code was inserted in place of an API' ' module.{}\n'.format(DUMMY_MODULE)) code = Unifier.insert_module_code(DUMMY_SCRIPT, import_name, module_name) assert code == new_code
def test_get_data(): from demisto_sdk.yaml_tools.unifier import Unifier with patch.object(Unifier, "__init__", lambda a, b, c, d, e: None): unifier = Unifier('', None, None, None) unifier.package_path = "tests/test_files/VulnDB/" unifier.dir_name = "Integrations" with open("tests/test_files/VulnDB/VulnDB_image.png", "rb") as image_file: image = image_file.read() data, found_data_path = unifier.get_data("*png") assert data == image assert found_data_path == "tests/test_files/VulnDB/VulnDB_image.png" unifier.dir_name = "Scripts" data, found_data_path = unifier.get_data("*png") assert data is None assert found_data_path is None
def test_insert_script_to_yml_exceptions(package_path, dir_name, file_path): from demisto_sdk.yaml_tools.unifier import Unifier with patch.object(Unifier, "__init__", lambda a, b, c, d, e: None): unifier = Unifier("", None, None, None) unifier.package_path = package_path unifier.dir_name = dir_name with open(file_path + ".yml", "r") as yml: test_yml_data = yaml.safe_load(yml) if dir_name == "Scripts": test_yml_data['script'] = 'blah' elif dir_name == "Integrations": test_yml_data['script']['script'] = 'blah' with pytest.raises(ValueError): unifier.insert_script_to_yml(".py", "", test_yml_data)
def create_unifieds_and_copy(self, package_dir, dest_dir='', skip_dest_dir=''): ''' For directories that have packages, aka subdirectories for each integration/script e.g. "Integrations", "Beta_Integrations", "Scripts". Creates a unified yml and writes it to the dest_dir Arguments: package_dir: (str) Path to directory in which there are package subdirectories. e.g. "Integrations", "Beta_Integrations", "Scripts" dest_dir: (str) Path to destination directory to which the unified yml for a package should be written skip_dest_dir: (str) Path to the directory to which the unified yml for a package should be written in the case the package is part of the skipped list ''' dest_dir = dest_dir if dest_dir else self.content_bundle skip_dest_dir = skip_dest_dir if skip_dest_dir else self.test_bundle scanned_packages = glob.glob(os.path.join(package_dir, '*/')) package_dir_name = os.path.basename(package_dir) for package in scanned_packages: ymls, _ = get_yml_paths_in_dir(package, error_msg='') if not ymls or (len(ymls) == 1 and ymls[0].endswith('_unified.yml')): msg = 'Skipping package: {} -'.format(package) if not ymls: print_warning( '{} No yml files found in the package directory'. format(msg)) else: print_warning( '{} Only unified yml found in the package directory'. format(msg)) continue unification_tool = Unifier(package, package_dir_name, dest_dir) if any(package_to_skip in package for package_to_skip in self.packages_to_skip): # there are some packages that we don't want to include in the content zip # for example HelloWorld integration unification_tool = Unifier(package, package_dir_name, skip_dest_dir) print('skipping {}'.format(package)) unification_tool.merge_script_package_to_yml()
def test_insert_image_to_yml(): from demisto_sdk.yaml_tools.unifier import Unifier with patch.object(Unifier, "__init__", lambda a, b, c, d, e: None): unifier = Unifier('', None, None, None) unifier.package_path = "tests/test_files/VulnDB/" unifier.dir_name = "Integrations" unifier.image_prefix = "data:image/png;base64," with open("tests/test_files/VulnDB/VulnDB_image.png", "rb") as image_file: image_data = image_file.read() image_data = unifier.image_prefix + base64.b64encode( image_data).decode('utf-8') with open("tests/test_files/VulnDB/VulnDB.yml", mode="r", encoding="utf-8") as yml_file: yml_text_test = yml_file.read() with open("tests/test_files/VulnDB/VulnDB.yml", "r") as yml: yml_data = yaml.safe_load(yml) yml_text, found_img_path = unifier.insert_image_to_yml( yml_data, yml_text_test) yml_text_test = 'image: ' + image_data + '\n' + yml_text_test assert found_img_path == "tests/test_files/VulnDB/VulnDB_image.png" assert yml_text == yml_text_test
def unify(**kwargs): unifier = Unifier(**kwargs) return unifier.merge_script_package_to_yml()
def test_check_api_module_imports(): from demisto_sdk.yaml_tools.unifier import Unifier module_import, module_name = Unifier.check_api_module_imports(DUMMY_SCRIPT) assert module_import == 'from MicrosoftApiModule import * # noqa: E402' assert module_name == 'MicrosoftApiModule'
def validate_added_files(self, added_files): # noqa: C901 """Validate the added files from your branch. In case we encounter an invalid file we set the self._is_valid param to False. Args: added_files (set): A set of the modified files in the current branch. """ for file_path in added_files: print('Validating {}'.format(file_path)) if re.match(TEST_PLAYBOOK_REGEX, file_path, re.IGNORECASE): continue structure_validator = StructureValidator(file_path) if not structure_validator.is_valid_file(): self._is_valid = False if self.validate_id_set: if not self.id_set_validator.is_file_valid_in_set(file_path): self._is_valid = False if self.id_set_validator.is_file_has_used_id(file_path): self._is_valid = False elif re.match(PLAYBOOK_REGEX, file_path, re.IGNORECASE): playbook_validator = PlaybookValidator(structure_validator) if not playbook_validator.is_valid_playbook(): self._is_valid = False elif checked_type(file_path, YML_INTEGRATION_REGEXES): image_validator = ImageValidator(file_path) if not image_validator.is_valid(): self._is_valid = False description_validator = DescriptionValidator(file_path) if not description_validator.is_valid(): self._is_valid = False integration_validator = IntegrationValidator( structure_validator) if not integration_validator.is_valid_file(validate_rn=False): self._is_valid = False elif checked_type(file_path, PACKAGE_SCRIPTS_REGEXES): unifier = Unifier(os.path.dirname(file_path)) yml_path, _ = unifier.get_script_package_data() # Set file path to the yml file structure_validator.file_path = yml_path script_validator = ScriptValidator(structure_validator) if not script_validator.is_valid_file(validate_rn=False): self._is_valid = False elif re.match(BETA_INTEGRATION_REGEX, file_path, re.IGNORECASE) or \ re.match(BETA_INTEGRATION_YML_REGEX, file_path, re.IGNORECASE): description_validator = DescriptionValidator(file_path) if not description_validator.is_valid_beta_description(): self._is_valid = False integration_validator = IntegrationValidator( structure_validator) if not integration_validator.is_valid_beta_integration(): self._is_valid = False elif re.match(IMAGE_REGEX, file_path, re.IGNORECASE): image_validator = ImageValidator(file_path) if not image_validator.is_valid(): self._is_valid = False # incident fields and indicator fields are using the same scheme. elif checked_type(file_path, JSON_INDICATOR_AND_INCIDENT_FIELDS): incident_field_validator = IncidentFieldValidator( structure_validator) if not incident_field_validator.is_valid_file(): self._is_valid = False elif checked_type(file_path, JSON_ALL_LAYOUT_REGEXES): layout_validator = LayoutValidator(structure_validator) if not layout_validator.is_valid_layout(): self._is_valid = False elif 'CHANGELOG' in file_path: self.is_valid_release_notes(file_path) elif checked_type(file_path, [REPUTATION_REGEX]): print_color( F'Skipping validation for file {file_path} since no validation is currently defined.', LOG_COLORS.YELLOW) elif checked_type(file_path, CHECKED_TYPES_REGEXES): pass else: print_error( "The file type of {} is not supported in validate command". format(file_path)) print_error( "validate command supports: Integrations, Scripts, Playbooks, " "Incident fields, Indicator fields, Images, Release notes, Layouts and Descriptions" ) self._is_valid = False
def validate_modified_files(self, modified_files): # noqa: C901 """Validate the modified files from your branch. In case we encounter an invalid file we set the self._is_valid param to False. Args: modified_files (set): A set of the modified files in the current branch. """ for file_path in modified_files: old_file_path = None if isinstance(file_path, tuple): old_file_path, file_path = file_path print('Validating {}'.format(file_path)) if not checked_type(file_path): print_warning( '- Skipping validation of non-content entity file.') continue if re.match(TEST_PLAYBOOK_REGEX, file_path, re.IGNORECASE): continue structure_validator = StructureValidator(file_path, old_file_path) if not structure_validator.is_valid_file(): self._is_valid = False if self.validate_id_set: if not self.id_set_validator.is_file_valid_in_set(file_path): self._is_valid = False elif checked_type(file_path, YML_INTEGRATION_REGEXES): image_validator = ImageValidator(file_path) if not image_validator.is_valid(): self._is_valid = False description_validator = DescriptionValidator(file_path) if not description_validator.is_valid(): self._is_valid = False integration_validator = IntegrationValidator( structure_validator) if self.is_backward_check and not integration_validator.is_backward_compatible( ): self._is_valid = False if not integration_validator.is_valid_file(): self._is_valid = False elif checked_type(file_path, YML_BETA_INTEGRATIONS_REGEXES): image_validator = ImageValidator(file_path) if not image_validator.is_valid(): self._is_valid = False description_validator = DescriptionValidator(file_path) if not description_validator.is_valid_beta_description(): self._is_valid = False integration_validator = IntegrationValidator( structure_validator) if not integration_validator.is_valid_beta_integration(): self._is_valid = False elif checked_type(file_path, [SCRIPT_REGEX]): script_validator = ScriptValidator(structure_validator) if self.is_backward_check and not script_validator.is_backward_compatible( ): self._is_valid = False if not script_validator.is_valid_file(): self._is_valid = False elif checked_type(file_path, PLAYBOOKS_REGEXES_LIST): playbook_validator = PlaybookValidator(structure_validator) if not playbook_validator.is_valid_playbook( is_new_playbook=False): self._is_valid = False elif checked_type(file_path, PACKAGE_SCRIPTS_REGEXES): unifier = Unifier(os.path.dirname(file_path)) yml_path, _ = unifier.get_script_package_data() # Set file path to the yml file structure_validator.file_path = yml_path script_validator = ScriptValidator(structure_validator) if self.is_backward_check and not script_validator.is_backward_compatible( ): self._is_valid = False if not script_validator.is_valid_file(): self._is_valid = False elif re.match(IMAGE_REGEX, file_path, re.IGNORECASE): image_validator = ImageValidator(file_path) if not image_validator.is_valid(): self._is_valid = False # incident fields and indicator fields are using the same scheme. elif checked_type(file_path, JSON_INDICATOR_AND_INCIDENT_FIELDS): incident_field_validator = IncidentFieldValidator( structure_validator) if not incident_field_validator.is_valid_file(): self._is_valid = False if self.is_backward_check and not incident_field_validator.is_backward_compatible( ): self._is_valid = False elif checked_type(file_path, JSON_ALL_LAYOUT_REGEXES): layout_validator = LayoutValidator(structure_validator) if not layout_validator.is_valid_layout(): self._is_valid = False elif 'CHANGELOG' in file_path: self.is_valid_release_notes(file_path) else: print_error( "The file type of {} is not supported in validate command". format(file_path)) print_error( "'validate' command supports: Integrations, Scripts, Playbooks, " "Incident fields, Indicator fields, Images, Release notes, Layouts and Descriptions" ) self._is_valid = False
def _get_lint_files(self): unifier = Unifier(self.project_dir) code_file = unifier.get_code_file('.py') return os.path.abspath(code_file)
def update_id_set(): branches = run_command("git branch") branch_name_reg = re.search(r"\* (.*)", branches) branch_name = branch_name_reg.group(1) print("Getting added files") files_string = run_command("git diff --name-status HEAD") second_files_string = run_command( "git diff --name-status origin/master...{}".format(branch_name)) added_files, modified_files, added_scripts, modified_scripts = \ get_changed_files(files_string + '\n' + second_files_string) if added_files or modified_files or added_scripts or modified_scripts: print("Updating id_set.json") with open('./Tests/id_set.json', 'r') as id_set_file: try: ids_dict = json.load(id_set_file, object_pairs_hook=OrderedDict) except ValueError as ex: if "Expecting property name" in str(ex): # if we got this error it means we have corrupted id_set.json # usually it will happen if we merged from master and we had a conflict in id_set.json # so we checkout the id_set.json to be exact as in master and then run update_id_set run_command("git checkout origin/master Tests/id_set.json") with open('./Tests/id_set.json', 'r') as id_set_file_from_master: ids_dict = json.load(id_set_file_from_master, object_pairs_hook=OrderedDict) else: raise test_playbook_set = ids_dict['TestPlaybooks'] integration_set = ids_dict['integrations'] playbook_set = ids_dict['playbooks'] script_set = ids_dict['scripts'] if added_files: for file_path in added_files: if re.match(INTEGRATION_REGEX, file_path, re.IGNORECASE) or \ re.match(INTEGRATION_YML_REGEX, file_path, re.IGNORECASE): add_new_object_to_id_set( get_script_or_integration_id(file_path), get_integration_data(file_path), integration_set) print("Adding {0} to id_set".format( get_script_or_integration_id(file_path))) if re.match(SCRIPT_REGEX, file_path, re.IGNORECASE): add_new_object_to_id_set( get_script_or_integration_id(file_path), get_script_data(file_path), script_set) print("Adding {0} to id_set".format( get_script_or_integration_id(file_path))) if re.match(PLAYBOOK_REGEX, file_path, re.IGNORECASE): add_new_object_to_id_set(collect_ids(file_path), get_playbook_data(file_path), playbook_set) print("Adding {0} to id_set".format(collect_ids(file_path))) if re.match(TEST_PLAYBOOK_REGEX, file_path, re.IGNORECASE): add_new_object_to_id_set(collect_ids(file_path), get_playbook_data(file_path), test_playbook_set) print("Adding {0} to id_set".format(collect_ids(file_path))) if re.match(TEST_SCRIPT_REGEX, file_path, re.IGNORECASE): add_new_object_to_id_set( get_script_or_integration_id(file_path), get_script_data(file_path), script_set) print("Adding {0} to id_set".format(collect_ids(file_path))) if modified_files: for file_path in modified_files: if re.match(INTEGRATION_REGEX, file_path, re.IGNORECASE) or \ re.match(INTEGRATION_YML_REGEX, file_path, re.IGNORECASE): id = get_script_or_integration_id(file_path) integration_data = get_integration_data(file_path) update_object_in_id_set(id, integration_data, file_path, integration_set) print("updated {0} in id_set".format(id)) if re.match(SCRIPT_REGEX, file_path, re.IGNORECASE) or re.match( TEST_SCRIPT_REGEX, file_path, re.IGNORECASE): id = get_script_or_integration_id(file_path) script_data = get_script_data(file_path) update_object_in_id_set(id, script_data, file_path, script_set) print("updated {0} in id_set".format(id)) if re.match(PLAYBOOK_REGEX, file_path, re.IGNORECASE): id = collect_ids(file_path) playbook_data = get_playbook_data(file_path) update_object_in_id_set(id, playbook_data, file_path, playbook_set) print("updated {0} in id_set".format(id)) if re.match(TEST_PLAYBOOK_REGEX, file_path, re.IGNORECASE): id = collect_ids(file_path) playbook_data = get_playbook_data(file_path) update_object_in_id_set(id, playbook_data, file_path, test_playbook_set) print("updated {0} in id_set".format(id)) if added_scripts: for added_script_package in added_scripts: unifier = Unifier(added_script_package) yml_path, code = unifier.get_script_package_data() add_new_object_to_id_set( get_script_or_integration_id(yml_path), get_script_data(yml_path, script_code=code), script_set) print("Adding {0} to id_set".format( get_script_or_integration_id(yml_path))) if modified_scripts: for modified_script_package in added_scripts: unifier = Unifier(modified_script_package) yml_path, code = unifier.get_script_package_data() update_object_in_id_set( get_script_or_integration_id(yml_path), get_script_data(yml_path, script_code=code), yml_path, script_set) print("Adding {0} to id_set".format( get_script_or_integration_id(yml_path))) if added_files or modified_files: new_ids_dict = OrderedDict() # we sort each time the whole set in case someone manually changed something # it shouldn't take too much time new_ids_dict['scripts'] = sort(script_set) new_ids_dict['playbooks'] = sort(playbook_set) new_ids_dict['integrations'] = sort(integration_set) new_ids_dict['TestPlaybooks'] = sort(test_playbook_set) with open('./Tests/id_set.json', 'w') as id_set_file: json.dump(new_ids_dict, id_set_file, indent=4) print("Finished updating id_set.json")