def test_get_docker_image_from_yml(): with patch.object(DockerImageValidator, '__init__', lambda x, y, z, w: None): # Test integration case docker_validator = DockerImageValidator(None, None, None) docker_validator.yml_file = get_yaml(TEST_INTEGRATION_FILE) docker_validator.is_integration = True docker_image = docker_validator.get_docker_image_from_yml() assert docker_image == "demisto/pyjwt:1.0" # Test script case docker_validator.yml_file = get_yaml(TEST_SCRIPT_FILE) docker_validator.is_integration = False docker_image = docker_validator.get_docker_image_from_yml() assert docker_image == "demisto/stix2:1.0.0.204"
def is_existing_image(self): """Check if the integration has an image.""" is_image_in_yml = False is_image_in_package = False data_dictionary = get_yaml(self.file_path) if not data_dictionary: return False if data_dictionary.get('image'): is_image_in_yml = True if not re.match(INTEGRATION_REGEX, self.file_path, re.IGNORECASE): package_path = os.path.dirname(self.file_path) if is_image_in_yml: print_error( "You have added an image in the yml " "file, please update the package {}".format(package_path)) return False image_path = glob.glob(package_path + '/*.png') if image_path: is_image_in_package = True if not (is_image_in_package or is_image_in_yml): print_error( "You have failed to add an image in the yml/package for {}". format(self.file_path)) self._is_valid = False return False return True
def get_playbook_data(file_path): playbook_data = OrderedDict() data_dictionary = get_yaml(file_path) id = data_dictionary.get('id', '-') name = data_dictionary.get('name', '-') tests = data_dictionary.get('tests') toversion = data_dictionary.get('toversion') fromversion = data_dictionary.get('fromversion') implementing_scripts = get_task_ids_from_playbook('scriptName', data_dictionary) implementing_playbooks = get_task_ids_from_playbook( 'playbookName', data_dictionary) command_to_integration = get_commmands_from_playbook(data_dictionary) playbook_data['name'] = name playbook_data['file_path'] = file_path if toversion: playbook_data['toversion'] = toversion if fromversion: playbook_data['fromversion'] = fromversion if implementing_scripts: playbook_data['implementing_scripts'] = implementing_scripts if implementing_playbooks: playbook_data['implementing_playbooks'] = implementing_playbooks if command_to_integration: playbook_data['command_to_integration'] = command_to_integration if tests: playbook_data['tests'] = tests return {id: playbook_data}
def oversize_image(self): """Check if the image if over sized, bigger than IMAGE_MAX_SIZE""" if re.match(IMAGE_REGEX, self.file_path, re.IGNORECASE): if os.path.getsize( self.file_path ) > self.IMAGE_MAX_SIZE: # disable-secrets-detection print_error( "{} has too large logo, please update the logo to be under 10kB" .format(self.file_path)) self._is_valid = False else: data_dictionary = get_yaml(self.file_path) if not data_dictionary: return image = data_dictionary.get('image', '') if ((len(image) - 22) / 4.0 ) * 3 > self.IMAGE_MAX_SIZE: # disable-secrets-detection print_error( "{} has too large logo, please update the logo to be under 10kB" .format(self.file_path)) self._is_valid = False
def get_integration_commands(file_path): cmd_list = [] data_dictionary = get_yaml(file_path) commands = data_dictionary.get('script', {}).get('commands', []) for command in commands: cmd_list.append(command.get('name')) return cmd_list
def __init__(self, yml_file_path, is_modified_file, is_integration): self.is_modified_file = is_modified_file self.is_integration = is_integration self.yml_file = get_yaml(yml_file_path) self.yml_docker_image = self.get_docker_image_from_yml() self.from_version = self.yml_file.get('fromversion', '0') self.docker_image_name, self.docker_image_tag = DockerImageValidator.parse_docker_image( self.yml_docker_image) self.is_latest_tag = True self.docker_image_latest_tag = DockerImageValidator.get_docker_image_latest_tag( self.docker_image_name, self.yml_docker_image) self.is_valid = True
def get_script_package_data(self): _, yml_path = get_yml_paths_in_dir(self.package_path, error_msg='') if not yml_path: raise Exception("No yml files found in package path: {}. " "Is this really a package dir? If not remove it.".format(self.package_path)) code_type = get_yaml(yml_path).get('type') unifier = Unifier(self.package_path) code_path = unifier.get_code_file(TYPE_TO_EXTENSION[code_type]) with open(code_path, 'r') as code_file: code = code_file.read() return yml_path, code
def _is_py_script_or_integration(file_path): file_yml = get_yaml(file_path) if re.match(INTEGRATION_REGEX, file_path, re.IGNORECASE): if file_yml.get('script', {}).get('type', 'javascript') != 'python': return False return True if re.match(SCRIPT_REGEX, file_path, re.IGNORECASE): if file_yml.get('type', 'javascript') != 'python': return False return True return False
def validate_no_old_format(self, old_format_files): """ Validate there are no files in the old format(unified yml file for the code and configuration). Args: old_format_files(set): file names which are in the old format. """ invalid_files = [] for f in old_format_files: yaml_data = get_yaml(f) if 'toversion' not in yaml_data: # we only fail on old format if no toversion (meaning it is latest) invalid_files.append(f) if invalid_files: print_error( 'You should update the following files to the package format, for further details please visit ' 'https://github.com/demisto/content/tree/master/docs/package_directory_structure. ' 'The files are:\n{}'.format('\n'.join(list(invalid_files)))) self._is_valid = False
def is_valid_beta_description(self): """Check if beta disclaimer exists in detailed description""" data_dictionary = get_yaml(self.file_path) description_in_yml = data_dictionary.get('detaileddescription', '') if data_dictionary else '' if not re.match(BETA_INTEGRATION_REGEX, self.file_path, re.IGNORECASE): package_path = os.path.dirname(self.file_path) try: md_file_path = glob.glob( os.path.join(os.path.dirname(self.file_path), '*_description.md'))[0] except IndexError: self._is_valid = False print_error( "No detailed description file was found in the package {}. Please add one," " and make sure it includes the beta disclaimer note." "It should contain the string in constant" "\"BETA_INTEGRATION_DISCLAIMER\"".format(package_path)) return False with open(md_file_path) as description_file: description = description_file.read() if BETA_INTEGRATION_DISCLAIMER not in description: self._is_valid = False print_error( "Detailed description in beta integration package {} " "dose not contain the beta disclaimer note. " "It should contain the string in constant" " \"BETA_INTEGRATION_DISCLAIMER\".".format(package_path)) return False else: return True elif BETA_INTEGRATION_DISCLAIMER not in description_in_yml: self._is_valid = False print_error("Detailed description field in beta integration {} " "dose not contain the beta disclaimer note." "It should contain the string in constant" " \"BETA_INTEGRATION_DISCLAIMER\".".format( self.file_path)) return False return True
def get_script_data(file_path, script_code=None): script_data = OrderedDict() data_dictionary = get_yaml(file_path) id = data_dictionary.get('commonfields', {}).get('id', '-') if script_code is None: script_code = data_dictionary.get('script', '') name = data_dictionary.get('name', '-') tests = data_dictionary.get('tests') toversion = data_dictionary.get('toversion') deprecated = data_dictionary.get('deprecated') fromversion = data_dictionary.get('fromversion') depends_on, command_to_integration = get_depends_on(data_dictionary) script_executions = sorted( list( set( re.findall(r"demisto.executeCommand\(['\"](\w+)['\"].*", script_code)))) script_data['name'] = name script_data['file_path'] = file_path if toversion: script_data['toversion'] = toversion if fromversion: script_data['fromversion'] = fromversion if deprecated: script_data['deprecated'] = deprecated if depends_on: script_data['depends_on'] = depends_on if script_executions: script_data['script_executions'] = script_executions if command_to_integration: script_data['command_to_integration'] = command_to_integration if tests: script_data['tests'] = tests return {id: script_data}
def load_image_from_yml(self): data_dictionary = get_yaml(self.file_path) if not data_dictionary: print_error( "{} isn't an image file or unified integration file.".format( self.file_path)) self._is_valid = False image = data_dictionary.get('image', '') if not image: print_error("{} is a yml file but has no image field.".format( self.file_path)) self._is_valid = False image_data = image.split('base64,') if image_data and len(image_data) == 2: return image_data[1] else: print_error("{}'s image field isn't in base64 encoding.".format( self.file_path)) self._is_valid = False
def process_test_playbook_path(file_path): """ Process a yml file in the testplyabook dir. Maybe either a script or playbook Arguments: file_path {string} -- path to yaml file Returns: pair -- first element is a playbook second is a script. each may be None """ print("adding {0} to id_set".format(file_path)) script = None playbook = None if checked_type( file_path, (TEST_SCRIPT_REGEX, PACKS_TEST_PLAYBOOKS_REGEX, TEST_PLAYBOOK_REGEX)): yml_data = get_yaml(file_path) if 'commonfields' in yml_data: # script files contain this key script = get_script_data(file_path) else: playbook = get_playbook_data(file_path) return playbook, script
def get_integration_data(file_path): integration_data = OrderedDict() data_dictionary = get_yaml(file_path) id = data_dictionary.get('commonfields', {}).get('id', '-') name = data_dictionary.get('name', '-') tests = data_dictionary.get('tests') toversion = data_dictionary.get('toversion') fromversion = data_dictionary.get('fromversion') commands = data_dictionary.get('script', {}).get('commands', []) cmd_list = [command.get('name') for command in commands] integration_data['name'] = name integration_data['file_path'] = file_path if toversion: integration_data['toversion'] = toversion if fromversion: integration_data['fromversion'] = fromversion if cmd_list: integration_data['commands'] = cmd_list if tests: integration_data['tests'] = tests return {id: integration_data}
def is_duplicate_description(self): """Check if the integration has a non-duplicate description .""" is_description_in_yml = False is_description_in_package = False package_path = None md_file_path = None if not re.match(INTEGRATION_REGEX, self.file_path, re.IGNORECASE) \ and not re.match(BETA_INTEGRATION_REGEX, self.file_path, re.IGNORECASE): package_path = os.path.dirname(self.file_path) try: md_file_path = glob.glob( os.path.join(os.path.dirname(self.file_path), '*_description.md'))[0] except IndexError: print_warning( "No detailed description file was found in the package {}." " Consider adding one.".format(package_path)) if md_file_path: is_description_in_package = True data_dictionary = get_yaml(self.file_path) if not data_dictionary: return is_description_in_package if data_dictionary.get('detaileddescription'): is_description_in_yml = True if is_description_in_package and is_description_in_yml: self._is_valid = False print_error( "A description was found both in the package and in the yml, " "please update the package {}.".format(package_path)) return False return True