def insert_script_to_yml(self, script_type, yml_unified, yml_data): script_path = self.get_code_file(script_type) with io.open(script_path, mode='r', encoding='utf-8') as script_file: script_code = script_file.read() # Check if the script imports an API module. If it does, # the API module code will be pasted in place of the import. module_import, module_name = self.check_api_module_imports(script_code) if module_import: script_code = self.insert_module_code(script_code, module_import, module_name) if script_type == '.py': clean_code = self.clean_python_code(script_code) if script_type == '.ps1': clean_code = self.clean_pwsh_code(script_code) if self.is_script_package: if yml_data.get('script', '') not in ('', '-'): print_warning( f'Script section is not empty in package {self.package_path}.' f'It should be blank or a dash(-).') yml_unified['script'] = FoldedScalarString(clean_code) else: if yml_data['script'].get('script', '') not in ('', '-'): print_warning( f'Script section is not empty in package {self.package_path}.' f'It should be blank or a dash(-).') yml_unified['script']['script'] = FoldedScalarString(clean_code) return yml_unified, script_path
def has_duplicate(id_set, id_to_check, object_type=None): duplicates = [duplicate for duplicate in id_set if duplicate.get(id_to_check)] if len(duplicates) < 2: return False for dup1, dup2 in itertools.combinations(duplicates, 2): dict1 = list(dup1.values())[0] dict2 = list(dup2.values())[0] dict1_from_version = LooseVersion(dict1.get('fromversion', '0.0.0')) dict2_from_version = LooseVersion(dict2.get('fromversion', '0.0.0')) dict1_to_version = LooseVersion(dict1.get('toversion', '99.99.99')) dict2_to_version = LooseVersion(dict2.get('toversion', '99.99.99')) if dict1['name'] != dict2['name']: print_warning('The following {} have the same ID ({}) but different names: ' '"{}", "{}".'.format(object_type, id_to_check, dict1['name'], dict2['name'])) # A: 3.0.0 - 3.6.0 # B: 3.5.0 - 4.5.0 # C: 3.5.2 - 3.5.4 # D: 4.5.0 - 99.99.99 if any([ dict1_from_version <= dict2_from_version < dict1_to_version, # will catch (B, C), (A, B), (A, C) dict1_from_version < dict2_to_version <= dict1_to_version, # will catch (B, C), (A, C) dict2_from_version <= dict1_from_version < dict2_to_version, # will catch (C, B), (B, A), (C, A) dict2_from_version < dict1_to_version <= dict2_to_version, # will catch (C, B), (C, A) ]): return True return False
def main(): option = option_handler() storage_bucket_name = option.bucket_name zip_path = option.zip_path artifacts_path = option.artifacts_path service_account = option.service_account circle_build = option.circle_build branch_name = option.branch_name gcp_path = option.gcp_path remove_test_playbooks = option.remove_test_playbooks # google cloud storage client initialized storage_client = init_storage_client(service_account) storage_bucket = storage_client.bucket(storage_bucket_name) if not circle_build or not branch_name: # Ignore build properties circle_build = '' branch_name = '' if not gcp_path: gcp_path = BUILD_GCP_PATH zipped_packs = [] success = True try: zipped_packs = download_packs_from_gcp(storage_bucket, gcp_path, zip_path, circle_build, branch_name) except Exception as e: print_error(f'Failed downloading packs: {e}') success = False if remove_test_playbooks: try: remove_test_playbooks_if_exist(zip_path, zipped_packs) except Exception as e: print_error(f'Failed removing test playbooks from packs: {e}') success = False if zipped_packs and success: try: zip_packs(zipped_packs, zip_path) except Exception as e: print_error(f'Failed zipping packs: {e}') success = False if success: print_success('Successfully zipped packs.') if artifacts_path: # Save in the artifacts shutil.copy(os.path.join(zip_path, ARTIFACT_NAME), os.path.join(artifacts_path, ARTIFACT_NAME)) else: print_error('Failed zipping packs.') sys.exit(1) else: print_warning('Did not find any packs to zip.') cleanup(zip_path)
def get_command_examples(commands_examples_input, specific_commands): """ get command examples from command file @param commands_examples_input: commands examples file or a comma separeted list of com @param specific_commands: commands specified by the user @return: a list of command examples """ if not commands_examples_input: return [] if os.path.isfile(commands_examples_input): with open(commands_examples_input, 'r') as examples_file: command_examples = examples_file.read().splitlines() else: print_warning( 'failed to open commands file, using commands as comma seperated list' ) command_examples = commands_examples_input.split(',') # Filter from the examples only the commands specified by the user if specific_commands: command_examples = [ command_ex for command_ex in command_examples if command_ex.split(' ')[0].strip('!') in specific_commands ] command_examples = list( filter(None, map(command_example_filter, command_examples))) or [] print('found the following commands:\n{}'.format( '\n'.join(command_examples))) return command_examples
def get_base_branch(pr_num): """Fetches the base branch name of PR num {pr_num} Args: pr_num (string): The string representation of the pr number Returns: string. The name of the base branch of the pr if succeeds, '' otherwise. """ # Disable insecure warnings requests.packages.urllib3.disable_warnings() # pylint: disable=no-member url = 'https://api.github.com/repos/demisto/content/pulls/{}'.format( pr_num) try: res = requests.get(url, verify=False) res.raise_for_status() pr = res.json() if pr and isinstance(pr, list) and len(pr) == 1: # github usually returns a list of PRs, if not pr is a dict pr = pr[0] return pr.get('base', {}).get('ref', '') except (requests.exceptions.HTTPError, ValueError) as e: # If we didn't succeed to fetch pr for any http error / res.json() we raise an error # then we don't want the build to fail print_warning('Unable to fetch pull request #{0}.\nError: {1}'.format( pr_num, str(e))) return ''
def get_new_entity_record(entity_path: str) -> Tuple[str, str]: data, _ = get_dict_from_file(entity_path) if 'layouts' in entity_path.lower(): layout_kind = LAYOUT_TYPE_TO_NAME.get(data.get('kind', '')) type_id = data.get('typeId', '') return f'{type_id} - {layout_kind}', '' name = data.get('name', entity_path) if 'integrations' in entity_path.lower() and data.get('display'): name = data.get('display') if 'classifiers' in entity_path.lower(): name = data.get('name') if not name: name = data.get('brandName') if name == entity_path: print_error(f'missing name for {entity_path}') # script entities has "comment" instead of "description" description = data.get('description', '') or data.get('comment', '') if not description: print_warning(f'missing description for {entity_path}') return name, description
def check_docker_image_changed(main_branch: str, packfile: str) -> Optional[str]: """ Checks whether the docker image was changed in master. :param main_branch: The git main branch packfile: The added or modified yml path :rtype: ``Optional[str]`` :return The latest docker image """ try: diff = run_command(f'git diff {main_branch} -- {packfile}', exit_on_error=False) except RuntimeError as e: if any(['is outside repository' in exp for exp in e.args]): return None else: print_warning( f'skipping docker image check, Encountered the following error:\n{e.args[0]}' ) return None else: diff_lines = diff.splitlines() for diff_line in diff_lines: if 'dockerimage:' in diff_line: # search whether exists a line that notes that the Docker image was # changed. split_line = diff_line.split() if split_line[0].startswith('+'): return split_line[-1] return None
def create_file_release_notes(change_type, full_file_name): """ Create release note for changed file. :param change_type: git change status (A, M, R*) :param full_file_name: path to file in repository :return: None """ if isinstance(full_file_name, tuple): _, full_file_name = full_file_name is_pack = is_file_path_in_pack(full_file_name) if is_pack: file_type = full_file_name.split("/")[2] else: file_type = full_file_name.split("/")[0] base_name = os.path.basename(full_file_name) file_suffix = os.path.splitext(base_name)[-1] file_type_mapping = RELEASE_NOTE_GENERATOR.get(file_type) if file_type_mapping is None or file_suffix not in CONTENT_FILE_SUFFIXES: print_warning("Unsupported file type: {}".format(full_file_name)) return if change_type != "R100": # only file name has changed (no actual data was modified if 'R' in change_type: # handle the same as modified change_type = 'M' file_type_mapping.add(change_type, CONTENT_LIB_PATH + full_file_name)
def create_markdown(release_notes_path: str, rn_string: str): if os.path.exists(release_notes_path): print_warning( f"Release notes were found at {release_notes_path}. Skipping") else: with open(release_notes_path, 'w') as fp: fp.write(rn_string)
def execute_update(self): if self.pack in IGNORED_PACK_NAMES: print_warning( f"Release notes are not required for the {self.pack} pack since this pack" f" is not versioned.") else: try: new_version, new_metadata = self.bump_version_number( self.pre_release) except ValueError as e: print_error(e) sys.exit(1) rn_path = self.return_release_notes_path(new_version) self.check_rn_dir(rn_path) changed_files = {} self.find_added_pack_files() for packfile in self.pack_files: file_name, file_type = self.identify_changed_file_type( packfile) changed_files[file_name] = file_type rn_string = self.build_rn_template(changed_files) if len(rn_string) > 0: self.commit_to_bump(new_metadata) self.create_markdown(rn_path, rn_string, changed_files) else: print_warning( "No changes which would belong in release notes were detected." )
def are_modules_installed_for_verify(self) -> bool: """ Check the following: 1. npm packages installed - see packs var for specific pack details. 2. node interperter exists. Returns: bool: True If all req ok else False """ missing_module = [] valid = True # Check node exist stdout, stderr, exit_code = run_command_os('node -v', cwd=self.content_path) if exit_code: print_warning( f'There is no node installed on the machine, Test Skipped, error - {stderr}, {stdout}' ) valid = False else: # Check npm modules exsits packs = ['@mdx-js/mdx', 'fs-extra', 'commander'] for pack in packs: stdout, stderr, exit_code = run_command_os( f'npm ls {pack}', cwd=self.content_path) if exit_code: missing_module.append(pack) if missing_module: valid = False print_warning( f"The npm modules: {missing_module} are not installed, Test Skipped, use " f"'npm install <module>' to install all required node dependencies" ) return valid
def is_docker_image_latest_tag(self): if 'demisto/python:1.3-alpine' == f'{self.docker_image_name}:{self.docker_image_tag}': # the docker image is the default one self.is_latest_tag = False print_error( 'The current docker image in the yml file is the default one: demisto/python:1.3-alpine,\n' 'Please create or use another docker image\n') return self.is_latest_tag if not self.docker_image_name or not self.docker_image_latest_tag: # If the docker image isn't in the format we expect it to be or we failed fetching the tag # We don't want to print any error msgs to user because they have already been printed self.is_latest_tag = False return self.is_latest_tag if self.docker_image_latest_tag != self.docker_image_tag: # If docker image tag is not the most updated one that exists in docker-hub print_warning( 'The docker image tag is not the latest, please update it.\n' 'The docker image tag in the yml file is: {}\n' 'The latest docker image tag in docker hub is: {}\n' 'You can check for the most updated version of {} here: https://hub.docker.com/r/{}/tags\n' .format(self.docker_image_tag, self.docker_image_latest_tag, self.docker_image_name, self.docker_image_name)) # the most updated tag should be numeric and not labeled "latest" if self.docker_image_latest_tag == "latest": self.is_latest_tag = False print_error( '"latest" tag is not allowed,\n' 'Please create or update to an updated versioned image\n' 'You can check for the most updated version of {} here: https://hub.docker.com/r/{}/tags\n' .format(self.docker_image_tag, self.docker_image_name)) return self.is_latest_tag
def get_private_packs(private_index_path): """ Get the list of ID and price of the private packs. Args: private_index_path: The path for the index of the private packs. Returns: private_packs: A list of ID and price of the private packs. """ try: metadata_files = glob.glob(f"{private_index_path}/**/metadata.json") except Exception as e: print_warning(f'Could not find metadata files in {private_index_path}: {str(e)}') return [] if not metadata_files: print_warning(f'No metadata files found in [{private_index_path}]') private_packs = [] for metadata_file_path in metadata_files: try: with open(metadata_file_path, "r") as metadata_file: metadata = json.load(metadata_file) if metadata: private_packs.append({ 'id': metadata.get('id'), 'price': metadata.get('price') }) except ValueError as e: print_error(f'Invalid JSON in the metadata file [{metadata_file_path}]: {str(e)}') return private_packs
def test_instances(secret_conf_path, server, username, password): integrations = get_integrations(secret_conf_path) instance_ids = [] failed_integrations = [] integrations_counter = 0 prints_manager = ParallelPrintsManager(1) content_installation_client = demisto_client.configure(base_url=server, username=username, password=password, verify_ssl=False) install_new_content(content_installation_client, server) for integration in integrations: c = demisto_client.configure(base_url=server, username=username, password=password, verify_ssl=False) integrations_counter += 1 integration_name = integration.get('name') integration_instance_name = integration.get('instance_name', '') integration_params = integration.get('params') devops_comments = integration.get('devops_comments') product_description = integration.get('product_description', '') is_byoi = integration.get('byoi', True) has_integration = integration.get('has_integration', True) validate_test = integration.get('validate_test', True) if has_integration: instance_id, failure_message, _ = __create_integration_instance( c, integration_name, integration_instance_name, integration_params, is_byoi, prints_manager, validate_test=validate_test) if failure_message == 'No configuration': print_warning( "Warning: skipping {} as it exists in content-test-conf conf.json but not " "in content repo".format(integration_name)) continue if not instance_id: print_error( 'Failed to create instance of {} with message: {}'.format( integration_name, failure_message)) failed_integrations.append( "{} {} - devops comments: {}".format( integration_name, product_description, devops_comments)) else: instance_ids.append(instance_id) print('Create integration %s succeed' % (integration_name, )) __delete_integrations_instances(c, instance_ids, prints_manager) prints_manager.execute_thread_prints(0) return failed_integrations, integrations_counter
def upload_core_packs_config(storage_bucket, packs_list): """Uploads corepacks.json file configuration to bucket. corepacks file includes core packs for server installation. Args: storage_bucket (google.cloud.storage.bucket.Bucket): gcs bucket where core packs config is uploaded. packs_list (list): list of initialized packs. """ # todo later check if it is not pre release and only then upload corepacks.json core_packs_public_urls = [c.public_storage_path for c in packs_list if c.name in GCPConfig.CORE_PACKS_LIST and c.public_storage_path] if not core_packs_public_urls: print(f"No core packs detected, skipping {GCPConfig.CORE_PACK_FILE_NAME} upload") return if len(core_packs_public_urls) != len(GCPConfig.CORE_PACKS_LIST): print_warning(f"Found core packs does not match configured core packs. " f"Found {len(core_packs_public_urls)} and configured {len(GCPConfig.CORE_PACKS_LIST)}, " f"skipping {GCPConfig.CORE_PACK_FILE_NAME} upload") # construct core pack data with public gcs urls core_packs_data = { 'corePacks': core_packs_public_urls } core_packs_config_path = os.path.join(GCPConfig.STORAGE_BASE_PATH, GCPConfig.CORE_PACK_FILE_NAME) blob = storage_bucket.blob(core_packs_config_path) blob.upload_from_string(json.dumps(core_packs_data, indent=4)) print_color(f"Finished uploading {GCPConfig.CORE_PACK_FILE_NAME} to storage.", LOG_COLORS.GREEN)
def update_content_version(self, content_ver: str = '', path: str = ''): regex = r'CONTENT_RELEASE_VERSION = .*' if not content_ver: try: with open('content-descriptor.json') as file_: descriptor = json.load(file_) content_ver = descriptor['release'] except (FileNotFoundError, json.JSONDecodeError, KeyError): print_error( 'Invalid descriptor file. make sure file content is a valid json with "release" key.' ) return try: if self.no_update_commonserverpython: return if not path: path = get_common_server_path('.') with open(path, 'r+') as file_: content = file_.read() content = re.sub(regex, f"CONTENT_RELEASE_VERSION = '{content_ver}'", content, re.M) file_.seek(0) file_.write(content) except Exception as ex: print_warning(f'Could not open CommonServerPython File - {ex}')
def copy_docs_files(content_bundle_path, packs_bundle_path): for doc_file in ('./Documentation/doc-CommonServer.json', './Documentation/doc-howto.json'): if os.path.exists(doc_file): if content_bundle_path: print(f'copying {doc_file} doc to content bundle\n') shutil.copyfile( doc_file, os.path.join(content_bundle_path, os.path.basename(doc_file))) # copy doc to packs bundle print(f'copying {doc_file} doc to content pack bundle\n') base_pack_doc_path = os.path.join(packs_bundle_path, BASE_PACK, "Documentation") if not os.path.exists(base_pack_doc_path): os.mkdir(base_pack_doc_path) shutil.copy( doc_file, os.path.join(base_pack_doc_path, os.path.basename(doc_file))) else: print_warning(f'{doc_file} was not found and ' 'therefore was not added to the content bundle')
def handle_existing_rn_with_docker_image(new_rn: str, header_by_type: str, docker_image: str, content_name: str) -> str: """ Receives the new RN to be written, performs operations to add the docker image to the given RN. Args: new_rn (str): new RN. header_by_type (str): Header of the RN to add docker image to, e.g 'Integrations', 'Scripts' docker_image (str): Docker image to add content_name (str): The content name to add the docker image entry to, e.g integration name, script name. Returns: (str): Updated RN """ # Writing or re-writing docker image to release notes. rn_parts = new_rn.split(header_by_type) new_rn_part = f'- Updated the Docker image to: *{docker_image}*.' if len(rn_parts) > 1: # Splitting again by content name to append the docker image release note to corresponding # content entry only content_parts = rn_parts[1].split(f'{content_name}\n') new_rn = f'{rn_parts[0]}{header_by_type}{content_parts[0]}{content_name}\n{new_rn_part}\n' \ f'{content_parts[1]}' else: print_warning( f'Could not parse release notes {new_rn} by header type: {header_by_type}' ) return new_rn
def get_remote_templates(self, files_list): """ Downloading the object related template-files and saving them in the output path. Args: files_list: List of files to download. Returns: bool. True if the files were downloaded and saved successfully, False otherwise. """ if self.is_integration: path = os.path.join('Packs', 'HelloWorld', 'Integrations', 'HelloWorld') os.mkdir(os.path.join(self.full_output_path, self.TEST_DATA_DIR)) else: path = os.path.join('Packs', 'HelloWorld', 'Scripts', 'HelloWorldScript') for file in files_list: try: file_content = tools.get_remote_file(os.path.join(path, file), return_content=True) with open(os.path.join(self.full_output_path, file), 'wb') as f: f.write(file_content) except Exception: print_warning( f"Could not fetch remote template - {file}. Using local templates instead." ) return False return True
def is_duplicate_description(self): """Check if the integration has a non-duplicate description .""" is_description_in_yml = False is_description_in_package = False package_path = None md_file_path = None if not re.match(PACKS_INTEGRATION_YML_REGEX, self.file_path, re.IGNORECASE): package_path = os.path.dirname(self.file_path) try: md_file_path = glob.glob(os.path.join(os.path.dirname(self.file_path), '*_description.md'))[0] except IndexError: print_warning("No detailed description file was found in the package {}." " Consider adding one.".format(package_path)) if md_file_path: is_description_in_package = True data_dictionary = get_yaml(self.file_path) if not data_dictionary: return is_description_in_package if data_dictionary.get('detaileddescription'): is_description_in_yml = True if is_description_in_package and is_description_in_yml: error_message, error_code = Errors.description_in_package_and_yml() if self.handle_error(error_message, error_code, file_path=package_path): self._is_valid = False return False return True
def are_modules_installed_for_verify(content_path: str) -> bool: """ Check the following: 1. npm packages installed - see packs var for specific pack details. 2. node interperter exists. Returns: bool: True If all req ok else False """ missing_module = [] valid = True # Check node exist stdout, stderr, exit_code = run_command_os('node -v', cwd=content_path) if exit_code: print_warning( f'There is no node installed on the machine, Test Skipped, error - {stderr}, {stdout}' ) valid = False else: # Check npm modules exsits stdout, stderr, exit_code = run_command_os( f'npm ls --json {" ".join(REQUIRED_MDX_PACKS)}', cwd=content_path) if exit_code: # all are missinig missing_module.extend(REQUIRED_MDX_PACKS) else: deps = json.loads(stdout).get('dependencies', {}) for pack in REQUIRED_MDX_PACKS: if pack not in deps: missing_module.append(pack) if missing_module: valid = False print_warning( f"The npm modules: {missing_module} are not installed, Readme mdx validation skipped. Use " f"'npm install' to install all required node dependencies") return valid
def _parse_stats_result(cls, stats_lines): """Parses the docker statics str and converts to Mib. Args: stats_lines (str): String that contains docker stats. Returns: list: List of dictionaries with parsed docker container statistics. """ stats_result = [] try: containers_stats = [json.loads(c) for c in stats_lines.splitlines()] for container_stat in containers_stats: memory_usage_stats = container_stat.get(cls.MEMORY_USAGE, '').split('/')[0].lower() if 'kib' in memory_usage_stats: mib_usage = float(memory_usage_stats.replace('kib', '').strip()) / 1024 elif 'gib' in memory_usage_stats: mib_usage = float(memory_usage_stats.replace('kib', '').strip()) * 1024 else: mib_usage = float(memory_usage_stats.replace('mib', '').strip()) stats_result.append({ 'memory_usage': mib_usage, 'pids': int(container_stat.get(cls.PIDS_USAGE)), 'container_name': container_stat.get(cls.CONTAINER_NAME), 'container_id': container_stat.get(cls.CONTAINER_ID) }) except Exception as e: print_warning("Failed in parsing docker stats result, returned empty list. Additional info: {}".format(e)) finally: return stats_result
def update_api_modules_dependents_rn(_pack, pre_release, update_type, added, modified, id_set_path=None): print_warning( "Changes introduced to APIModule, trying to update dependent integrations." ) if not id_set_path: if not os.path.isfile('./Tests/id_set.json'): print_error( "Failed to update integrations dependent on the APIModule pack - no id_set.json is " "available. Please run `demisto-sdk create-id-set` to generate it, and rerun this command." ) return id_set_path = './Tests/id_set.json' with open(id_set_path, 'r') as conf_file: id_set = json.load(conf_file) api_module_set = get_api_module_ids(added) api_module_set = api_module_set.union(get_api_module_ids(modified)) integrations = get_api_module_integrations_set( api_module_set, id_set.get('integrations', [])) for integration in integrations: integration_path = integration.get('file_path') integration_pack = integration.get('pack') update_pack_rn = UpdateRN(pack_path=integration_pack, update_type=update_type, modified_files_in_pack={integration_path}, pre_release=pre_release, added_files=set(), pack=integration_pack) update_pack_rn.execute_update()
def add_pr_comment(comment): """Add comment to the pull request. Args: comment (string): The comment text. """ token = os.environ['CONTENT_GITHUB_TOKEN'] branch_name = os.environ['CIRCLE_BRANCH'] sha1 = os.environ['CIRCLE_SHA1'] query = f'?q={sha1}+repo:demisto/content+is:pr+is:open+head:{branch_name}+is:open' url = 'https://api.github.com/search/issues' headers = {'Authorization': 'Bearer ' + token} try: res = requests.get(url + query, headers=headers, verify=False) res = handle_github_response(res) if res and res.get('total_count', 0) == 1: issue_url = res['items'][0].get('comments_url') if res.get( 'items', []) else None if issue_url: res = requests.post(issue_url, json={'body': comment}, headers=headers, verify=False) handle_github_response(res) else: print_warning( 'Add pull request comment failed: There is more then one open pull request for branch {}.' .format(branch_name)) except Exception as e: print_warning('Add pull request comment failed: {}'.format(e))
def copy_packs_content_to_packs_bundle(self, packs): """ Copy content in packs to the bundle that gets zipped to 'content_packs.zip'. Preserves directory structure except that packages inside the "Integrations" or "Scripts" directory inside a pack are flattened. Adds file prefixes according to how server expects to ingest the files, e.g. 'integration-' is prepended to integration yml filenames and 'script-' is prepended to script yml filenames and so on and so forth. """ for pack in packs: pack_name = os.path.basename(pack) if pack_name in self.packs_to_skip: continue pack_dst = os.path.join(self.packs_bundle, pack_name) os.mkdir(pack_dst) pack_dirs = get_child_directories(pack) pack_files = get_child_files(pack) # copy first level pack files over for file_path in pack_files: shutil.copy( file_path, os.path.join(pack_dst, os.path.basename(file_path))) # handle content directories in the pack for content_dir in pack_dirs: dir_name = os.path.basename(content_dir) dest_dir = os.path.join(pack_dst, dir_name) os.mkdir(dest_dir) if dir_name in DIR_TO_PREFIX: packages_dirs = get_child_directories(content_dir) for package_dir in packages_dirs: ymls, _ = get_yml_paths_in_dir(package_dir, error_msg='') if not ymls or (len(ymls) == 1 and ymls[0].endswith('_unified.yml')): msg = 'Skipping package: {} -'.format(package_dir) if not ymls: print_warning( '{} No yml files found in the package directory' .format(msg)) else: print_warning( '{} Only unified yml found in the package directory' .format(msg)) continue package_dir_name = os.path.basename(package_dir) unifier = Unifier(package_dir, dir_name, dest_dir) unifier.merge_script_package_to_yml() # also copy CHANGELOG markdown files over (should only be one per package) package_files = get_child_files(package_dir) changelog_files = [ file_path for file_path in package_files if 'CHANGELOG.md' in file_path ] for md_file_path in changelog_files: md_out_name = '{}-{}_CHANGELOG.md'.format( DIR_TO_PREFIX.get(dir_name), package_dir_name) shutil.copyfile( md_file_path, os.path.join(dest_dir, md_out_name)) else: self.copy_dir_files(content_dir, dest_dir)
def get_file_description(path, file_type) -> str: """ Gets the file description. :param path: The file path file_type: The file type :rtype: ``str`` :return The file description if exists otherwise returns %%UPDATE_RN%% """ if not os.path.isfile(path): print_warning( f'Cannot get file description: "{path}" file does not exist') return '' elif file_type in (FileType.PLAYBOOK, FileType.INTEGRATION): yml_file = get_yaml(path) return yml_file.get('description', '') elif file_type == FileType.SCRIPT: yml_file = get_yaml(path) return yml_file.get('comment', '') elif file_type in (FileType.CLASSIFIER, FileType.REPORT, FileType.WIDGET, FileType.DASHBOARD, FileType.JOB): json_file = get_json(path) return json_file.get('description', '') return '%%UPDATE_RN%%'
def print_packs_summary(packs_list): """Prints summary of packs uploaded to gcs. Args: packs_list (list): list of initialized packs. """ successful_packs = [pack for pack in packs_list if pack.status == PackStatus.SUCCESS.name] skipped_packs = [pack for pack in packs_list if pack.status == PackStatus.PACK_ALREADY_EXISTS.name] failed_packs = [pack for pack in packs_list if pack not in successful_packs and pack not in skipped_packs] print("\n") print("--------------------------------------- Packs Upload Summary ---------------------------------------") print(f"Total number of packs: {len(packs_list)}") if successful_packs: print_color(f"Number of successful uploaded packs: {len(successful_packs)}", LOG_COLORS.GREEN) successful_packs_table = _build_summary_table(successful_packs) print_color(successful_packs_table, LOG_COLORS.GREEN) if skipped_packs: print_warning(f"Number of skipped packs: {len(skipped_packs)}") skipped_packs_table = _build_summary_table(skipped_packs) print_warning(skipped_packs_table) if failed_packs: print_error(f"Number of failed packs: {len(failed_packs)}") failed_packs_table = _build_summary_table(failed_packs) print_error(failed_packs_table) sys.exit(1)
def is_aliased_fields_are_valid(self) -> bool: """ Validates that the aliased fields (fields that appear as Aliases in another field) are valid. invalid aliased fields are 1. fields that are in another field's Aliases list, and present in the same marketplace of that field. 2. fields that are in another field's Aliases list, and also contain Aliases (nested aliasing) Returns: (bool): True if aliased fields are valid. """ if not self.id_set_file: print_warning('Validate will skip since an id set file was not provided') return True aliases = self.current_file.get('Aliases', []) if not aliases: return True is_valid = True validators_and_error_generators = [ (self.is_alias_has_invalid_marketplaces, Errors.invalid_marketplaces_in_alias), (self.is_alias_has_inner_alias, Errors.aliases_with_inner_alias), ] for validator, error_generator in validators_and_error_generators: invalid_aliases = [alias.get("cliname") for alias in self._get_incident_fields_by_aliases(aliases) if validator(alias)] if invalid_aliases: error_message, error_code = error_generator(invalid_aliases) if self.handle_error(error_message, error_code, file_path=self.file_path, warning=self.structure_validator.quiet_bc): is_valid = False return is_valid
def main(): """ Main function for iterating over existing packs folder in content repo and creating json of all packs dependencies. The logic of pack dependency is identical to sdk find-dependencies command. """ option = option_handler() output_path = option.output_path id_set_path = option.id_set_path IGNORED_FILES.append( GCPConfig.BASE_PACK) # skip dependency calculation of Base pack # loading id set json with open(id_set_path, 'r') as id_set_file: id_set = json.load(id_set_file) pack_dependencies_result = {} print("Starting dependencies calculation") # starting iteration over pack folders for pack in os.scandir(PACKS_FULL_PATH): if not pack.is_dir() or pack.name in IGNORED_FILES: print_warning( f"Skipping dependency calculation of {pack.name} pack.") continue # skipping ignored packs print(f"Calculating {pack.name} pack dependencies.") try: dependency_graph = PackDependencies.build_dependency_graph( pack_id=pack.name, id_set=id_set, verbose_file=VerboseFile(''), ) first_level_dependencies, all_level_dependencies = parse_for_pack_metadata( dependency_graph, pack.name) except Exception as e: print_error( f"Failed calculating {pack.name} pack dependencies. Additional info:\n{e}" ) continue pack_dependencies_result[pack.name] = { "dependencies": first_level_dependencies, "displayedImages": list(first_level_dependencies.keys()), "allLevelDependencies": all_level_dependencies, "path": os.path.join(PACKS_FOLDER, pack.name), "fullPath": pack.path } print( f"Number of created pack dependencies entries: {len(pack_dependencies_result.keys())}" ) # finished iteration over pack folders print_color("Finished dependencies calculation", LOG_COLORS.GREEN) with open(output_path, 'w') as pack_dependencies_file: json.dump(pack_dependencies_result, pack_dependencies_file, indent=4) print_color(f"Created packs dependencies file at: {output_path}", LOG_COLORS.GREEN)
def create_pack_rn(self, rn_path: str, changed_files: dict, new_metadata: dict, new_version: str) -> bool: """ Checks whether the pack requires a new rn and if so, creates it. :param rn_path (str): The rn path changed_files (dict): The changed files details new_metadata (dict): The new pack metadata new_version (str): The new version str representation, e.g 1.0.2, 1.11.2 etc. :rtype: ``bool`` :return Whether the RN was updated successfully or not """ rn_string = self.handle_existing_rn_version_path(rn_path) if not rn_string: rn_string = self.build_rn_template(changed_files) if len(rn_string) > 0 or self.is_force: if self.is_bump_required(): self.write_metadata_to_file(new_metadata) self.create_markdown(rn_path, rn_string, changed_files) self.build_rn_config_file(new_version) try: run_command(f'git add {rn_path}', exit_on_error=False) except RuntimeError: print_warning( f'Could not add the release note files to git: {rn_path}') if self.is_bc and self.bc_path: try: run_command(f'git add {self.bc_path}', exit_on_error=False) except RuntimeError: print_warning( f'Could not add the release note config file to git: {rn_path}' ) if self.existing_rn_changed: print_color( f"Finished updating release notes for {self.pack}.", LOG_COLORS.GREEN) if not self.text: print_color( f"\nNext Steps:\n - Please review the " f"created release notes found at {rn_path} and document any changes you " f"made by replacing '%%UPDATE_RN%%'.\n - Commit " f"the new release notes to your branch.\nFor information regarding proper" f" format of the release notes, please refer to " f"https://xsoar.pan.dev/docs/integrations/changelog", LOG_COLORS.GREEN) return True else: print_color( f"No changes to {self.pack} pack files were detected from the previous time " "this command was run. The release notes have not been " "changed.", LOG_COLORS.GREEN) else: print_color( "No changes which would belong in release notes were detected.", LOG_COLORS.YELLOW) return False