def copy_packs_content_to_packs_bundle(self, packs): ''' Copy content in packs to the bundle that gets zipped to 'content_packs.zip'. Preserves directory structure except that packages inside the "Integrations" or "Scripts" directory inside a pack are flattened. Adds file prefixes according to how server expects to ingest the files, e.g. 'integration-' is prepended to integration yml filenames and 'script-' is prepended to script yml filenames and so on and so forth. ''' for pack in packs: pack_name = os.path.basename(pack) if pack_name in self.packs_to_skip: continue pack_dst = os.path.join(self.packs_bundle, pack_name) os.mkdir(pack_dst) pack_dirs = get_child_directories(pack) pack_files = get_child_files(pack) # copy first level pack files over for file_path in pack_files: shutil.copy( file_path, os.path.join(pack_dst, os.path.basename(file_path))) # handle content directories in the pack for content_dir in pack_dirs: dir_name = os.path.basename(content_dir) dest_dir = os.path.join(pack_dst, dir_name) os.mkdir(dest_dir) if dir_name in DIR_TO_PREFIX: packages_dirs = get_child_directories(content_dir) for package_dir in packages_dirs: ymls, _ = get_yml_paths_in_dir(package_dir, error_msg='') if not ymls or (len(ymls) == 1 and ymls[0].endswith('_unified.yml')): msg = 'Skipping package: {} -'.format(package_dir) if not ymls: print_warning( '{} No yml files found in the package directory' .format(msg)) else: print_warning( '{} Only unified yml found in the package directory' .format(msg)) continue package_dir_name = os.path.basename(package_dir) unifier = Unifier(package_dir, dir_name, dest_dir) unifier.merge_script_package_to_yml() # also copy CHANGELOG markdown files over (should only be one per package) package_files = get_child_files(package_dir) changelog_files = [ file_path for file_path in package_files if 'CHANGELOG.md' in file_path ] for md_file_path in changelog_files: md_out_name = '{}-{}_CHANGELOG.md'.format( DIR_TO_PREFIX.get(dir_name), package_dir_name) shutil.copyfile( md_file_path, os.path.join(dest_dir, md_out_name)) else: self.copy_dir_files(content_dir, dest_dir)
def copy_packs_content_to_old_bundles(self, packs): ''' Copy relevant content (yml and json files) from packs to the appropriate bundle. Test playbooks to the bundle that gets zipped to 'content_test.zip' and the rest of the content to the bundle that gets zipped to 'content_new.zip'. Adds file prefixes where necessary according to how server expects to ingest the files. ''' for pack in packs: if os.path.basename(pack) in self.packs_to_skip: continue # each pack directory has it's own content subdirs, 'Integrations', # 'Scripts', 'TestPlaybooks', 'Layouts' etc. sub_dirs_paths = get_child_directories(pack) for sub_dir_path in sub_dirs_paths: dir_name = os.path.basename(sub_dir_path) if dir_name == 'TestPlaybooks': self.copy_test_files(sub_dir_path) else: # handle one-level deep content self.copy_dir_files(sub_dir_path, self.content_bundle) if dir_name in DIR_TO_PREFIX: # then it's a directory with nested packages that need to be handled # handle nested packages self.create_unifieds_and_copy(sub_dir_path)
def create_content(self): '''Creates the content artifact zip files "content_test.zip", "content_new.zip", and "content_packs.zip"''' print('Starting to create content artifact...') try: print('creating dir for bundles...') for bundle_dir in [ self.content_bundle, self.test_bundle, self.packs_bundle ]: os.mkdir(bundle_dir) self.add_tools_to_bundle(self.content_bundle) for package_dir in DIR_TO_PREFIX: # handles nested package directories self.create_unifieds_and_copy(package_dir) for content_dir in self.content_directories: print(f'Copying dir {content_dir} to bundles...') self.copy_dir_files(content_dir, self.content_bundle) self.copy_test_files() # handle copying packs content to bundles for zipping to content_new.zip and content_test.zip packs = get_child_directories(PACKS_DIR) self.copy_packs_content_to_old_bundles(packs) # handle copying packs content to packs_bundle for zipping to `content_packs.zip` self.copy_packs_content_to_packs_bundle(packs) print('Copying content descriptor to content and test bundles') for bundle_dir in [self.content_bundle, self.test_bundle]: shutil.copyfile( 'content-descriptor.json', os.path.join(bundle_dir, 'content-descriptor.json')) if os.path.exists('./Documentation/doc-CommonServer.json'): print('copying common server doc to content bundle') shutil.copyfile( './Documentation/doc-CommonServer.json', os.path.join(self.content_bundle, 'doc-CommonServer.json')) else: print_warning( './Documentation/doc-CommonServer.json was not found and ' 'therefore was not added to the content bundle') print('Compressing bundles...') shutil.make_archive(self.content_zip, 'zip', self.content_bundle) shutil.make_archive(self.test_zip, 'zip', self.test_bundle) shutil.make_archive(self.packs_zip, 'zip', self.packs_bundle) shutil.copyfile("./Tests/id_set.json", os.path.join(self.artifacts_path, "id_set.json")) if os.path.exists('release-notes.md'): print('copying release-notes.md to artifacts directory "{}"'. format(self.artifacts_path)) shutil.copyfile( 'release-notes.md', os.path.join(self.artifacts_path, 'release-notes.md')) else: print_warning( 'release-notes.md was not found in the content directory and therefore not ' 'copied over to the artifacts directory') print( f'finished creating the content artifacts at "{os.path.abspath(self.artifacts_path)}"' ) finally: if not self.preserve_bundles: if os.path.exists(self.content_bundle): shutil.rmtree(self.content_bundle) if os.path.exists(self.test_bundle): shutil.rmtree(self.test_bundle) if os.path.exists(self.packs_bundle): shutil.rmtree(self.packs_bundle)