def copy_packs_content_to_packs_bundle(self, packs):
'''
Copy content in packs to the bundle that gets zipped to 'content_packs.zip'. Preserves directory structure
except that packages inside the "Integrations" or "Scripts" directory inside a pack are flattened. Adds file
prefixes according to how server expects to ingest the files, e.g. 'integration-' is prepended to integration
yml filenames and 'script-' is prepended to script yml filenames and so on and so forth.
'''
for pack in packs:
pack_name = os.path.basename(pack)
if pack_name in self.packs_to_skip:
continue
pack_dst = os.path.join(self.packs_bundle, pack_name)
os.mkdir(pack_dst)
pack_dirs = get_child_directories(pack)
pack_files = get_child_files(pack)
# copy first level pack files over
for file_path in pack_files:
shutil.copy(
file_path,
os.path.join(pack_dst, os.path.basename(file_path)))
# handle content directories in the pack
for content_dir in pack_dirs:
dir_name = os.path.basename(content_dir)
dest_dir = os.path.join(pack_dst, dir_name)
os.mkdir(dest_dir)
if dir_name in DIR_TO_PREFIX:
packages_dirs = get_child_directories(content_dir)
for package_dir in packages_dirs:
ymls, _ = get_yml_paths_in_dir(package_dir,
error_msg='')
if not ymls or (len(ymls) == 1
and ymls[0].endswith('_unified.yml')):
msg = 'Skipping package: {} -'.format(package_dir)
if not ymls:
print_warning(
'{} No yml files found in the package directory'
.format(msg))
else:
print_warning(
'{} Only unified yml found in the package directory'
.format(msg))
continue
package_dir_name = os.path.basename(package_dir)
unifier = Unifier(package_dir, dir_name, dest_dir)
unifier.merge_script_package_to_yml()
# also copy CHANGELOG markdown files over (should only be one per package)
package_files = get_child_files(package_dir)
changelog_files = [
file_path for file_path in package_files
if 'CHANGELOG.md' in file_path
]
for md_file_path in changelog_files:
md_out_name = '{}-{}_CHANGELOG.md'.format(
DIR_TO_PREFIX.get(dir_name), package_dir_name)
shutil.copyfile(
md_file_path,
os.path.join(dest_dir, md_out_name))
else:
self.copy_dir_files(content_dir, dest_dir)
def copy_packs_content_to_old_bundles(self, packs):
'''
Copy relevant content (yml and json files) from packs to the appropriate bundle. Test playbooks to the
bundle that gets zipped to 'content_test.zip' and the rest of the content to the bundle that gets zipped to
'content_new.zip'. Adds file prefixes where necessary according to how server expects to ingest the files.
'''
for pack in packs:
if os.path.basename(pack) in self.packs_to_skip:
continue
# each pack directory has it's own content subdirs, 'Integrations',
# 'Scripts', 'TestPlaybooks', 'Layouts' etc.
sub_dirs_paths = get_child_directories(pack)
for sub_dir_path in sub_dirs_paths:
dir_name = os.path.basename(sub_dir_path)
if dir_name == 'TestPlaybooks':
self.copy_test_files(sub_dir_path)
else:
# handle one-level deep content
self.copy_dir_files(sub_dir_path, self.content_bundle)
if dir_name in DIR_TO_PREFIX:
# then it's a directory with nested packages that need to be handled
# handle nested packages
self.create_unifieds_and_copy(sub_dir_path)
def create_content(self):
'''Creates the content artifact zip files "content_test.zip", "content_new.zip", and "content_packs.zip"'''
print('Starting to create content artifact...')
try:
print('creating dir for bundles...')
for bundle_dir in [
self.content_bundle, self.test_bundle, self.packs_bundle
]:
os.mkdir(bundle_dir)
self.add_tools_to_bundle(self.content_bundle)
for package_dir in DIR_TO_PREFIX:
# handles nested package directories
self.create_unifieds_and_copy(package_dir)
for content_dir in self.content_directories:
print(f'Copying dir {content_dir} to bundles...')
self.copy_dir_files(content_dir, self.content_bundle)
self.copy_test_files()
# handle copying packs content to bundles for zipping to content_new.zip and content_test.zip
packs = get_child_directories(PACKS_DIR)
self.copy_packs_content_to_old_bundles(packs)
# handle copying packs content to packs_bundle for zipping to `content_packs.zip`
self.copy_packs_content_to_packs_bundle(packs)
print('Copying content descriptor to content and test bundles')
for bundle_dir in [self.content_bundle, self.test_bundle]:
shutil.copyfile(
'content-descriptor.json',
os.path.join(bundle_dir, 'content-descriptor.json'))
if os.path.exists('./Documentation/doc-CommonServer.json'):
print('copying common server doc to content bundle')
shutil.copyfile(
'./Documentation/doc-CommonServer.json',
os.path.join(self.content_bundle, 'doc-CommonServer.json'))
else:
print_warning(
'./Documentation/doc-CommonServer.json was not found and '
'therefore was not added to the content bundle')
print('Compressing bundles...')
shutil.make_archive(self.content_zip, 'zip', self.content_bundle)
shutil.make_archive(self.test_zip, 'zip', self.test_bundle)
shutil.make_archive(self.packs_zip, 'zip', self.packs_bundle)
shutil.copyfile("./Tests/id_set.json",
os.path.join(self.artifacts_path, "id_set.json"))
if os.path.exists('release-notes.md'):
print('copying release-notes.md to artifacts directory "{}"'.
format(self.artifacts_path))
shutil.copyfile(
'release-notes.md',
os.path.join(self.artifacts_path, 'release-notes.md'))
else:
print_warning(
'release-notes.md was not found in the content directory and therefore not '
'copied over to the artifacts directory')
print(
f'finished creating the content artifacts at "{os.path.abspath(self.artifacts_path)}"'
)
finally:
if not self.preserve_bundles:
if os.path.exists(self.content_bundle):
shutil.rmtree(self.content_bundle)
if os.path.exists(self.test_bundle):
shutil.rmtree(self.test_bundle)
if os.path.exists(self.packs_bundle):
shutil.rmtree(self.packs_bundle)