def validate_all_files(self):
"""Validate all files in the repo are in the right format."""
# go over packs
for root, dirs, _ in os.walk(PACKS_DIR):
for dir_in_dirs in dirs:
for directory in PACKS_DIRECTORIES:
for inner_root, inner_dirs, files in os.walk(
os.path.join(root, dir_in_dirs, directory)):
for inner_dir in inner_dirs:
if inner_dir.startswith('.'):
continue
project_dir = os.path.join(inner_root, inner_dir)
_, file_path = get_yml_paths_in_dir(
os.path.normpath(project_dir),
Errors.no_yml_file(project_dir))
if file_path:
print("Validating {}".format(file_path))
structure_validator = StructureValidator(
file_path)
if not structure_validator.is_valid_scheme():
self._is_valid = False
# go over regular content entities
for directory in DIR_LIST_FOR_REGULAR_ENTETIES:
print_color('Validating {} directory:'.format(directory),
LOG_COLORS.GREEN)
for root, dirs, files in os.walk(directory):
for file_name in files:
file_path = os.path.join(root, file_name)
# skipping hidden files
if not file_name.endswith('.yml'):
continue
print('Validating ' + file_name)
structure_validator = StructureValidator(file_path)
if not structure_validator.is_valid_scheme():
self._is_valid = False
# go over regular PACKAGE_SUPPORTING_DIRECTORIES entities
for directory in PACKAGE_SUPPORTING_DIRECTORIES:
for root, dirs, files in os.walk(directory):
for inner_dir in dirs:
if inner_dir.startswith('.'):
continue
project_dir = os.path.join(root, inner_dir)
_, file_path = get_yml_paths_in_dir(
project_dir, Errors.no_yml_file(project_dir))
if file_path:
print('Validating ' + file_path)
structure_validator = StructureValidator(file_path)
if not structure_validator.is_valid_scheme():
self._is_valid = False
def copy_packs_content_to_packs_bundle(self, packs):
'''
Copy content in packs to the bundle that gets zipped to 'content_packs.zip'. Preserves directory structure
except that packages inside the "Integrations" or "Scripts" directory inside a pack are flattened. Adds file
prefixes according to how server expects to ingest the files, e.g. 'integration-' is prepended to integration
yml filenames and 'script-' is prepended to script yml filenames and so on and so forth.
'''
for pack in packs:
pack_name = os.path.basename(pack)
if pack_name in self.packs_to_skip:
continue
pack_dst = os.path.join(self.packs_bundle, pack_name)
os.mkdir(pack_dst)
pack_dirs = get_child_directories(pack)
pack_files = get_child_files(pack)
# copy first level pack files over
for file_path in pack_files:
shutil.copy(
file_path,
os.path.join(pack_dst, os.path.basename(file_path)))
# handle content directories in the pack
for content_dir in pack_dirs:
dir_name = os.path.basename(content_dir)
dest_dir = os.path.join(pack_dst, dir_name)
os.mkdir(dest_dir)
if dir_name in DIR_TO_PREFIX:
packages_dirs = get_child_directories(content_dir)
for package_dir in packages_dirs:
ymls, _ = get_yml_paths_in_dir(package_dir,
error_msg='')
if not ymls or (len(ymls) == 1
and ymls[0].endswith('_unified.yml')):
msg = 'Skipping package: {} -'.format(package_dir)
if not ymls:
print_warning(
'{} No yml files found in the package directory'
.format(msg))
else:
print_warning(
'{} Only unified yml found in the package directory'
.format(msg))
continue
package_dir_name = os.path.basename(package_dir)
unifier = Unifier(package_dir, dir_name, dest_dir)
unifier.merge_script_package_to_yml()
# also copy CHANGELOG markdown files over (should only be one per package)
package_files = get_child_files(package_dir)
changelog_files = [
file_path for file_path in package_files
if 'CHANGELOG.md' in file_path
]
for md_file_path in changelog_files:
md_out_name = '{}-{}_CHANGELOG.md'.format(
DIR_TO_PREFIX.get(dir_name), package_dir_name)
shutil.copyfile(
md_file_path,
os.path.join(dest_dir, md_out_name))
else:
self.copy_dir_files(content_dir, dest_dir)
def copy_dir_yml(self, dir_path, bundle):
"""
Copy the yml files inside a directory to a bundle.
:param dir_path: source directory
:param bundle: destination bundle
:return: None
"""
scan_files, _ = get_yml_paths_in_dir(dir_path, error_msg='')
content_files = 0
dir_name = os.path.basename(dir_path)
copy_func = self.copy_playbook_yml if dir_name in [
'Playbooks', 'TestPlaybooks'
] else self.copy_content_yml
for path in scan_files:
if len(os.path.basename(path)) >= self.file_name_max_size:
self.long_file_names.append(path)
with open(path, 'r') as file_:
yml_info = yaml.safe_load(file_)
ver = yml_info.get('fromversion', '0')
print(f' - processing: {ver} ({path})')
copy_func(path, os.path.join(bundle, os.path.basename(path)),
yml_info)
content_files += 1
print(f' - total files: {content_files}')
def test_get_yml_paths_in_dir(self, dir_path):
yml_paths, first_yml_path = tools.get_yml_paths_in_dir(dir_path, error_msg='')
yml_paths_test = glob.glob(os.path.join(dir_path, '*yml'))
assert sorted(yml_paths) == sorted(yml_paths_test)
if yml_paths_test:
assert first_yml_path == yml_paths_test[0]
else:
assert not first_yml_path
def merge_script_package_to_yml(self):
"""Merge the various components to create an output yml file
"""
print("Merging package: {}".format(self.package_path))
if self.package_path.endswith('/'):
self.package_path = self.package_path.rstrip('/')
package_dir_name = os.path.basename(self.package_path)
output_filename = '{}-{}.yml'.format(DIR_TO_PREFIX[self.dir_name],
package_dir_name)
if self.dest_path:
self.dest_path = os.path.join(self.dest_path, output_filename)
else:
self.dest_path = os.path.join(self.dir_name, output_filename)
yml_paths, yml_path = get_yml_paths_in_dir(
self.package_path, Errors.no_yml_file(self.package_path))
for path in yml_paths:
# The plugin creates a unified YML file for the package.
# In case this script runs locally and there is a unified YML file in the package we need to ignore it.
# Also,
# we don't take the unified file by default because
# there might be packages that were not created by the plugin.
if 'unified' not in path:
yml_path = path
break
with open(yml_path, 'r') as yml_file:
yml_data = yaml.safe_load(yml_file)
script_obj = yml_data
if self.dir_name != SCRIPTS_DIR:
script_obj = yml_data['script']
script_type = TYPE_TO_EXTENSION[script_obj['type']]
with io.open(yml_path, mode='r', encoding='utf-8') as yml_file:
yml_text = yml_file.read()
yml_text, script_path = self.insert_script_to_yml(
script_type, yml_text, yml_data)
image_path = None
desc_path = None
if self.dir_name in (INTEGRATIONS_DIR, BETA_INTEGRATIONS_DIR):
yml_text, image_path = self.insert_image_to_yml(yml_data, yml_text)
yml_text, desc_path = self.insert_description_to_yml(
yml_data, yml_text)
output_map = self.write_yaml_with_docker(yml_text, yml_data,
script_obj)
unifier_outputs = list(
output_map.keys()), yml_path, script_path, image_path, desc_path
print_color("Created unified yml: {}".format(unifier_outputs[0][0]),
LOG_COLORS.GREEN)
return unifier_outputs
def get_script_package_data(self):
_, yml_path = get_yml_paths_in_dir(self.package_path, error_msg='')
if not yml_path:
raise Exception("No yml files found in package path: {}. "
"Is this really a package dir? If not remove it.".format(self.package_path))
code_type = get_yaml(yml_path).get('type')
unifier = Unifier(self.package_path)
code_path = unifier.get_code_file(TYPE_TO_EXTENSION[code_type])
with open(code_path, 'r') as code_file:
code = code_file.read()
return yml_path, code
def create_unifieds_and_copy(self,
package_dir,
dest_dir='',
skip_dest_dir=''):
'''
For directories that have packages, aka subdirectories for each integration/script
e.g. "Integrations", "Beta_Integrations", "Scripts". Creates a unified yml and writes
it to the dest_dir
Arguments:
package_dir: (str)
Path to directory in which there are package subdirectories. e.g. "Integrations",
"Beta_Integrations", "Scripts"
dest_dir: (str)
Path to destination directory to which the unified yml for a package should be written
skip_dest_dir: (str)
Path to the directory to which the unified yml for a package should be written in the
case the package is part of the skipped list
'''
dest_dir = dest_dir if dest_dir else self.content_bundle
skip_dest_dir = skip_dest_dir if skip_dest_dir else self.test_bundle
scanned_packages = glob.glob(os.path.join(package_dir, '*/'))
package_dir_name = os.path.basename(package_dir)
for package in scanned_packages:
ymls, _ = get_yml_paths_in_dir(package, error_msg='')
if not ymls or (len(ymls) == 1
and ymls[0].endswith('_unified.yml')):
msg = 'Skipping package: {} -'.format(package)
if not ymls:
print_warning(
'{} No yml files found in the package directory'.
format(msg))
else:
print_warning(
'{} Only unified yml found in the package directory'.
format(msg))
continue
unification_tool = Unifier(package, package_dir_name, dest_dir)
if any(package_to_skip in package
for package_to_skip in self.packages_to_skip):
# there are some packages that we don't want to include in the content zip
# for example HelloWorld integration
unification_tool = Unifier(package, package_dir_name,
skip_dest_dir)
print('skipping {}'.format(package))
unification_tool.merge_script_package_to_yml()
def run_dev_packages(self) -> int:
return_code = 0
# load yaml
_, yml_path = get_yml_paths_in_dir(
self.project_dir, Errors.no_yml_file(self.project_dir))
if not yml_path:
return 1
print_v('Using yaml file: {}'.format(yml_path))
with open(yml_path, 'r') as yml_file:
yml_data = yaml.safe_load(yml_file)
script_obj = yml_data
if isinstance(script_obj.get('script'), dict):
script_obj = script_obj.get('script')
script_type = script_obj.get('type')
if script_type != 'python':
if script_type == 'powershell':
# TODO powershell linting
return 0
print(
'Script is not of type "python". Found type: {}. Nothing to do.'
.format(script_type))
return 0
dockers = get_all_docker_images(script_obj)
py_num = get_python_version(dockers[0], self.log_verbose)
self.lock.acquire()
print_color(
"============ Starting process for: {} ============\n".format(
self.project_dir), LOG_COLORS.YELLOW)
if self.lock.locked():
self.lock.release()
self._setup_dev_files(py_num)
if self.run_args['flake8']:
result_val = self.run_flake8(py_num)
if result_val:
return_code = result_val
if self.run_args['mypy']:
result_val = self.run_mypy(py_num)
if result_val:
return_code = result_val
if self.run_args['bandit']:
result_val = self.run_bandit(py_num)
if result_val:
return_code = result_val
for docker in dockers:
for try_num in (1, 2):
print_v("Using docker image: {}".format(docker))
py_num = get_python_version(docker, self.log_verbose)
try:
if self.run_args['tests'] or self.run_args['pylint']:
if py_num == 2.7:
requirements = self.requirements_2
else:
requirements = self.requirements_3
docker_image_created = self._docker_image_create(
docker, requirements)
output, status_code = self._docker_run(
docker_image_created)
self.lock.acquire()
print_color(
"\n========== Running tests/pylint for: {} ========="
.format(self.project_dir), LOG_COLORS.YELLOW)
if status_code == 1:
raise subprocess.CalledProcessError(*output)
else:
print(output)
print_color(
"============ Finished process for: {} "
"with docker: {} ============\n".format(
self.project_dir, docker),
LOG_COLORS.GREEN)
if self.lock.locked():
self.lock.release()
break # all is good no need to retry
except subprocess.CalledProcessError as ex:
if ex.output:
print_color(
"=========================== ERROR IN {}==========================="
"\n{}\n".format(self.project_dir, ex.output),
LOG_COLORS.RED)
else:
print_color(
"========= Test Failed on {}, Look at the error/s above ========\n"
.format(self.project_dir), LOG_COLORS.RED)
return_code = 1
if not self.log_verbose:
sys.stderr.write(
"Need a more detailed log? try running with the -v options as so: \n{} -v\n\n"
.format(" ".join(sys.argv[:])))
if self.lock.locked():
self.lock.release()
# circle ci docker setup sometimes fails on
if try_num > 1 or not ex.output or 'read: connection reset by peer' not in ex.output:
return 2
else:
sys.stderr.write(
"Retrying as failure seems to be docker communication related...\n"
)
finally:
sys.stdout.flush()
sys.stderr.flush()
return return_code
