def get_zipped_packs_names(zip_path):
"""
Creates a list of dictionaries containing a pack name as key and the latest zip file path of the pack as value.
Args:
zip_path: path containing all the packs copied from the storage bucket
Returns:
A dictionary containing each pack name and it's zip path.
{'Slack': 'content/packs/slack/1.3.19/slack.zip', 'qualys': 'content/packs/qualys/2.0/qualys.zip'}
"""
zipped_packs = {}
zip_path = os.path.join(zip_path, 'packs') # directory of the packs
dir_entries = os.listdir(zip_path)
packs_list = [pack.name for pack in os.scandir(PACKS_FULL_PATH)] # list of all packs from repo
for entry in dir_entries:
entry_path = os.path.join(zip_path, entry)
if entry not in IGNORED_FILES and entry in packs_list and os.path.isdir(entry_path):
# This is a pack directory, should keep only most recent release zip
pack_files = get_files_in_dir(entry_path, ['zip'])
latest_zip = get_latest_pack_zip_from_pack_files(entry, pack_files)
if not latest_zip:
logging.warning(f'Failed to get the zip of the pack {entry} from GCP')
continue
logging.info(f"Found latest zip of {entry}, which is {latest_zip}")
zipped_packs[Path(latest_zip).stem] = latest_zip
if not zipped_packs:
raise Exception('No zip files were found')
return zipped_packs
def build_pack_content_object(self, content_entity: str, entity_instance_path: str) -> dict:
"""
Build the pack content object the represents an entity instance.
For example: HelloWorld Integration in Packs/HelloWorld.
:param content_entity: The content entity, for example Integrations
:param entity_instance_path: For example, for integration: ~/.../content/Packs/TestPack/Integrations/HelloWorld
and for layout: ~/.../content/Packs/TestPack/Layout/layout-HelloWorldLayout.json
:return: A pack content object. For example, INTEGRATION_PACK_OBJECT / LAYOUT_PACK_OBJECT variables
in downloader_test.py
"""
# If the entity_instance_path is a file then get_files_in_dir will return the list: [entity_instance_path]
file_paths: list = get_files_in_dir(entity_instance_path, CONTENT_FILE_ENDINGS, recursive=False)
# If it's integration/script, all files under it should have the main details of the yml file,
# otherwise we'll use the file's details.
content_object: dict = dict()
main_id, main_name = self.get_main_file_details(content_entity, entity_instance_path)
# if main file doesn't exist/no entity instance path exist the content object won't be added to the pack content
if all((main_id, main_name, file_paths)):
content_object = {main_name: list()}
# For example take a look at INTEGRATION_CUSTOM_CONTENT_OBJECT variable in downloader_test.py
for file_path in file_paths:
content_object[main_name].append({
'name': main_name,
'id': main_id,
'path': file_path,
'file_ending': retrieve_file_ending(file_path)
})
return content_object
def test_recursive_pack(self):
pack_dir = 'demisto_sdk/commands/download/tests/tests_env/content/Packs/TestPack'
files = [
f'{pack_dir}/Integrations/TestIntegration/TestIntegration.py',
f'{pack_dir}/Integrations/TestIntegration/TestIntegration_testt.py',
f'{pack_dir}/Scripts/TestScript/TestScript.py'
]
assert sorted(get_files_in_dir(pack_dir, ['py'])) == sorted(files)
def test_not_recursive(self):
project_dir = 'demisto_sdk/commands/download'
files = [
f'{project_dir}/__init__.py', f'{project_dir}/downloader.py',
f'{project_dir}/README.md'
]
assert sorted(get_files_in_dir(project_dir, ['py', 'md'],
False)) == sorted(files)
def test_recursive(self):
integrations_dir = 'demisto_sdk/commands/download/tests/tests_env/content/Packs/TestPack/Integrations'
integration_instance_dir = f'{integrations_dir}/TestIntegration'
files = [
f'{integration_instance_dir}/TestIntegration.py',
f'{integration_instance_dir}/TestIntegration_testt.py'
]
assert sorted(get_files_in_dir(integrations_dir,
['py'])) == sorted(files)
def format_manager(input: str = None, output: str = None, from_version: str = '', no_validate: bool = None):
"""
Format_manager is a function that activated format command on different type of files.
Args:
input: (str) The path of the specific file.
from_version: (str) in case of specific value for from_version that needs to be updated.
output: (str) The path to save the formatted file to.
no_validate (flag): Whether the user specifies not to run validate after format.
Returns:
int 0 in case of success 1 otherwise
"""
if input:
files = get_files_in_dir(input, ['json', 'yml', 'py'])
else:
files = [file['name'] for file in
get_changed_files(filter_results=lambda _file: not _file.pop('status') == 'D')]
if output and not output.endswith(('yml', 'json', 'py')):
raise Exception("The given output path is not a specific file path.\n"
"Only file path can be a output path. Please specify a correct output.")
log_list = []
error_list = []
if files:
for file in files:
file_path = file.replace('\\', '/')
file_type = find_type(file_path)
if file_type:
file_type = file_type.value
info_res, err_res, skip_res = run_format_on_file(input=file_path, file_type=file_type,
from_version=from_version, output=output,
no_validate=no_validate)
if err_res:
error_list.append("err_res")
if err_res:
log_list.extend([(err_res, print_error)])
if info_res:
log_list.extend([(info_res, print_success)])
if skip_res:
log_list.extend([(skip_res, print_warning)])
else:
log_list.append(([f'Failed format file {input}.' + "No such file or directory"], print_error))
if error_list:
for string, print_func in log_list:
print_func('\n'.join(string))
return 1
for string, print_func in log_list:
print_func('\n'.join(string))
return 0
def check_separators_in_files(self):
"""
Check if there are separators in the script files names.
Returns:
true if the files names are valid and there is no separators, and false if not.
"""
# Gets the all script files that may have the script name as base name
files_to_check = get_files_in_dir(os.path.dirname(self.file_path),
['yml', 'py'], False)
valid_files = []
invalid_files = []
for file_path in files_to_check:
file_name = os.path.basename(file_path)
if file_name.endswith('_test.py') or file_name.endswith(
'_unified.yml'):
base_name = file_name.rsplit('_', 1)[0]
else:
base_name = file_name.rsplit('.', 1)[0]
valid_base_name = self.remove_separators_from_name(base_name)
if valid_base_name != base_name:
valid_files.append(
valid_base_name.join(file_name.rsplit(base_name, 1)))
invalid_files.append(file_name)
if invalid_files:
error_message, error_code = Errors.file_name_has_separators(
'script', invalid_files, valid_files)
if self.handle_error(error_message,
error_code,
file_path=self.file_path):
self.is_valid = False
return False
return True
def test_project_dir_is_file(self):
project_dir = 'demisto_sdk/commands/download/downloader.py'
assert get_files_in_dir(project_dir, ['py']) == [project_dir]
def format_manager(input: str = None,
output: str = None,
from_version: str = '',
no_validate: bool = False,
verbose: bool = False,
update_docker: bool = False,
assume_yes: bool = False,
deprecate: bool = False,
use_git: bool = False,
prev_ver: str = None,
include_untracked: bool = False,
add_tests: bool = None,
interactive: bool = True,
id_set_path: str = None,
clear_cache: bool = False):
"""
Format_manager is a function that activated format command on different type of files.
Args:
input: (str) The path of the specific file.
from_version: (str) in case of specific value for from_version that needs to be updated.
output: (str) The path to save the formatted file to.
no_validate (flag): Whether the user specifies not to run validate after format.
verbose (bool): Whether to print verbose logs or not
update_docker (flag): Whether to update the docker image.
assume_yes (bool): Whether to assume "yes" as answer to all prompts and run non-interactively
deprecate (bool): Whether to deprecate the entity
use_git (bool): Use git to automatically recognize which files changed and run format on them
prev_ver (str): Against which branch should the difference be recognized
include_untracked (bool): Whether to include untracked files when checking against git
interactive (bool): Whether to run the format interactively or not (usually for contribution management)
add_tests (bool): Whether to exclude tests automatically.
id_set_path (str): The path of the id_set.json file.
clear_cache (bool): wether to clear the cache
Returns:
int 0 in case of success 1 otherwise
"""
prev_ver = prev_ver if prev_ver else 'demisto/master'
supported_file_types = ['json', 'yml', 'py', 'md']
use_git = use_git or not input
if input:
files = get_files_in_dir(input, supported_file_types)
elif use_git:
files = get_files_to_format_from_git(supported_file_types, prev_ver,
include_untracked)
if output and not output.endswith(('yml', 'json', 'py')):
raise Exception(
"The given output path is not a specific file path.\n"
"Only file path can be a output path. Please specify a correct output."
)
log_list = []
error_list: List[Tuple[int, int]] = []
if files:
format_excluded_file = excluded_files + ['pack_metadata.json']
for file in files:
file_path = file.replace('\\', '/')
file_type = find_type(file_path, clear_cache=clear_cache)
current_excluded_files = format_excluded_file[:]
dirname = os.path.dirname(file_path)
if dirname.endswith('CommonServerPython'):
current_excluded_files.remove('CommonServerPython.py')
if os.path.basename(file_path) in current_excluded_files:
continue
if any(test_dir in str(dirname)
for test_dir in TESTS_AND_DOC_DIRECTORIES):
continue
if file_type and file_type.value not in UNFORMATTED_FILES:
file_type = file_type.value
info_res, err_res, skip_res = run_format_on_file(
input=file_path,
file_type=file_type,
from_version=from_version,
interactive=interactive,
output=output,
no_validate=no_validate,
verbose=verbose,
update_docker=update_docker,
assume_yes=assume_yes,
deprecate=deprecate,
add_tests=add_tests,
id_set_path=id_set_path)
if err_res:
log_list.extend([(err_res, print_error)])
if info_res:
log_list.extend([(info_res, print_success)])
if skip_res:
log_list.extend([(skip_res, print_warning)])
elif file_type:
log_list.append(([
f"Ignoring format for {file_path} as {file_type.value} is currently not "
f"supported by format command"
], print_warning))
else:
log_list.append(([
f"Was unable to identify the file type for the following file: {file_path}"
], print_error))
update_content_entity_ids(files, verbose)
else:
if not use_git:
log_list.append(([
f'Failed format file {input}.' + "No such file or directory"
], print_error))
return 1
print('') # Just adding a new line before summary
for string, print_func in log_list:
print_func('\n'.join(string))
if error_list:
return 1
return 0
def format_manager(input: str = None,
output: str = None,
from_version: str = '',
no_validate: bool = False,
verbose: bool = False,
update_docker: bool = False,
assume_yes: bool = False):
"""
Format_manager is a function that activated format command on different type of files.
Args:
input: (str) The path of the specific file.
from_version: (str) in case of specific value for from_version that needs to be updated.
output: (str) The path to save the formatted file to.
no_validate (flag): Whether the user specifies not to run validate after format.
verbose (bool): Whether to print verbose logs or not
update_docker (flag): Whether to update the docker image.
assume_yes (bool): Whether to assume "yes" as answer to all prompts and run non-interactively
Returns:
int 0 in case of success 1 otherwise
"""
if input:
files = get_files_in_dir(input, ['json', 'yml', 'py'])
else:
files = [file['name'] for file in
get_changed_files(filter_results=lambda _file: not _file.pop('status') == 'D')]
if output and not output.endswith(('yml', 'json', 'py')):
raise Exception("The given output path is not a specific file path.\n"
"Only file path can be a output path. Please specify a correct output.")
log_list = []
error_list = []
if files:
format_excluded_file = excluded_files + ['pack_metadata.json']
for file in files:
file_path = file.replace('\\', '/')
file_type = find_type(file_path)
current_excluded_files = format_excluded_file[:]
dirname = os.path.dirname(file_path)
if dirname.endswith('CommonServerPython'):
current_excluded_files.remove('CommonServerPython.py')
if os.path.basename(file_path) in current_excluded_files:
continue
if dirname.endswith('test_data') or dirname.endswith('doc_imgs'):
continue
if file_type and file_type.value not in UNFORMATTED_FILES:
file_type = file_type.value
info_res, err_res, skip_res = run_format_on_file(input=file_path,
file_type=file_type,
from_version=from_version,
output=output,
no_validate=no_validate,
verbose=verbose,
update_docker=update_docker,
assume_yes=assume_yes)
if err_res:
error_list.append("err_res")
if err_res:
log_list.extend([(err_res, print_error)])
if info_res:
log_list.extend([(info_res, print_success)])
if skip_res:
log_list.extend([(skip_res, print_warning)])
elif file_type:
log_list.append(([f"Ignoring format for {file_path} as {file_type.value} is currently not "
f"supported by format command"], print_warning))
else:
log_list.append(([f"Was unable to identify the file type for the following file: {file_path}"],
print_error))
else:
log_list.append(([f'Failed format file {input}.' + "No such file or directory"], print_error))
print('') # Just adding a new line before summary
for string, print_func in log_list:
print_func('\n'.join(string))
if error_list:
return 1
return 0
