def _get_attachment_name(attachment_name, file_path):
name = FileUtils.basename(file_path)
if attachment_name:
name: str = attachment_name
if not name.endswith(".zip"):
name += ".zip"
return name
def __init__(self, output_dir: str, args):
self.request_limit = args.req_limit if hasattr(args, "req_limit") and args.req_limit else 1
self.full_email_conf: FullEmailConfig = FullEmailConfig(args)
self.jenkins_url = args.jenkins_url
self.job_name = args.job_name
self.num_prev_days = args.num_prev_days
tc_filters_raw = args.tc_filters if hasattr(args, "tc_filters") and args.tc_filters else []
self.tc_filters: List[TestcaseFilter] = [TestcaseFilter(*tcf.split(":")) for tcf in tc_filters_raw]
if not self.tc_filters:
LOG.warning("TESTCASE FILTER IS NOT SET!")
self.send_mail: bool = not args.skip_mail
self.enable_file_cache: bool = not args.disable_file_cache
self.output_dir = ProjectUtils.get_session_dir_under_child_dir(FileUtils.basename(output_dir))
self.full_cmd: str = OsUtils.determine_full_command_filtered(filter_password=True)
def __init__(self, output_dir: str, args, branch_names: Dict[BranchType,
str]):
self.output_dir = ProjectUtils.get_session_dir_under_child_dir(
FileUtils.basename(output_dir))
self.commit_author_exceptions = args.commit_author_exceptions
self.console_mode = True if "console_mode" in args and args.console_mode else False
self.save_to_file = not self.console_mode
self.fail_on_missing_jira_id = False
self.run_legacy_script = args.run_legacy_script
self.legacy_compare_script_path = BranchComparatorConfig.find_git_compare_script(
)
self.matching_algorithm: CommitMatchingAlgorithm = args.algorithm
self.branch_names = branch_names
self.repo_type: RepoType = (RepoType[args.repo_type.upper()]
if hasattr(args, "repo_type") else
RepoType.DOWNSTREAM)
self.full_cmd: str or None = None
def _create_zip_file(src_files, file, compress=False):
kwargs = {}
if compress:
kwargs["compression"] = zipfile.ZIP_DEFLATED
kwargs[
"compresslevel"] = Z_DEFAULT_COMPRESSION # https://docs.python.org/3/library/zlib.html#zlib.compress
zip_file = zipfile.ZipFile(file, "w", **kwargs)
LOG.info(
f"Creating zip file. Target file: {zip_file.filename}, Input files: {src_files}"
)
for src_file in src_files:
if FileUtils.is_dir(src_file):
ZipFileUtils._add_dir_to_zip(src_file, zip_file)
else:
LOG.debug(
f"Adding file '{src_file}' to zip file '${zip_file.filename}'"
)
zip_file.write(src_file, FileUtils.basename(src_file))
zip_file.close()
file.seek(0)
return file
def __init__(self, output_dir: str, args, upstream_base_branch: str,
upstream_repo: GitWrapper, downstream_repo: GitWrapper):
self.output_dir = ProjectUtils.get_session_dir_under_child_dir(
FileUtils.basename(output_dir))
self.execution_mode = (ExecutionMode.MANUAL_BRANCH_MODE
if hasattr(args, "branches") and args.branches
else ExecutionMode.AUTO_BRANCH_MODE)
self.downstream_branches = args.branches if hasattr(
args, "branches") else []
self.upstream_repo_path = upstream_repo.repo_path
self.downstream_repo_path = downstream_repo.repo_path
self.jira_id = args.jira_id
self.output_dir = output_dir
self.upstream_base_branch = upstream_base_branch
self.force_mode = args.force_mode if hasattr(args,
"force_mode") else False
self.ignore_changes = args.ignore_changes if hasattr(
args, "ignore_changes") else False
self.full_cmd: str or None = None
self._validate(downstream_repo)
self.umbrella_result_basedir = FileUtils.join_path(
self.output_dir, self.jira_id)
def __init__(self, parser, args, output_dir: str):
self._validate_args(parser, args)
self.console_mode = getattr(args, "console mode", False)
self.gmail_query = args.gmail_query
self.smart_subject_query = args.smart_subject_query
self.request_limit = getattr(args, "request_limit", 1000000)
self.account_email: str = args.account_email
self.testcase_filters = TestCaseFilters(
TestCaseFilters.convert_raw_match_expressions_to_objs(
getattr(args, "match_expression", None)),
getattr(args, "aggregate_filters", []),
)
self.skip_lines_starting_with: List[str] = getattr(
args, "skip_lines_starting_with", [])
self.email_content_line_sep = getattr(args,
"email_content_line_separator",
DEFAULT_LINE_SEP)
self.truncate_subject_with: str = getattr(args, "truncate_subject",
None)
self.abbrev_tc_package: str = getattr(args, "abbrev_testcase_package",
None)
self.summary_mode = args.summary_mode
self.output_dir = output_dir
self.email_cache_dir = FileUtils.join_path(output_dir, "email_cache")
self.session_dir = ProjectUtils.get_session_dir_under_child_dir(
FileUtils.basename(output_dir))
self.full_cmd: str = OsUtils.determine_full_command_filtered(
filter_password=True)
if self.operation_mode == OperationMode.GSHEET:
worksheet_names: List[str] = [
self.get_worksheet_name(tcf)
for tcf in self.testcase_filters.ALL_VALID_FILTERS
]
LOG.info(
f"Adding worksheets to {self.gsheet_options.__class__.__name__}. "
f"Generated worksheet names: {worksheet_names}")
for worksheet_name in worksheet_names:
self.gsheet_options.add_worksheet(worksheet_name)
def run(self):
LOG.info(
"Starting zipping latest command data... \n "
f"PLEASE NOTE THAT ACTUAL OUTPUT DIR AND DESTINATION FILES CAN CHANGE, IF NOT SPECIFIED\n"
f"Output dir: {self.config.output_dir}\n"
f"Input files: {self.config.input_files}\n "
f"Destination filename: {self.config.dest_filename}\n "
f"Ignore file types: {self.config.ignore_filetypes}\n ")
input_files: List[str] = self.config.input_files
sum_len_all_files: int = 0
all_ignores_files: int = 0
if self.config.ignore_filetypes:
input_files = []
# TODO move this whole thing to pythoncommons
for input_file in self.config.input_files:
if FileUtils.is_dir(input_file):
all_files = FileUtils.find_files(input_file,
regex=".*",
full_path_result=True)
sum_len_all_files += len(all_files)
files_to_ignore = set()
for ext in self.config.ignore_filetypes:
new_files_to_ignore = FileUtils.find_files(
input_file, extension=ext, full_path_result=True)
all_ignores_files += len(new_files_to_ignore)
LOG.debug(
f"Found {len(new_files_to_ignore)} files to ignore in directory '{input_file}': "
f"{StringUtils.list_to_multiline_string(files_to_ignore)}"
)
files_to_ignore.update(new_files_to_ignore)
files_to_keep = list(
set(all_files).difference(files_to_ignore))
tmp_dir: tempfile.TemporaryDirectory = tempfile.TemporaryDirectory(
)
tmp_dir_path = tmp_dir.name
FileUtils.copy_files_to_dir(files_to_keep,
tmp_dir_path,
cut_path=input_file)
input_files.append(tmp_dir_path)
else:
input_files.append(input_file)
sum_len_all_files += 1
temp_dir_dest: bool = True if not self.config.output_dir or self.config.output_dir.startswith(
"/tmp") else False
if self.config.output_dir:
dest_filepath = FileUtils.join_path(self.config.output_dir,
self.config.dest_filename)
zip_file: BufferedWriter = ZipFileUtils.create_zip_file(
input_files, dest_filepath, compress=True)
else:
zip_file: BufferedWriter = ZipFileUtils.create_zip_as_tmp_file(
input_files, self.config.dest_filename, compress=True)
zip_file_name = zip_file.name
no_of_files_in_zip: int = ZipFileUtils.get_number_of_files_in_zip(
zip_file_name)
if self.config.ignore_filetypes and (
sum_len_all_files - all_ignores_files) != no_of_files_in_zip:
raise ValueError(f"Unexpected number of files in zip. "
f"All files: {sum_len_all_files}, "
f"all ignored files: {all_ignores_files}, "
f"number of files in zip: {no_of_files_in_zip}, "
f"zip file: {zip_file_name}")
LOG.info(
f"Finished writing command data to zip file: {zip_file_name}, "
f"size: {FileUtils.get_file_size(zip_file_name)}")
FileUtils.create_symlink_path_dir(LATEST_DATA_ZIP_LINK_NAME,
zip_file_name,
self.config.project_out_root)
# Create a latest link for the command as well
FileUtils.create_symlink_path_dir(self.cmd_type.command_data_zip_name,
zip_file_name,
self.config.project_out_root)
# Save command data file per command to home dir when temp dir mode is being used
if temp_dir_dest:
zip_file_name_real: str = f"{self.cmd_type.command_data_name}-real.zip"
target_file_path = FileUtils.join_path(
self.config.project_out_root,
FileUtils.basename(zip_file_name_real))
FileUtils.copy_file(zip_file_name, target_file_path)
