def _get_attachment_name(attachment_name, file_path): name = FileUtils.basename(file_path) if attachment_name: name: str = attachment_name if not name.endswith(".zip"): name += ".zip" return name
def __init__(self, output_dir: str, args): self.request_limit = args.req_limit if hasattr(args, "req_limit") and args.req_limit else 1 self.full_email_conf: FullEmailConfig = FullEmailConfig(args) self.jenkins_url = args.jenkins_url self.job_name = args.job_name self.num_prev_days = args.num_prev_days tc_filters_raw = args.tc_filters if hasattr(args, "tc_filters") and args.tc_filters else [] self.tc_filters: List[TestcaseFilter] = [TestcaseFilter(*tcf.split(":")) for tcf in tc_filters_raw] if not self.tc_filters: LOG.warning("TESTCASE FILTER IS NOT SET!") self.send_mail: bool = not args.skip_mail self.enable_file_cache: bool = not args.disable_file_cache self.output_dir = ProjectUtils.get_session_dir_under_child_dir(FileUtils.basename(output_dir)) self.full_cmd: str = OsUtils.determine_full_command_filtered(filter_password=True)
def __init__(self, output_dir: str, args, branch_names: Dict[BranchType, str]): self.output_dir = ProjectUtils.get_session_dir_under_child_dir( FileUtils.basename(output_dir)) self.commit_author_exceptions = args.commit_author_exceptions self.console_mode = True if "console_mode" in args and args.console_mode else False self.save_to_file = not self.console_mode self.fail_on_missing_jira_id = False self.run_legacy_script = args.run_legacy_script self.legacy_compare_script_path = BranchComparatorConfig.find_git_compare_script( ) self.matching_algorithm: CommitMatchingAlgorithm = args.algorithm self.branch_names = branch_names self.repo_type: RepoType = (RepoType[args.repo_type.upper()] if hasattr(args, "repo_type") else RepoType.DOWNSTREAM) self.full_cmd: str or None = None
def _create_zip_file(src_files, file, compress=False): kwargs = {} if compress: kwargs["compression"] = zipfile.ZIP_DEFLATED kwargs[ "compresslevel"] = Z_DEFAULT_COMPRESSION # https://docs.python.org/3/library/zlib.html#zlib.compress zip_file = zipfile.ZipFile(file, "w", **kwargs) LOG.info( f"Creating zip file. Target file: {zip_file.filename}, Input files: {src_files}" ) for src_file in src_files: if FileUtils.is_dir(src_file): ZipFileUtils._add_dir_to_zip(src_file, zip_file) else: LOG.debug( f"Adding file '{src_file}' to zip file '${zip_file.filename}'" ) zip_file.write(src_file, FileUtils.basename(src_file)) zip_file.close() file.seek(0) return file
def __init__(self, output_dir: str, args, upstream_base_branch: str, upstream_repo: GitWrapper, downstream_repo: GitWrapper): self.output_dir = ProjectUtils.get_session_dir_under_child_dir( FileUtils.basename(output_dir)) self.execution_mode = (ExecutionMode.MANUAL_BRANCH_MODE if hasattr(args, "branches") and args.branches else ExecutionMode.AUTO_BRANCH_MODE) self.downstream_branches = args.branches if hasattr( args, "branches") else [] self.upstream_repo_path = upstream_repo.repo_path self.downstream_repo_path = downstream_repo.repo_path self.jira_id = args.jira_id self.output_dir = output_dir self.upstream_base_branch = upstream_base_branch self.force_mode = args.force_mode if hasattr(args, "force_mode") else False self.ignore_changes = args.ignore_changes if hasattr( args, "ignore_changes") else False self.full_cmd: str or None = None self._validate(downstream_repo) self.umbrella_result_basedir = FileUtils.join_path( self.output_dir, self.jira_id)
def __init__(self, parser, args, output_dir: str): self._validate_args(parser, args) self.console_mode = getattr(args, "console mode", False) self.gmail_query = args.gmail_query self.smart_subject_query = args.smart_subject_query self.request_limit = getattr(args, "request_limit", 1000000) self.account_email: str = args.account_email self.testcase_filters = TestCaseFilters( TestCaseFilters.convert_raw_match_expressions_to_objs( getattr(args, "match_expression", None)), getattr(args, "aggregate_filters", []), ) self.skip_lines_starting_with: List[str] = getattr( args, "skip_lines_starting_with", []) self.email_content_line_sep = getattr(args, "email_content_line_separator", DEFAULT_LINE_SEP) self.truncate_subject_with: str = getattr(args, "truncate_subject", None) self.abbrev_tc_package: str = getattr(args, "abbrev_testcase_package", None) self.summary_mode = args.summary_mode self.output_dir = output_dir self.email_cache_dir = FileUtils.join_path(output_dir, "email_cache") self.session_dir = ProjectUtils.get_session_dir_under_child_dir( FileUtils.basename(output_dir)) self.full_cmd: str = OsUtils.determine_full_command_filtered( filter_password=True) if self.operation_mode == OperationMode.GSHEET: worksheet_names: List[str] = [ self.get_worksheet_name(tcf) for tcf in self.testcase_filters.ALL_VALID_FILTERS ] LOG.info( f"Adding worksheets to {self.gsheet_options.__class__.__name__}. " f"Generated worksheet names: {worksheet_names}") for worksheet_name in worksheet_names: self.gsheet_options.add_worksheet(worksheet_name)
def run(self): LOG.info( "Starting zipping latest command data... \n " f"PLEASE NOTE THAT ACTUAL OUTPUT DIR AND DESTINATION FILES CAN CHANGE, IF NOT SPECIFIED\n" f"Output dir: {self.config.output_dir}\n" f"Input files: {self.config.input_files}\n " f"Destination filename: {self.config.dest_filename}\n " f"Ignore file types: {self.config.ignore_filetypes}\n ") input_files: List[str] = self.config.input_files sum_len_all_files: int = 0 all_ignores_files: int = 0 if self.config.ignore_filetypes: input_files = [] # TODO move this whole thing to pythoncommons for input_file in self.config.input_files: if FileUtils.is_dir(input_file): all_files = FileUtils.find_files(input_file, regex=".*", full_path_result=True) sum_len_all_files += len(all_files) files_to_ignore = set() for ext in self.config.ignore_filetypes: new_files_to_ignore = FileUtils.find_files( input_file, extension=ext, full_path_result=True) all_ignores_files += len(new_files_to_ignore) LOG.debug( f"Found {len(new_files_to_ignore)} files to ignore in directory '{input_file}': " f"{StringUtils.list_to_multiline_string(files_to_ignore)}" ) files_to_ignore.update(new_files_to_ignore) files_to_keep = list( set(all_files).difference(files_to_ignore)) tmp_dir: tempfile.TemporaryDirectory = tempfile.TemporaryDirectory( ) tmp_dir_path = tmp_dir.name FileUtils.copy_files_to_dir(files_to_keep, tmp_dir_path, cut_path=input_file) input_files.append(tmp_dir_path) else: input_files.append(input_file) sum_len_all_files += 1 temp_dir_dest: bool = True if not self.config.output_dir or self.config.output_dir.startswith( "/tmp") else False if self.config.output_dir: dest_filepath = FileUtils.join_path(self.config.output_dir, self.config.dest_filename) zip_file: BufferedWriter = ZipFileUtils.create_zip_file( input_files, dest_filepath, compress=True) else: zip_file: BufferedWriter = ZipFileUtils.create_zip_as_tmp_file( input_files, self.config.dest_filename, compress=True) zip_file_name = zip_file.name no_of_files_in_zip: int = ZipFileUtils.get_number_of_files_in_zip( zip_file_name) if self.config.ignore_filetypes and ( sum_len_all_files - all_ignores_files) != no_of_files_in_zip: raise ValueError(f"Unexpected number of files in zip. " f"All files: {sum_len_all_files}, " f"all ignored files: {all_ignores_files}, " f"number of files in zip: {no_of_files_in_zip}, " f"zip file: {zip_file_name}") LOG.info( f"Finished writing command data to zip file: {zip_file_name}, " f"size: {FileUtils.get_file_size(zip_file_name)}") FileUtils.create_symlink_path_dir(LATEST_DATA_ZIP_LINK_NAME, zip_file_name, self.config.project_out_root) # Create a latest link for the command as well FileUtils.create_symlink_path_dir(self.cmd_type.command_data_zip_name, zip_file_name, self.config.project_out_root) # Save command data file per command to home dir when temp dir mode is being used if temp_dir_dest: zip_file_name_real: str = f"{self.cmd_type.command_data_name}-real.zip" target_file_path = FileUtils.join_path( self.config.project_out_root, FileUtils.basename(zip_file_name_real)) FileUtils.copy_file(zip_file_name, target_file_path)