def branch_comparator(self, args, parser=None): branch_comparator = BranchComparator(args, self.downstream_repo, self.upstream_repo, self.branch_comparator_output_dir) FileUtils.create_symlink_path_dir( CommandType.BRANCH_COMPARATOR.session_link_name, branch_comparator.config.output_dir, self.project_out_root) branch_comparator.run()
def unit_test_result_aggregator(self, args, parser=None): ut_results_aggregator = UnitTestResultAggregator( args, parser, self.unit_test_result_aggregator_output_dir) FileUtils.create_symlink_path_dir( CommandType.UNIT_TEST_RESULT_AGGREGATOR.session_link_name, ut_results_aggregator.config.session_dir, self.project_out_root, ) ut_results_aggregator.run()
def fetch_jira_umbrella_data(self, args, parser=None): jira_umbrella_fetcher = UpstreamJiraUmbrellaFetcher( args, self.upstream_repo, self.downstream_repo, self.jira_umbrella_data_dir, DEFAULT_BASE_BRANCH) FileUtils.create_symlink_path_dir( CommandType.FETCH_JIRA_UMBRELLA_DATA.session_link_name, jira_umbrella_fetcher.config.umbrella_result_basedir, self.project_out_root, ) jira_umbrella_fetcher.run()
def run(self): LOG.info( "Starting zipping latest command data... \n " f"PLEASE NOTE THAT ACTUAL OUTPUT DIR AND DESTINATION FILES CAN CHANGE, IF NOT SPECIFIED\n" f"Output dir: {self.config.output_dir}\n" f"Input files: {self.config.input_files}\n " f"Destination filename: {self.config.dest_filename}\n " f"Ignore file types: {self.config.ignore_filetypes}\n ") input_files: List[str] = self.config.input_files sum_len_all_files: int = 0 all_ignores_files: int = 0 if self.config.ignore_filetypes: input_files = [] # TODO move this whole thing to pythoncommons for input_file in self.config.input_files: if FileUtils.is_dir(input_file): all_files = FileUtils.find_files(input_file, regex=".*", full_path_result=True) sum_len_all_files += len(all_files) files_to_ignore = set() for ext in self.config.ignore_filetypes: new_files_to_ignore = FileUtils.find_files( input_file, extension=ext, full_path_result=True) all_ignores_files += len(new_files_to_ignore) LOG.debug( f"Found {len(new_files_to_ignore)} files to ignore in directory '{input_file}': " f"{StringUtils.list_to_multiline_string(files_to_ignore)}" ) files_to_ignore.update(new_files_to_ignore) files_to_keep = list( set(all_files).difference(files_to_ignore)) tmp_dir: tempfile.TemporaryDirectory = tempfile.TemporaryDirectory( ) tmp_dir_path = tmp_dir.name FileUtils.copy_files_to_dir(files_to_keep, tmp_dir_path, cut_path=input_file) input_files.append(tmp_dir_path) else: input_files.append(input_file) sum_len_all_files += 1 temp_dir_dest: bool = True if not self.config.output_dir or self.config.output_dir.startswith( "/tmp") else False if self.config.output_dir: dest_filepath = FileUtils.join_path(self.config.output_dir, self.config.dest_filename) zip_file: BufferedWriter = ZipFileUtils.create_zip_file( input_files, dest_filepath, compress=True) else: zip_file: BufferedWriter = ZipFileUtils.create_zip_as_tmp_file( input_files, self.config.dest_filename, compress=True) zip_file_name = zip_file.name no_of_files_in_zip: int = ZipFileUtils.get_number_of_files_in_zip( zip_file_name) if self.config.ignore_filetypes and ( sum_len_all_files - all_ignores_files) != no_of_files_in_zip: raise ValueError(f"Unexpected number of files in zip. " f"All files: {sum_len_all_files}, " f"all ignored files: {all_ignores_files}, " f"number of files in zip: {no_of_files_in_zip}, " f"zip file: {zip_file_name}") LOG.info( f"Finished writing command data to zip file: {zip_file_name}, " f"size: {FileUtils.get_file_size(zip_file_name)}") FileUtils.create_symlink_path_dir(LATEST_DATA_ZIP_LINK_NAME, zip_file_name, self.config.project_out_root) # Create a latest link for the command as well FileUtils.create_symlink_path_dir(self.cmd_type.command_data_zip_name, zip_file_name, self.config.project_out_root) # Save command data file per command to home dir when temp dir mode is being used if temp_dir_dest: zip_file_name_real: str = f"{self.cmd_type.command_data_name}-real.zip" target_file_path = FileUtils.join_path( self.config.project_out_root, FileUtils.basename(zip_file_name_real)) FileUtils.copy_file(zip_file_name, target_file_path)
# TODO Revisit all exception handling: ValueError vs. exit() calls # Methods should throw exceptions, exit should be handled in this method yarn_dev_tools = YarnDevTools() # Parse args, commands will be mapped to YarnDevTools functions in ArgParser.parse_args args, parser = ArgParser.parse_args(yarn_dev_tools) log_file = Setup.init_logger( execution_mode=ExecutionMode.PRODUCTION, console_debug=args.debug, postfix=args.command, repos=[ yarn_dev_tools.upstream_repo.repo, yarn_dev_tools.downstream_repo.repo ], verbose=args.verbose, ) cmd_type = CommandType.from_str(args.command) if cmd_type not in IGNORE_LATEST_SYMLINK_COMMANDS: FileUtils.create_symlink_path_dir(cmd_type.log_link_name, log_file, yarn_dev_tools.project_out_root) else: LOG.info( f"Skipping to re-create symlink as command is: {args.command}") # Call the handler function args.func(args, parser=parser) end_time = time.time() LOG.info("Execution of script took %d seconds", end_time - start_time)