def branch_comparator(self, args, parser=None):
branch_comparator = BranchComparator(args, self.downstream_repo,
self.upstream_repo,
self.branch_comparator_output_dir)
FileUtils.create_symlink_path_dir(
CommandType.BRANCH_COMPARATOR.session_link_name,
branch_comparator.config.output_dir, self.project_out_root)
branch_comparator.run()
def unit_test_result_aggregator(self, args, parser=None):
ut_results_aggregator = UnitTestResultAggregator(
args, parser, self.unit_test_result_aggregator_output_dir)
FileUtils.create_symlink_path_dir(
CommandType.UNIT_TEST_RESULT_AGGREGATOR.session_link_name,
ut_results_aggregator.config.session_dir,
self.project_out_root,
)
ut_results_aggregator.run()
def fetch_jira_umbrella_data(self, args, parser=None):
jira_umbrella_fetcher = UpstreamJiraUmbrellaFetcher(
args, self.upstream_repo, self.downstream_repo,
self.jira_umbrella_data_dir, DEFAULT_BASE_BRANCH)
FileUtils.create_symlink_path_dir(
CommandType.FETCH_JIRA_UMBRELLA_DATA.session_link_name,
jira_umbrella_fetcher.config.umbrella_result_basedir,
self.project_out_root,
)
jira_umbrella_fetcher.run()
def run(self):
LOG.info(
"Starting zipping latest command data... \n "
f"PLEASE NOTE THAT ACTUAL OUTPUT DIR AND DESTINATION FILES CAN CHANGE, IF NOT SPECIFIED\n"
f"Output dir: {self.config.output_dir}\n"
f"Input files: {self.config.input_files}\n "
f"Destination filename: {self.config.dest_filename}\n "
f"Ignore file types: {self.config.ignore_filetypes}\n ")
input_files: List[str] = self.config.input_files
sum_len_all_files: int = 0
all_ignores_files: int = 0
if self.config.ignore_filetypes:
input_files = []
# TODO move this whole thing to pythoncommons
for input_file in self.config.input_files:
if FileUtils.is_dir(input_file):
all_files = FileUtils.find_files(input_file,
regex=".*",
full_path_result=True)
sum_len_all_files += len(all_files)
files_to_ignore = set()
for ext in self.config.ignore_filetypes:
new_files_to_ignore = FileUtils.find_files(
input_file, extension=ext, full_path_result=True)
all_ignores_files += len(new_files_to_ignore)
LOG.debug(
f"Found {len(new_files_to_ignore)} files to ignore in directory '{input_file}': "
f"{StringUtils.list_to_multiline_string(files_to_ignore)}"
)
files_to_ignore.update(new_files_to_ignore)
files_to_keep = list(
set(all_files).difference(files_to_ignore))
tmp_dir: tempfile.TemporaryDirectory = tempfile.TemporaryDirectory(
)
tmp_dir_path = tmp_dir.name
FileUtils.copy_files_to_dir(files_to_keep,
tmp_dir_path,
cut_path=input_file)
input_files.append(tmp_dir_path)
else:
input_files.append(input_file)
sum_len_all_files += 1
temp_dir_dest: bool = True if not self.config.output_dir or self.config.output_dir.startswith(
"/tmp") else False
if self.config.output_dir:
dest_filepath = FileUtils.join_path(self.config.output_dir,
self.config.dest_filename)
zip_file: BufferedWriter = ZipFileUtils.create_zip_file(
input_files, dest_filepath, compress=True)
else:
zip_file: BufferedWriter = ZipFileUtils.create_zip_as_tmp_file(
input_files, self.config.dest_filename, compress=True)
zip_file_name = zip_file.name
no_of_files_in_zip: int = ZipFileUtils.get_number_of_files_in_zip(
zip_file_name)
if self.config.ignore_filetypes and (
sum_len_all_files - all_ignores_files) != no_of_files_in_zip:
raise ValueError(f"Unexpected number of files in zip. "
f"All files: {sum_len_all_files}, "
f"all ignored files: {all_ignores_files}, "
f"number of files in zip: {no_of_files_in_zip}, "
f"zip file: {zip_file_name}")
LOG.info(
f"Finished writing command data to zip file: {zip_file_name}, "
f"size: {FileUtils.get_file_size(zip_file_name)}")
FileUtils.create_symlink_path_dir(LATEST_DATA_ZIP_LINK_NAME,
zip_file_name,
self.config.project_out_root)
# Create a latest link for the command as well
FileUtils.create_symlink_path_dir(self.cmd_type.command_data_zip_name,
zip_file_name,
self.config.project_out_root)
# Save command data file per command to home dir when temp dir mode is being used
if temp_dir_dest:
zip_file_name_real: str = f"{self.cmd_type.command_data_name}-real.zip"
target_file_path = FileUtils.join_path(
self.config.project_out_root,
FileUtils.basename(zip_file_name_real))
FileUtils.copy_file(zip_file_name, target_file_path)
# TODO Revisit all exception handling: ValueError vs. exit() calls
# Methods should throw exceptions, exit should be handled in this method
yarn_dev_tools = YarnDevTools()
# Parse args, commands will be mapped to YarnDevTools functions in ArgParser.parse_args
args, parser = ArgParser.parse_args(yarn_dev_tools)
log_file = Setup.init_logger(
execution_mode=ExecutionMode.PRODUCTION,
console_debug=args.debug,
postfix=args.command,
repos=[
yarn_dev_tools.upstream_repo.repo,
yarn_dev_tools.downstream_repo.repo
],
verbose=args.verbose,
)
cmd_type = CommandType.from_str(args.command)
if cmd_type not in IGNORE_LATEST_SYMLINK_COMMANDS:
FileUtils.create_symlink_path_dir(cmd_type.log_link_name, log_file,
yarn_dev_tools.project_out_root)
else:
LOG.info(
f"Skipping to re-create symlink as command is: {args.command}")
# Call the handler function
args.func(args, parser=parser)
end_time = time.time()
LOG.info("Execution of script took %d seconds", end_time - start_time)