def parse_subjiras_and_jira_titles_from_umbrella_html(
html_doc, to_file, filter_ids, find_all_links=True):
soup = BeautifulSoup(html_doc, "html.parser")
result_dict = {}
links = []
if find_all_links:
links = soup.find_all("a", attrs={"class": "issue-link"})
else:
table = soup.find("table", id="issuetable")
if table is not None:
links = table.find_all("a", attrs={"class": "issue-link"})
for link in links:
jira_id = link.attrs["data-issue-key"]
jira_title = str(link.contents[0])
# There are 2 anchors with class: 'issue-link' per row. Only store the one with valid title.
if len(jira_title.strip()) > 0:
result_dict[jira_id] = jira_title
if filter_ids:
LOG.info("Filtering ids from result list: %s", filter_ids)
result_dict = {
jira_id: title
for jira_id, title in result_dict.items()
if jira_id not in filter_ids
}
FileUtils.save_to_file(
to_file, StringUtils.dict_to_multiline_string(result_dict))
return result_dict
def find_upstream_commits_and_save_to_file(self):
# It's quite complex to grep for multiple jira IDs with gitpython, so let's rather call an external command
git_log_result = self.upstream_repo.log(ORIGIN_TRUNK,
oneline_with_date=True)
cmd, output = CommandRunner.egrep_with_cli(
git_log_result, self.intermediate_results_file,
self.data.piped_jira_ids)
normal_commit_lines = output.split("\n")
modified_log_lines = self._find_missing_upstream_commits_by_message(
git_log_result, normal_commit_lines)
self.data.matched_upstream_commit_list = normal_commit_lines + modified_log_lines
if not self.data.matched_upstream_commit_list:
raise ValueError(
f"Cannot find any commits for jira: {self.config.jira_id}")
LOG.info("Number of matched commits: %s",
self.data.no_of_matched_commits)
LOG.debug(
"Matched commits: \n%s",
StringUtils.list_to_multiline_string(
self.data.matched_upstream_commit_list))
# Commits in reverse order (oldest first)
self.data.matched_upstream_commit_list.reverse()
self.convert_to_commit_data_objects_upstream()
FileUtils.save_to_file(
self.commits_file,
StringUtils.list_to_multiline_string(
self.data.matched_upstream_commit_hashes))
def _write_to_file_or_console(self, contents, output_type, add_sep_to_end=False):
if self.config.console_mode:
LOG.info(f"Printing {output_type}: {contents}")
else:
fn_prefix = self._convert_output_type_str_to_file_prefix(output_type, add_sep_to_end=add_sep_to_end)
f = self._generate_filename(self.config.output_dir, fn_prefix)
LOG.info(f"Saving {output_type} to file: {f}")
FileUtils.save_to_file(f, contents)
def _write_to_file_or_console_branch_data(self, branch: BranchData, contents, output_type):
if self.config.console_mode:
LOG.info(f"Printing {output_type} for branch {branch.type.name}: {contents}")
else:
fn_prefix = self._convert_output_type_str_to_file_prefix(output_type)
f = self._generate_filename(self.config.output_dir, fn_prefix, branch.shortname)
LOG.info(f"Saving {output_type} for branch {branch.type.name} to file: {f}")
FileUtils.save_to_file(f, contents)
def save_diff_to_patch_file(diff, file):
if not diff or diff == "":
LOG.error("Diff was empty. Patch file is not created!")
return False
else:
diff += os.linesep
LOG.info("Saving diff to patch file: %s", file)
LOG.debug("Diff: %s", diff)
FileUtils.save_to_file(file, diff)
return True
def print_and_save_summary(self, rendered_summary):
LOG.info(rendered_summary.printable_summary_str)
filename = FileUtils.join_path(self.config.output_dir, SUMMARY_FILE_TXT)
LOG.info(f"Saving summary to text file: {filename}")
FileUtils.save_to_file(filename, rendered_summary.writable_summary_str)
filename = FileUtils.join_path(self.config.output_dir, SUMMARY_FILE_HTML)
LOG.info(f"Saving summary to html file: {filename}")
FileUtils.save_to_file(filename, rendered_summary.html_summary)
def test_with_bad_patch_content(self):
patch_file = FileUtils.join_path(self.dummy_patches_dir,
PATCH_FILENAME)
FileUtils.save_to_file(patch_file, "dummycontents")
args = Object()
args.patch_file = patch_file
review_branch_creator = ReviewBranchCreator(args, self.repo_wrapper,
BASE_BRANCH,
REMOTE_BASE_BRANCH)
self.assertRaises(ValueError, review_branch_creator.run)
def save_to_test_file(cls, dir_name: str, filename: str,
file_contents: str):
if not dir_name:
raise ValueError("Dir name should be specified!")
if not filename:
raise ValueError("Filename should be specified!")
project_name = cls._validate_project_for_child_dir_creation()
cls.validate_test_child_dir(dir_name, project_name)
dir_path = cls.CHILD_DIR_TEST_DICT[project_name][dir_name]
FileUtils.save_to_file(FileUtils.join_path(dir_path, filename),
file_contents)
def parse_subjiras_from_umbrella_html(html_doc, to_file, filter_ids):
soup = BeautifulSoup(html_doc, "html.parser")
issue_keys = []
for link in soup.find_all("a", attrs={"class": "issue-link"}):
issue_keys.append(link.attrs["data-issue-key"])
if filter_ids:
LOG.info("Filtering ids from result list: %s", filter_ids)
issue_keys = [
issue for issue in issue_keys if issue not in filter_ids
]
# Filter dupes
issue_keys = list(set(issue_keys))
FileUtils.save_to_file(
to_file, StringUtils.list_to_multiline_string(issue_keys))
return issue_keys
def save_changed_files_to_file(self):
list_of_changed_files = []
for c_hash in self.data.matched_upstream_commit_hashes:
changed_files = self.upstream_repo.diff_tree(c_hash,
no_commit_id=True,
name_only=True,
recursive=True)
list_of_changed_files.append(changed_files)
LOG.debug("List of changed files for commit hash '%s': %s", c_hash,
changed_files)
# Filter dupes, flatten list of lists
list_of_changed_files = [y for x in list_of_changed_files for y in x]
self.data.list_of_changed_files = list(set(list_of_changed_files))
LOG.info("Got %d unique changed files",
len(self.data.list_of_changed_files))
FileUtils.save_to_file(
self.changed_files_file,
StringUtils.list_to_multiline_string(
self.data.list_of_changed_files))
def egrep_with_cli(git_log_result: List[str],
file: str,
grep_for: str,
escape_single_quotes=True,
escape_double_quotes=True,
fail_on_empty_output=True,
fail_on_error=True):
FileUtils.save_to_file(
file, StringUtils.list_to_multiline_string(git_log_result))
if escape_single_quotes or escape_double_quotes:
grep_for = StringUtils.escape_str(
grep_for,
escape_single_quotes=escape_single_quotes,
escape_double_quotes=escape_double_quotes)
cli_command = f"cat {file} | egrep \"{grep_for}\""
return CommandRunner.run_cli_command(
cli_command,
fail_on_empty_output=fail_on_empty_output,
fail_on_error=fail_on_error)
def _write_to_configured_destinations(
self,
data: str,
data_type: TableDataType,
add_sep_to_end=False,
):
"""
Destinations: Console, File or both
:param data:
:param add_sep_to_end:
:return:
"""
if self.console_mode:
LOG.info(f"Printing {data_type.key}: {data}")
else:
fn_prefix = self._convert_output_type_str_to_file_prefix(
data_type.key, add_sep_to_end=add_sep_to_end)
f = self._generate_filename(self.output_dir, fn_prefix)
LOG.info(f"Saving {data_type.key} to file: {f}")
FileUtils.save_to_file(f, data)
def test_executing_script_from_uncommon_directory(self):
# Can't use /tmp as it's platform dependent.
# On MacOS, it's mounted as /private/tmp but some Linux systems don't have /private/tmp.
# Let's use python's built-in temp dir creation methods.
tmp_dir: tempfile.TemporaryDirectory = tempfile.TemporaryDirectory()
script_dir = FileUtils.ensure_dir_created(
FileUtils.join_path(tmp_dir.name, "python"))
sys.path.append(script_dir)
script_abs_path = script_dir + os.sep + "hello_world.py"
contents = "from pythoncommons.project_utils import ProjectUtils,ProjectRootDeterminationStrategy\n" \
"print(\"hello world\")\n" \
"basedir = ProjectUtils.get_output_basedir('test', project_root_determination_strategy=ProjectRootDeterminationStrategy.SYS_PATH)\n" \
"logfilename = ProjectUtils.get_default_log_file('test')\n"
FileUtils.save_to_file(script_abs_path, contents)
os.system(f'python3 {script_abs_path}')
proc = subprocess.run(["python3", script_abs_path],
capture_output=True)
# print(f"stdout: {proc.stdout}")
# print(f"stderr: {str(proc.stderr)}")
print(f"exit code: {proc.returncode}")
self.assertEqual(0, proc.returncode)
def write_all_changes_files(self):
"""
Iterate over changed files, print all matching changes to the particular file
Create changes file for each touched file
:return:
"""
LOG.info("Recording changes of individual files...")
for idx, changed_file in enumerate(self.data.list_of_changed_files):
target_file = FileUtils.join_path(
self.config.umbrella_result_basedir, "changes",
os.path.basename(changed_file))
FileUtils.ensure_file_exists(target_file, create=True)
# NOTE: It seems impossible to call the following command with gitpython:
# git log --follow --oneline -- <file>
# Use a simple CLI command instead
# TODO check if change file exists - It can happen that it was deleted
cli_command = (
f"cd {self.upstream_repo.repo_path} && "
f"git log {ORIGIN_TRUNK} --follow --oneline -- {changed_file} | "
f'egrep "{self.data.piped_jira_ids}"')
LOG.info("[%d / %d] CLI command: %s", idx + 1,
len(self.data.list_of_changed_files), cli_command)
cmd, output = CommandRunner.run_cli_command(
cli_command,
fail_on_empty_output=False,
print_command=False,
fail_on_error=False)
if output:
LOG.info("Saving changes result to file: %s", target_file)
FileUtils.save_to_file(target_file, output)
else:
LOG.error(
f"Failed to detect changes of file: {changed_file}. CLI command was: {cli_command}. "
f"This seems to be a programming error. Exiting...")
FileUtils.save_to_file(target_file, "")
sys.exit(1)
def add_some_file_changes(self, commit=False, commit_message_prefix=None):
FileUtils.save_to_file(
FileUtils.join_path(self.sandbox_repo_path, DUMMYFILE_1),
DUMMYFILE_1)
FileUtils.save_to_file(
FileUtils.join_path(self.sandbox_repo_path, DUMMYFILE_2),
DUMMYFILE_2)
yarn_config_java = FileUtils.join_path(self.sandbox_repo_path,
YARNCONFIGURATION_PATH)
FileUtils.append_to_file(yarn_config_java, "dummy_changes_to_conf_1\n")
FileUtils.append_to_file(yarn_config_java, "dummy_changes_to_conf_2\n")
if commit:
commit_msg = "test_commit"
if commit_message_prefix:
commit_msg = commit_message_prefix + commit_msg
self.repo_wrapper.commit(
commit_msg,
author=Actor("A test author", "*****@*****.**"),
committer=Actor("A test committer", "*****@*****.**"),
add_files_to_index=[
DUMMYFILE_1, DUMMYFILE_2, yarn_config_java
],
)
def write_summary_file(self):
FileUtils.save_to_file(
self.summary_file,
self.data.render_summary_string(
self.config.umbrella_result_basedir))
def download_jira_html(jira_server, jira_id, to_file):
resp = requests.get(jira_server + jira_id)
resp.raise_for_status()
FileUtils.save_to_file(to_file, resp.text)
return resp.text
def process_regular_summary(self, rendered_summary: str, filename: str):
LOG.info(rendered_summary)
filename = FileUtils.join_path(self.output_dir, filename)
LOG.info(f"Saving summary to text file: {filename}")
FileUtils.save_to_file(filename, rendered_summary)
def process_html_summary(self, rendered_summary: str, filename: str):
# Doesn't make sense to print HTML summary to console
filename = FileUtils.join_path(self.output_dir, filename)
LOG.info(f"Saving summary to html file: {filename}")
FileUtils.save_to_file(filename, rendered_summary)