def parse_subjiras_and_jira_titles_from_umbrella_html( html_doc, to_file, filter_ids, find_all_links=True): soup = BeautifulSoup(html_doc, "html.parser") result_dict = {} links = [] if find_all_links: links = soup.find_all("a", attrs={"class": "issue-link"}) else: table = soup.find("table", id="issuetable") if table is not None: links = table.find_all("a", attrs={"class": "issue-link"}) for link in links: jira_id = link.attrs["data-issue-key"] jira_title = str(link.contents[0]) # There are 2 anchors with class: 'issue-link' per row. Only store the one with valid title. if len(jira_title.strip()) > 0: result_dict[jira_id] = jira_title if filter_ids: LOG.info("Filtering ids from result list: %s", filter_ids) result_dict = { jira_id: title for jira_id, title in result_dict.items() if jira_id not in filter_ids } FileUtils.save_to_file( to_file, StringUtils.dict_to_multiline_string(result_dict)) return result_dict
def find_upstream_commits_and_save_to_file(self): # It's quite complex to grep for multiple jira IDs with gitpython, so let's rather call an external command git_log_result = self.upstream_repo.log(ORIGIN_TRUNK, oneline_with_date=True) cmd, output = CommandRunner.egrep_with_cli( git_log_result, self.intermediate_results_file, self.data.piped_jira_ids) normal_commit_lines = output.split("\n") modified_log_lines = self._find_missing_upstream_commits_by_message( git_log_result, normal_commit_lines) self.data.matched_upstream_commit_list = normal_commit_lines + modified_log_lines if not self.data.matched_upstream_commit_list: raise ValueError( f"Cannot find any commits for jira: {self.config.jira_id}") LOG.info("Number of matched commits: %s", self.data.no_of_matched_commits) LOG.debug( "Matched commits: \n%s", StringUtils.list_to_multiline_string( self.data.matched_upstream_commit_list)) # Commits in reverse order (oldest first) self.data.matched_upstream_commit_list.reverse() self.convert_to_commit_data_objects_upstream() FileUtils.save_to_file( self.commits_file, StringUtils.list_to_multiline_string( self.data.matched_upstream_commit_hashes))
def _write_to_file_or_console(self, contents, output_type, add_sep_to_end=False): if self.config.console_mode: LOG.info(f"Printing {output_type}: {contents}") else: fn_prefix = self._convert_output_type_str_to_file_prefix(output_type, add_sep_to_end=add_sep_to_end) f = self._generate_filename(self.config.output_dir, fn_prefix) LOG.info(f"Saving {output_type} to file: {f}") FileUtils.save_to_file(f, contents)
def _write_to_file_or_console_branch_data(self, branch: BranchData, contents, output_type): if self.config.console_mode: LOG.info(f"Printing {output_type} for branch {branch.type.name}: {contents}") else: fn_prefix = self._convert_output_type_str_to_file_prefix(output_type) f = self._generate_filename(self.config.output_dir, fn_prefix, branch.shortname) LOG.info(f"Saving {output_type} for branch {branch.type.name} to file: {f}") FileUtils.save_to_file(f, contents)
def save_diff_to_patch_file(diff, file): if not diff or diff == "": LOG.error("Diff was empty. Patch file is not created!") return False else: diff += os.linesep LOG.info("Saving diff to patch file: %s", file) LOG.debug("Diff: %s", diff) FileUtils.save_to_file(file, diff) return True
def print_and_save_summary(self, rendered_summary): LOG.info(rendered_summary.printable_summary_str) filename = FileUtils.join_path(self.config.output_dir, SUMMARY_FILE_TXT) LOG.info(f"Saving summary to text file: {filename}") FileUtils.save_to_file(filename, rendered_summary.writable_summary_str) filename = FileUtils.join_path(self.config.output_dir, SUMMARY_FILE_HTML) LOG.info(f"Saving summary to html file: {filename}") FileUtils.save_to_file(filename, rendered_summary.html_summary)
def test_with_bad_patch_content(self): patch_file = FileUtils.join_path(self.dummy_patches_dir, PATCH_FILENAME) FileUtils.save_to_file(patch_file, "dummycontents") args = Object() args.patch_file = patch_file review_branch_creator = ReviewBranchCreator(args, self.repo_wrapper, BASE_BRANCH, REMOTE_BASE_BRANCH) self.assertRaises(ValueError, review_branch_creator.run)
def save_to_test_file(cls, dir_name: str, filename: str, file_contents: str): if not dir_name: raise ValueError("Dir name should be specified!") if not filename: raise ValueError("Filename should be specified!") project_name = cls._validate_project_for_child_dir_creation() cls.validate_test_child_dir(dir_name, project_name) dir_path = cls.CHILD_DIR_TEST_DICT[project_name][dir_name] FileUtils.save_to_file(FileUtils.join_path(dir_path, filename), file_contents)
def parse_subjiras_from_umbrella_html(html_doc, to_file, filter_ids): soup = BeautifulSoup(html_doc, "html.parser") issue_keys = [] for link in soup.find_all("a", attrs={"class": "issue-link"}): issue_keys.append(link.attrs["data-issue-key"]) if filter_ids: LOG.info("Filtering ids from result list: %s", filter_ids) issue_keys = [ issue for issue in issue_keys if issue not in filter_ids ] # Filter dupes issue_keys = list(set(issue_keys)) FileUtils.save_to_file( to_file, StringUtils.list_to_multiline_string(issue_keys)) return issue_keys
def save_changed_files_to_file(self): list_of_changed_files = [] for c_hash in self.data.matched_upstream_commit_hashes: changed_files = self.upstream_repo.diff_tree(c_hash, no_commit_id=True, name_only=True, recursive=True) list_of_changed_files.append(changed_files) LOG.debug("List of changed files for commit hash '%s': %s", c_hash, changed_files) # Filter dupes, flatten list of lists list_of_changed_files = [y for x in list_of_changed_files for y in x] self.data.list_of_changed_files = list(set(list_of_changed_files)) LOG.info("Got %d unique changed files", len(self.data.list_of_changed_files)) FileUtils.save_to_file( self.changed_files_file, StringUtils.list_to_multiline_string( self.data.list_of_changed_files))
def egrep_with_cli(git_log_result: List[str], file: str, grep_for: str, escape_single_quotes=True, escape_double_quotes=True, fail_on_empty_output=True, fail_on_error=True): FileUtils.save_to_file( file, StringUtils.list_to_multiline_string(git_log_result)) if escape_single_quotes or escape_double_quotes: grep_for = StringUtils.escape_str( grep_for, escape_single_quotes=escape_single_quotes, escape_double_quotes=escape_double_quotes) cli_command = f"cat {file} | egrep \"{grep_for}\"" return CommandRunner.run_cli_command( cli_command, fail_on_empty_output=fail_on_empty_output, fail_on_error=fail_on_error)
def _write_to_configured_destinations( self, data: str, data_type: TableDataType, add_sep_to_end=False, ): """ Destinations: Console, File or both :param data: :param add_sep_to_end: :return: """ if self.console_mode: LOG.info(f"Printing {data_type.key}: {data}") else: fn_prefix = self._convert_output_type_str_to_file_prefix( data_type.key, add_sep_to_end=add_sep_to_end) f = self._generate_filename(self.output_dir, fn_prefix) LOG.info(f"Saving {data_type.key} to file: {f}") FileUtils.save_to_file(f, data)
def test_executing_script_from_uncommon_directory(self): # Can't use /tmp as it's platform dependent. # On MacOS, it's mounted as /private/tmp but some Linux systems don't have /private/tmp. # Let's use python's built-in temp dir creation methods. tmp_dir: tempfile.TemporaryDirectory = tempfile.TemporaryDirectory() script_dir = FileUtils.ensure_dir_created( FileUtils.join_path(tmp_dir.name, "python")) sys.path.append(script_dir) script_abs_path = script_dir + os.sep + "hello_world.py" contents = "from pythoncommons.project_utils import ProjectUtils,ProjectRootDeterminationStrategy\n" \ "print(\"hello world\")\n" \ "basedir = ProjectUtils.get_output_basedir('test', project_root_determination_strategy=ProjectRootDeterminationStrategy.SYS_PATH)\n" \ "logfilename = ProjectUtils.get_default_log_file('test')\n" FileUtils.save_to_file(script_abs_path, contents) os.system(f'python3 {script_abs_path}') proc = subprocess.run(["python3", script_abs_path], capture_output=True) # print(f"stdout: {proc.stdout}") # print(f"stderr: {str(proc.stderr)}") print(f"exit code: {proc.returncode}") self.assertEqual(0, proc.returncode)
def write_all_changes_files(self): """ Iterate over changed files, print all matching changes to the particular file Create changes file for each touched file :return: """ LOG.info("Recording changes of individual files...") for idx, changed_file in enumerate(self.data.list_of_changed_files): target_file = FileUtils.join_path( self.config.umbrella_result_basedir, "changes", os.path.basename(changed_file)) FileUtils.ensure_file_exists(target_file, create=True) # NOTE: It seems impossible to call the following command with gitpython: # git log --follow --oneline -- <file> # Use a simple CLI command instead # TODO check if change file exists - It can happen that it was deleted cli_command = ( f"cd {self.upstream_repo.repo_path} && " f"git log {ORIGIN_TRUNK} --follow --oneline -- {changed_file} | " f'egrep "{self.data.piped_jira_ids}"') LOG.info("[%d / %d] CLI command: %s", idx + 1, len(self.data.list_of_changed_files), cli_command) cmd, output = CommandRunner.run_cli_command( cli_command, fail_on_empty_output=False, print_command=False, fail_on_error=False) if output: LOG.info("Saving changes result to file: %s", target_file) FileUtils.save_to_file(target_file, output) else: LOG.error( f"Failed to detect changes of file: {changed_file}. CLI command was: {cli_command}. " f"This seems to be a programming error. Exiting...") FileUtils.save_to_file(target_file, "") sys.exit(1)
def add_some_file_changes(self, commit=False, commit_message_prefix=None): FileUtils.save_to_file( FileUtils.join_path(self.sandbox_repo_path, DUMMYFILE_1), DUMMYFILE_1) FileUtils.save_to_file( FileUtils.join_path(self.sandbox_repo_path, DUMMYFILE_2), DUMMYFILE_2) yarn_config_java = FileUtils.join_path(self.sandbox_repo_path, YARNCONFIGURATION_PATH) FileUtils.append_to_file(yarn_config_java, "dummy_changes_to_conf_1\n") FileUtils.append_to_file(yarn_config_java, "dummy_changes_to_conf_2\n") if commit: commit_msg = "test_commit" if commit_message_prefix: commit_msg = commit_message_prefix + commit_msg self.repo_wrapper.commit( commit_msg, author=Actor("A test author", "*****@*****.**"), committer=Actor("A test committer", "*****@*****.**"), add_files_to_index=[ DUMMYFILE_1, DUMMYFILE_2, yarn_config_java ], )
def write_summary_file(self): FileUtils.save_to_file( self.summary_file, self.data.render_summary_string( self.config.umbrella_result_basedir))
def download_jira_html(jira_server, jira_id, to_file): resp = requests.get(jira_server + jira_id) resp.raise_for_status() FileUtils.save_to_file(to_file, resp.text) return resp.text
def process_regular_summary(self, rendered_summary: str, filename: str): LOG.info(rendered_summary) filename = FileUtils.join_path(self.output_dir, filename) LOG.info(f"Saving summary to text file: {filename}") FileUtils.save_to_file(filename, rendered_summary)
def process_html_summary(self, rendered_summary: str, filename: str): # Doesn't make sense to print HTML summary to console filename = FileUtils.join_path(self.output_dir, filename) LOG.info(f"Saving summary to html file: {filename}") FileUtils.save_to_file(filename, rendered_summary)