def test_YARN_10496(self): project_out_root = ProjectUtils.get_test_output_basedir(PROJECT_NAME, allow_python_commons_as_project=True) result_basedir = FileUtils.join_path(project_out_root, "jira-data") FileUtils.ensure_dir_created(result_basedir) jira_id = "YARN-10496" jira_html_file = FileUtils.join_path(result_basedir, "jira.html") jira_list_file = FileUtils.join_path(result_basedir, "jira-list.txt") jira_html = JiraUtils.download_jira_html( "https://issues.apache.org/jira/browse/", jira_id, jira_html_file ) jira_ids_and_titles = JiraUtils.parse_subjiras_and_jira_titles_from_umbrella_html( jira_html, jira_list_file, filter_ids=[jira_id] ) expected_jira_ids = ['YARN-10169', 'YARN-10504', 'YARN-10505', 'YARN-10506', 'YARN-10512', 'YARN-10513', 'YARN-10521', 'YARN-10522', 'YARN-10524', 'YARN-10525', 'YARN-10531', 'YARN-10532', 'YARN-10535', 'YARN-10564', 'YARN-10565', 'YARN-10571', 'YARN-10573', 'YARN-10574', 'YARN-10576', 'YARN-10577', 'YARN-10578', 'YARN-10579', 'YARN-10581', 'YARN-10582', 'YARN-10583', 'YARN-10584', 'YARN-10587', 'YARN-10590', 'YARN-10592', 'YARN-10596', 'YARN-10598', 'YARN-10599', 'YARN-10600', 'YARN-10604', 'YARN-10605', 'YARN-10609', 'YARN-10614', 'YARN-10615', 'YARN-10620', 'YARN-10622', 'YARN-10624'] all_list_items_found = all(id1 in jira_ids_and_titles.keys() for id1 in expected_jira_ids) self.assertTrue(all_list_items_found) expected_mappings = {'YARN-10624': 'Support max queues limit configuration in new auto created queue, consistent with old auto created.'} self.assertEqual(expected_mappings['YARN-10624'], jira_ids_and_titles['YARN-10624']) self.assertTrue(isinstance(jira_ids_and_titles['YARN-10624'], str))
def _get_session_dir_under_child_dir(cls, child_dir_name, test: bool = False): child_dir_type: str = "child dir" if not test else "test child dir" dir_dict = cls.CHILD_DIR_DICT if not test else cls.CHILD_DIR_TEST_DICT if not child_dir_name: raise ValueError( f"Project {child_dir_type} name should be specified!") project_name = cls._validate_project_for_child_dir_creation() if project_name in dir_dict and child_dir_name in dir_dict[ project_name]: stored_dir = dir_dict[project_name][child_dir_name] LOG.debug( f"Found already stored {child_dir_type} for project '{project_name}': {stored_dir}" ) session_dir = FileUtils.join_path( stored_dir, f"session-{DateUtils.now_formatted('%Y%m%d_%H%M%S')}") FileUtils.ensure_dir_created(session_dir) return session_dir else: raise ValueError( f"Cannot find stored {child_dir_type} for project. " f"Project: {project_name}, " f"Child dir: {child_dir_name}, " f"All stored {child_dir_type}s: {dir_dict}")
def determine_new_patch_filename(self): patch_dir = FileUtils.join_path(self.patch_basedir, self.patch_branch) FileUtils.ensure_dir_created(patch_dir) found_patches = FileUtils.find_files(patch_dir, regex=self.patch_branch + PATCH_FILE_REGEX, single_level=True) new_patch_filename, new_patch_num = PatchUtils.get_next_filename( patch_dir, found_patches) # Double-check new filename vs. putting it altogether manually new_patch_filename_sanity = FileUtils.join_path( self.patch_basedir, self.patch_branch, f"{self.patch_branch}.{str(new_patch_num)}{PATCH_EXTENSION}") # If this is a new patch, use the appended name, # Otherwise, use the generated filename if new_patch_num == FIRST_PATCH_NUMBER: new_patch_filename = new_patch_filename_sanity if new_patch_filename != new_patch_filename_sanity: raise ValueError( "File paths do not match. " f"Calculated: {new_patch_filename}, Concatenated: {new_patch_filename_sanity}" ) self.new_patch_filename = new_patch_filename
def get_output_basedir(cls, basedir_name: str, ensure_created=True, allow_python_commons_as_project=False, project_root_determination_strategy=None): if not basedir_name: raise ValueError("Basedir name should be specified!") project_name = cls.verify_caller_filename_valid( allow_python_commons_as_project=allow_python_commons_as_project, project_root_determination_strategy= project_root_determination_strategy) proj_basedir = FileUtils.join_path(PROJECTS_BASEDIR, basedir_name) if project_name in cls.PROJECT_BASEDIR_DICT: old_basedir = cls.PROJECT_BASEDIR_DICT[project_name] if old_basedir != proj_basedir: raise ValueError( "Project is already registered with a different output basedir. Details: \n" f"Old basedir name: {old_basedir.split(os.sep)[-1]}\n" f"Project basedir's old full path: {old_basedir}\n" f"New basedir name would be: {basedir_name}\n" f"Project basedir's new full path would be: {proj_basedir}\n" ) cls.PROJECT_BASEDIR_DICT[project_name] = proj_basedir if ensure_created: FileUtils.ensure_dir_created(proj_basedir) return proj_basedir
def test_run_in_a_non_git_repo_working_dir(self): working_dir = FileUtils.join_path("/tmp", "dummydir") FileUtils.ensure_dir_created(working_dir) format_patch_saver = FormatPatchSaver(self.setup_args(), working_dir, self.current_datetime) self.assertRaises(ValueError, format_patch_saver.run)
def remove_test_files_and_recreate_dir(cls, dir_name: str, clazz): project_name = cls._validate_project_for_child_dir_creation() cls.validate_test_child_dir(dir_name, project_name) dir_path = cls.CHILD_DIR_TEST_DICT[project_name][dir_name] LOG.info(f"Removing dir: {dir_path}") FileUtils.remove_files(dir_path, ".*") LOG.info(f"Recreating dir: {dir_path}") new_dir = FileUtils.ensure_dir_created(dir_path) LOG.info("Basedir of %s is: %s", clazz.__name__, new_dir) return new_dir
def _execute_compare_script( config, branches, working_dir) -> Dict[BranchType, Tuple[str, str]]: compare_script = config.legacy_compare_script_path master_br_name = branches.get_branch(BranchType.MASTER).shortname feature_br_name = branches.get_branch(BranchType.FEATURE).shortname output_dir = FileUtils.join_path(config.output_dir, "git_compare_script_output") FileUtils.ensure_dir_created(output_dir) results: Dict[BranchType, Tuple[str, str]] = { BranchType.MASTER: LegacyScriptRunner._exec_script_only_on_master( compare_script, feature_br_name, master_br_name, output_dir, working_dir), BranchType.FEATURE: LegacyScriptRunner._exec_script_only_on_feature( compare_script, feature_br_name, master_br_name, output_dir, working_dir), } return results
def get_test_output_child_dir(cls, dir_name: str, ensure_created=True, special_parent_dir=None): if not dir_name: raise ValueError("Dir name should be specified!") project_name = cls._validate_project_for_child_dir_creation() if project_name in cls.CHILD_DIR_TEST_DICT and dir_name in cls.CHILD_DIR_TEST_DICT[ project_name]: stored_dir = cls.CHILD_DIR_TEST_DICT[project_name][dir_name] LOG.debug( f"Found already stored child test dir for project '{project_name}': {stored_dir}" ) FileUtils.ensure_dir_created(stored_dir) return stored_dir if special_parent_dir: if not FileUtils.does_path_exist(special_parent_dir): raise ValueError( f"Specified parent dir does not exist: {special_parent_dir}" ) LOG.debug( f"Parent dir of new child directory will be: {special_parent_dir}" ) parent_dir = special_parent_dir new_child_dir = FileUtils.join_path(parent_dir, dir_name) else: # Default parent dir: Basedir of project # New child dir: basedir/test/<new child dir name> parent_dir = cls.PROJECT_BASEDIR_DICT[project_name] new_child_dir = FileUtils.join_path(parent_dir, TEST_OUTPUT_DIR_NAME, dir_name) if project_name not in cls.CHILD_DIR_TEST_DICT: cls.CHILD_DIR_TEST_DICT[project_name] = {} cls.CHILD_DIR_TEST_DICT[project_name][dir_name] = new_child_dir if ensure_created: FileUtils.ensure_dir_created(new_child_dir) return new_child_dir
def get_output_child_dir(cls, dir_name: str, ensure_created=True): if not dir_name: raise ValueError("Dir name should be specified!") project_name = cls._validate_project_for_child_dir_creation() if project_name in cls.CHILD_DIR_DICT and dir_name in cls.CHILD_DIR_DICT[ project_name]: stored_dir = cls.CHILD_DIR_DICT[project_name][dir_name] LOG.debug( f"Found already stored child dir for project '{project_name}': {stored_dir}" ) FileUtils.ensure_dir_created(stored_dir) return stored_dir proj_basedir = cls.PROJECT_BASEDIR_DICT[project_name] new_child_dir = FileUtils.join_path(proj_basedir, dir_name) if project_name not in cls.CHILD_DIR_DICT: cls.CHILD_DIR_DICT[project_name] = {} cls.CHILD_DIR_DICT[project_name][dir_name] = new_child_dir if ensure_created: FileUtils.ensure_dir_created(new_child_dir) return new_child_dir
def test_executing_script_from_uncommon_directory(self): # Can't use /tmp as it's platform dependent. # On MacOS, it's mounted as /private/tmp but some Linux systems don't have /private/tmp. # Let's use python's built-in temp dir creation methods. tmp_dir: tempfile.TemporaryDirectory = tempfile.TemporaryDirectory() script_dir = FileUtils.ensure_dir_created( FileUtils.join_path(tmp_dir.name, "python")) sys.path.append(script_dir) script_abs_path = script_dir + os.sep + "hello_world.py" contents = "from pythoncommons.project_utils import ProjectUtils,ProjectRootDeterminationStrategy\n" \ "print(\"hello world\")\n" \ "basedir = ProjectUtils.get_output_basedir('test', project_root_determination_strategy=ProjectRootDeterminationStrategy.SYS_PATH)\n" \ "logfilename = ProjectUtils.get_default_log_file('test')\n" FileUtils.save_to_file(script_abs_path, contents) os.system(f'python3 {script_abs_path}') proc = subprocess.run(["python3", script_abs_path], capture_output=True) # print(f"stdout: {proc.stdout}") # print(f"stderr: {str(proc.stderr)}") print(f"exit code: {proc.returncode}") self.assertEqual(0, proc.returncode)
def create_new_export_dir(self): dt_string = DateUtils.now_formatted("%Y%m%d_%H%M%S") dirname = FileUtils.ensure_dir_created( os.path.join(self.exports_dir, f"{EXPORTED_DIR_NAME_PREFIX}-{dt_string}")) return dirname
def setup_dirs(self): self.project_out_root = ProjectUtils.get_output_basedir(PROJECT_NAME) self.search_basedir = self.options.search_basedir FileUtils.ensure_dir_created(self.search_basedir)