def test_YARN_10496(self):
project_out_root = ProjectUtils.get_test_output_basedir(PROJECT_NAME, allow_python_commons_as_project=True)
result_basedir = FileUtils.join_path(project_out_root, "jira-data")
FileUtils.ensure_dir_created(result_basedir)
jira_id = "YARN-10496"
jira_html_file = FileUtils.join_path(result_basedir, "jira.html")
jira_list_file = FileUtils.join_path(result_basedir, "jira-list.txt")
jira_html = JiraUtils.download_jira_html(
"https://issues.apache.org/jira/browse/", jira_id, jira_html_file
)
jira_ids_and_titles = JiraUtils.parse_subjiras_and_jira_titles_from_umbrella_html(
jira_html, jira_list_file, filter_ids=[jira_id]
)
expected_jira_ids = ['YARN-10169', 'YARN-10504', 'YARN-10505', 'YARN-10506', 'YARN-10512', 'YARN-10513',
'YARN-10521', 'YARN-10522', 'YARN-10524', 'YARN-10525', 'YARN-10531', 'YARN-10532',
'YARN-10535', 'YARN-10564', 'YARN-10565', 'YARN-10571', 'YARN-10573', 'YARN-10574',
'YARN-10576', 'YARN-10577', 'YARN-10578', 'YARN-10579', 'YARN-10581', 'YARN-10582',
'YARN-10583', 'YARN-10584', 'YARN-10587', 'YARN-10590', 'YARN-10592', 'YARN-10596',
'YARN-10598', 'YARN-10599', 'YARN-10600', 'YARN-10604', 'YARN-10605', 'YARN-10609',
'YARN-10614', 'YARN-10615', 'YARN-10620', 'YARN-10622', 'YARN-10624']
all_list_items_found = all(id1 in jira_ids_and_titles.keys() for id1 in expected_jira_ids)
self.assertTrue(all_list_items_found)
expected_mappings = {'YARN-10624': 'Support max queues limit configuration in new auto created queue, consistent with old auto created.'}
self.assertEqual(expected_mappings['YARN-10624'], jira_ids_and_titles['YARN-10624'])
self.assertTrue(isinstance(jira_ids_and_titles['YARN-10624'], str))
def _get_session_dir_under_child_dir(cls,
child_dir_name,
test: bool = False):
child_dir_type: str = "child dir" if not test else "test child dir"
dir_dict = cls.CHILD_DIR_DICT if not test else cls.CHILD_DIR_TEST_DICT
if not child_dir_name:
raise ValueError(
f"Project {child_dir_type} name should be specified!")
project_name = cls._validate_project_for_child_dir_creation()
if project_name in dir_dict and child_dir_name in dir_dict[
project_name]:
stored_dir = dir_dict[project_name][child_dir_name]
LOG.debug(
f"Found already stored {child_dir_type} for project '{project_name}': {stored_dir}"
)
session_dir = FileUtils.join_path(
stored_dir,
f"session-{DateUtils.now_formatted('%Y%m%d_%H%M%S')}")
FileUtils.ensure_dir_created(session_dir)
return session_dir
else:
raise ValueError(
f"Cannot find stored {child_dir_type} for project. "
f"Project: {project_name}, "
f"Child dir: {child_dir_name}, "
f"All stored {child_dir_type}s: {dir_dict}")
def determine_new_patch_filename(self):
patch_dir = FileUtils.join_path(self.patch_basedir, self.patch_branch)
FileUtils.ensure_dir_created(patch_dir)
found_patches = FileUtils.find_files(patch_dir,
regex=self.patch_branch +
PATCH_FILE_REGEX,
single_level=True)
new_patch_filename, new_patch_num = PatchUtils.get_next_filename(
patch_dir, found_patches)
# Double-check new filename vs. putting it altogether manually
new_patch_filename_sanity = FileUtils.join_path(
self.patch_basedir, self.patch_branch,
f"{self.patch_branch}.{str(new_patch_num)}{PATCH_EXTENSION}")
# If this is a new patch, use the appended name,
# Otherwise, use the generated filename
if new_patch_num == FIRST_PATCH_NUMBER:
new_patch_filename = new_patch_filename_sanity
if new_patch_filename != new_patch_filename_sanity:
raise ValueError(
"File paths do not match. "
f"Calculated: {new_patch_filename}, Concatenated: {new_patch_filename_sanity}"
)
self.new_patch_filename = new_patch_filename
def get_output_basedir(cls,
basedir_name: str,
ensure_created=True,
allow_python_commons_as_project=False,
project_root_determination_strategy=None):
if not basedir_name:
raise ValueError("Basedir name should be specified!")
project_name = cls.verify_caller_filename_valid(
allow_python_commons_as_project=allow_python_commons_as_project,
project_root_determination_strategy=
project_root_determination_strategy)
proj_basedir = FileUtils.join_path(PROJECTS_BASEDIR, basedir_name)
if project_name in cls.PROJECT_BASEDIR_DICT:
old_basedir = cls.PROJECT_BASEDIR_DICT[project_name]
if old_basedir != proj_basedir:
raise ValueError(
"Project is already registered with a different output basedir. Details: \n"
f"Old basedir name: {old_basedir.split(os.sep)[-1]}\n"
f"Project basedir's old full path: {old_basedir}\n"
f"New basedir name would be: {basedir_name}\n"
f"Project basedir's new full path would be: {proj_basedir}\n"
)
cls.PROJECT_BASEDIR_DICT[project_name] = proj_basedir
if ensure_created:
FileUtils.ensure_dir_created(proj_basedir)
return proj_basedir
def test_run_in_a_non_git_repo_working_dir(self):
working_dir = FileUtils.join_path("/tmp", "dummydir")
FileUtils.ensure_dir_created(working_dir)
format_patch_saver = FormatPatchSaver(self.setup_args(), working_dir,
self.current_datetime)
self.assertRaises(ValueError, format_patch_saver.run)
def remove_test_files_and_recreate_dir(cls, dir_name: str, clazz):
project_name = cls._validate_project_for_child_dir_creation()
cls.validate_test_child_dir(dir_name, project_name)
dir_path = cls.CHILD_DIR_TEST_DICT[project_name][dir_name]
LOG.info(f"Removing dir: {dir_path}")
FileUtils.remove_files(dir_path, ".*")
LOG.info(f"Recreating dir: {dir_path}")
new_dir = FileUtils.ensure_dir_created(dir_path)
LOG.info("Basedir of %s is: %s", clazz.__name__, new_dir)
return new_dir
def _execute_compare_script(
config, branches,
working_dir) -> Dict[BranchType, Tuple[str, str]]:
compare_script = config.legacy_compare_script_path
master_br_name = branches.get_branch(BranchType.MASTER).shortname
feature_br_name = branches.get_branch(BranchType.FEATURE).shortname
output_dir = FileUtils.join_path(config.output_dir,
"git_compare_script_output")
FileUtils.ensure_dir_created(output_dir)
results: Dict[BranchType, Tuple[str, str]] = {
BranchType.MASTER:
LegacyScriptRunner._exec_script_only_on_master(
compare_script, feature_br_name, master_br_name, output_dir,
working_dir),
BranchType.FEATURE:
LegacyScriptRunner._exec_script_only_on_feature(
compare_script, feature_br_name, master_br_name, output_dir,
working_dir),
}
return results
def get_test_output_child_dir(cls,
dir_name: str,
ensure_created=True,
special_parent_dir=None):
if not dir_name:
raise ValueError("Dir name should be specified!")
project_name = cls._validate_project_for_child_dir_creation()
if project_name in cls.CHILD_DIR_TEST_DICT and dir_name in cls.CHILD_DIR_TEST_DICT[
project_name]:
stored_dir = cls.CHILD_DIR_TEST_DICT[project_name][dir_name]
LOG.debug(
f"Found already stored child test dir for project '{project_name}': {stored_dir}"
)
FileUtils.ensure_dir_created(stored_dir)
return stored_dir
if special_parent_dir:
if not FileUtils.does_path_exist(special_parent_dir):
raise ValueError(
f"Specified parent dir does not exist: {special_parent_dir}"
)
LOG.debug(
f"Parent dir of new child directory will be: {special_parent_dir}"
)
parent_dir = special_parent_dir
new_child_dir = FileUtils.join_path(parent_dir, dir_name)
else:
# Default parent dir: Basedir of project
# New child dir: basedir/test/<new child dir name>
parent_dir = cls.PROJECT_BASEDIR_DICT[project_name]
new_child_dir = FileUtils.join_path(parent_dir,
TEST_OUTPUT_DIR_NAME, dir_name)
if project_name not in cls.CHILD_DIR_TEST_DICT:
cls.CHILD_DIR_TEST_DICT[project_name] = {}
cls.CHILD_DIR_TEST_DICT[project_name][dir_name] = new_child_dir
if ensure_created:
FileUtils.ensure_dir_created(new_child_dir)
return new_child_dir
def get_output_child_dir(cls, dir_name: str, ensure_created=True):
if not dir_name:
raise ValueError("Dir name should be specified!")
project_name = cls._validate_project_for_child_dir_creation()
if project_name in cls.CHILD_DIR_DICT and dir_name in cls.CHILD_DIR_DICT[
project_name]:
stored_dir = cls.CHILD_DIR_DICT[project_name][dir_name]
LOG.debug(
f"Found already stored child dir for project '{project_name}': {stored_dir}"
)
FileUtils.ensure_dir_created(stored_dir)
return stored_dir
proj_basedir = cls.PROJECT_BASEDIR_DICT[project_name]
new_child_dir = FileUtils.join_path(proj_basedir, dir_name)
if project_name not in cls.CHILD_DIR_DICT:
cls.CHILD_DIR_DICT[project_name] = {}
cls.CHILD_DIR_DICT[project_name][dir_name] = new_child_dir
if ensure_created:
FileUtils.ensure_dir_created(new_child_dir)
return new_child_dir
def test_executing_script_from_uncommon_directory(self):
# Can't use /tmp as it's platform dependent.
# On MacOS, it's mounted as /private/tmp but some Linux systems don't have /private/tmp.
# Let's use python's built-in temp dir creation methods.
tmp_dir: tempfile.TemporaryDirectory = tempfile.TemporaryDirectory()
script_dir = FileUtils.ensure_dir_created(
FileUtils.join_path(tmp_dir.name, "python"))
sys.path.append(script_dir)
script_abs_path = script_dir + os.sep + "hello_world.py"
contents = "from pythoncommons.project_utils import ProjectUtils,ProjectRootDeterminationStrategy\n" \
"print(\"hello world\")\n" \
"basedir = ProjectUtils.get_output_basedir('test', project_root_determination_strategy=ProjectRootDeterminationStrategy.SYS_PATH)\n" \
"logfilename = ProjectUtils.get_default_log_file('test')\n"
FileUtils.save_to_file(script_abs_path, contents)
os.system(f'python3 {script_abs_path}')
proc = subprocess.run(["python3", script_abs_path],
capture_output=True)
# print(f"stdout: {proc.stdout}")
# print(f"stderr: {str(proc.stderr)}")
print(f"exit code: {proc.returncode}")
self.assertEqual(0, proc.returncode)
def create_new_export_dir(self):
dt_string = DateUtils.now_formatted("%Y%m%d_%H%M%S")
dirname = FileUtils.ensure_dir_created(
os.path.join(self.exports_dir,
f"{EXPORTED_DIR_NAME_PREFIX}-{dt_string}"))
return dirname
def setup_dirs(self):
self.project_out_root = ProjectUtils.get_output_basedir(PROJECT_NAME)
self.search_basedir = self.options.search_basedir
FileUtils.ensure_dir_created(self.search_basedir)
