def run_comparator_and_send_mail(self,
repo_type: RepoType,
algorithm="simple",
run_legacy_script=True):
date_str = self.current_date_formatted()
feature_branch = OsUtils.get_env_value(
BranchComparatorEnvVar.FEATURE_BRANCH.value,
"origin/CDH-7.1-maint")
master_branch = OsUtils.get_env_value(
BranchComparatorEnvVar.MASTER_BRANCH.value, "origin/cdpd-master")
authors_to_filter = "*****@*****.**"
self._run_comparator(
repo_type,
master_branch,
feature_branch,
authors_to_filter,
debug=True,
algorithm=algorithm,
run_legacy_script=run_legacy_script,
)
self.run_zipper(CommandType.BRANCH_COMPARATOR, debug=True)
sender = "YARN branch diff reporter"
subject = f"YARN branch diff report [{algorithm} algorithm, start date: {date_str}]"
attachment_fnname: str = f"command_data_{algorithm}_{date_str}.zip"
self.send_latest_command_data_in_email(
sender=sender,
subject=subject,
attachment_filename=attachment_fnname)
def start(self, basedir):
LOG.info("Starting CDSW runner...")
repo_type_env = OsUtils.get_env_value(
BranchComparatorEnvVar.REPO_TYPE.value, RepoType.DOWNSTREAM.value)
repo_type: RepoType = RepoType[repo_type_env.upper()]
if repo_type == RepoType.DOWNSTREAM:
self.run_clone_downstream_repos_script(basedir)
elif repo_type == RepoType.UPSTREAM:
# If we are in upstream mode, make sure downstream dir exist
# Currently, yarndevtools requires both repos to be present when initializing.
# BranchComparator is happy with one single repository, upstream or downstream, exclusively.
# Git init the other repository so everything will be alright
FileUtils.create_new_dir(HADOOP_CLOUDERA_BASEDIR)
FileUtils.change_cwd(HADOOP_CLOUDERA_BASEDIR)
os.system("git init")
self.run_clone_upstream_repos_script(basedir)
# TODO investigate why legacy script fails!
self.run_comparator_and_send_mail(repo_type,
algorithm="simple",
run_legacy_script=False)
self.run_comparator_and_send_mail(repo_type,
algorithm="grouped",
run_legacy_script=False)
def run(self):
self.config.full_cmd = OsUtils.determine_full_command()
LOG.info(f"Starting Branch comparator... \n{str(self.config)}")
self.validate_branches()
# TODO Make fetching optional, argparse argument
# self.repo.fetch(all=True)
matching_result = self.compare()
if self.config.run_legacy_script:
LegacyScriptRunner.start(self.config, self.branches,
self.repo.repo_path, matching_result)
def ensure_required_env_vars_are_present(self):
upstream_hadoop_dir = OsUtils.get_env_value(ENV_HADOOP_DEV_DIR, None)
downstream_hadoop_dir = OsUtils.get_env_value(ENV_CLOUDERA_HADOOP_ROOT,
None)
if not upstream_hadoop_dir:
raise ValueError(
f"Upstream Hadoop dir (env var: {ENV_HADOOP_DEV_DIR}) is not set!"
)
if not downstream_hadoop_dir:
raise ValueError(
f"Downstream Hadoop dir (env var: {ENV_CLOUDERA_HADOOP_ROOT}) is not set!"
)
# Verify if dirs are created
FileUtils.verify_if_dir_is_created(downstream_hadoop_dir)
FileUtils.verify_if_dir_is_created(upstream_hadoop_dir)
self.env = {
LOADED_ENV_DOWNSTREAM_DIR: downstream_hadoop_dir,
LOADED_ENV_UPSTREAM_DIR: upstream_hadoop_dir
}
def __init__(self, output_dir: str, args):
self.request_limit = args.req_limit if hasattr(args, "req_limit") and args.req_limit else 1
self.full_email_conf: FullEmailConfig = FullEmailConfig(args)
self.jenkins_url = args.jenkins_url
self.job_name = args.job_name
self.num_prev_days = args.num_prev_days
tc_filters_raw = args.tc_filters if hasattr(args, "tc_filters") and args.tc_filters else []
self.tc_filters: List[TestcaseFilter] = [TestcaseFilter(*tcf.split(":")) for tcf in tc_filters_raw]
if not self.tc_filters:
LOG.warning("TESTCASE FILTER IS NOT SET!")
self.send_mail: bool = not args.skip_mail
self.enable_file_cache: bool = not args.disable_file_cache
self.output_dir = ProjectUtils.get_session_dir_under_child_dir(FileUtils.basename(output_dir))
self.full_cmd: str = OsUtils.determine_full_command_filtered(filter_password=True)
def cdsw_runner_env_dict(cls):
env_dict = {
e.value: OsUtils.get_env_value(e.value, None)
for e in [
CdswEnvVar.MAIL_ACC_USER,
CdswEnvVar.MAIL_ACC_PASSWORD,
BranchComparatorEnvVar.REPO_TYPE,
BranchComparatorEnvVar.MASTER_BRANCH,
BranchComparatorEnvVar.FEATURE_BRANCH,
EnvVar.IGNORE_SMTP_AUTH_ERROR,
]
}
# Manually fix PYTHONPATH like CDSW init script does
env_dict.update([cls.create_python_path_env_var(YARN_DEV_TOOLS_ROOT_DIR)])
return env_dict
def run(self):
self.config.full_cmd = OsUtils.determine_full_command()
LOG.info(f"Starting umbrella jira fetcher... \n{str(self.config)}")
self.log_current_branch()
self.upstream_repo.fetch(all=True)
self.downstream_repo.fetch(all=True)
if self.config.force_mode:
LOG.info("FORCE MODE is on")
self.do_fetch()
else:
loaded = self.load_pickled_umbrella_data()
if not loaded:
self.do_fetch()
else:
LOG.info("Loaded pickled data from: %s",
self.pickled_data_file)
self.print_summary()
def run_test_result_aggregator_and_send_mail(self):
self._run_aggregator(
exec_mode="gsheet",
gsheet_client_secret=OsUtils.get_env_value(
UnitTestResultAggregatorEnvVar.GSHEET_CLIENT_SECRET.value),
gsheet_spreadsheet=OsUtils.get_env_value(
UnitTestResultAggregatorEnvVar.GSHEET_SPREADHSHEET.value),
gsheet_worksheet=OsUtils.get_env_value(
UnitTestResultAggregatorEnvVar.GSHEET_WORKSHEET.value),
account_email=OsUtils.get_env_value(
CdswEnvVar.MAIL_ACC_USER.value),
request_limit=OsUtils.get_env_value(
UnitTestResultAggregatorEnvVar.REQUEST_LIMIT.value),
match_expression=OsUtils.get_env_value(
UnitTestResultAggregatorEnvVar.MATCH_EXPRESSION.value),
abbreviate_tc_package=OsUtils.get_env_value(
UnitTestResultAggregatorOptionalEnvVar.ABBREV_TC_PACKAGE.value
),
aggregate_filters=OsUtils.get_env_value(
UnitTestResultAggregatorOptionalEnvVar.AGGREGATE_FILTERS.value
),
gsheet_compare_with_jira_table=OsUtils.get_env_value(
UnitTestResultAggregatorOptionalEnvVar.
GSHEET_COMPARE_WITH_JIRA_TABLE.value),
)
self.run_zipper(CommandType.UNIT_TEST_RESULT_AGGREGATOR, debug=True)
date_str = self.current_date_formatted()
sender = "YARN unit test aggregator"
subject = f"YARN unit test aggregator report [start date: {date_str}]"
attachment_fnname: str = f"command_data_{date_str}.zip"
self.send_latest_command_data_in_email(
sender=sender,
subject=subject,
attachment_filename=attachment_fnname,
email_body_file=REPORT_FILE_SHORT_HTML,
)
def run(self):
LOG.info(
f"Starting sending latest command data in email.\n Config: {str(self.config)}"
)
zip_extract_dest = FileUtils.join_path(os.sep, "tmp", "extracted_zip")
ZipFileUtils.extract_zip_file(self.config.email.attachment_file,
zip_extract_dest)
# Pick file from zip that will be the email's body
email_body_file = FileUtils.join_path(os.sep, zip_extract_dest,
self.config.email_body_file)
FileUtils.ensure_file_exists(email_body_file)
email_body_contents: str = FileUtils.read_file(email_body_file)
body_mimetype: EmailMimeType = self._determine_body_mimetype_by_attachment(
email_body_file)
email_service = EmailService(self.config.email.email_conf)
try:
email_service.send_mail(
self.config.email.sender,
self.config.email.subject,
email_body_contents,
self.config.email.recipients,
self.config.email.attachment_file,
body_mimetype=body_mimetype,
override_attachment_filename=self.config.email.
attachment_filename,
)
except SMTPAuthenticationError as smtpe:
ignore_smpth_auth_env: str = OsUtils.get_env_value(
EnvVar.IGNORE_SMTP_AUTH_ERROR.value, "")
LOG.info(
f"Recognized env var '{EnvVar.IGNORE_SMTP_AUTH_ERROR.value}': {ignore_smpth_auth_env}"
)
if not ignore_smpth_auth_env:
raise smtpe
else:
# Swallow exeption
LOG.exception(
f"SMTP auth error occurred but env var "
f"'{EnvVar.IGNORE_SMTP_AUTH_ERROR.value}' was set",
exc_info=True,
)
LOG.info("Finished sending email to recipients")
def setUpClass(cls):
# Test expects that MAIL_ACC_PASSWORD is set with env var
if CdswEnvVar.MAIL_ACC_PASSWORD.value not in os.environ:
raise ValueError(f"Please set '{CdswEnvVar.MAIL_ACC_PASSWORD.value}' env var and re-run the test!")
cls._setup_logging()
cls.repo_root_dir = FileUtils.find_repo_root_dir(__file__, REPO_ROOT_DIRNAME)
found_cdsw_dirs = FileUtils.find_files(
cls.repo_root_dir,
find_type=FindResultType.DIRS,
regex=CDSW_DIRNAME,
parent_dir="yarndevtools",
single_level=False,
full_path_result=True,
)
if len(found_cdsw_dirs) != 1:
raise ValueError(
f"Expected to find 1 dir with name {CDSW_DIRNAME} "
f"and parent dir 'yarndevtools'. "
f"Actual results: {found_cdsw_dirs}"
)
cls.repo_cdsw_root_dir = found_cdsw_dirs[0]
cls.yarn_dev_tools_results_dir = FileUtils.join_path(cls.repo_cdsw_root_dir, "yarndevtools-results")
cls.branchdiff_cdsw_runner_script = YarnCdswBranchDiffTests.find_cdsw_runner_script(
os.path.join(cls.repo_cdsw_root_dir, BRANCH_DIFF_REPORTER_DIR_NAME)
)
cls.docker_test_setup = DockerTestSetup(
DOCKER_IMAGE, create_image=CREATE_IMAGE, dockerfile_location=cls.repo_cdsw_root_dir, logger=CMD_LOG
)
exec_mode_env: str = OsUtils.get_env_value(CdswEnvVar.TEST_EXECUTION_MODE.value, TestExecMode.CLOUDERA.value)
cls.exec_mode: TestExecMode = TestExecMode[exec_mode_env.upper()]
# !! WARNING: User-specific settings !!
if cls.exec_mode == TestExecMode.CLOUDERA:
# We need both upstream / downstream repos for Cloudera-mode
os.environ[CdswEnvVar.CLOUDERA_HADOOP_ROOT.value] = "/Users/snemeth/development/cloudera/hadoop/"
os.environ[CdswEnvVar.HADOOP_DEV_DIR.value] = "/Users/snemeth/development/apache/hadoop"
elif cls.exec_mode == TestExecMode.UPSTREAM:
os.environ[CdswEnvVar.HADOOP_DEV_DIR.value] = "/Users/snemeth/development/apache/hadoop"
os.environ[BranchComparatorEnvVar.REPO_TYPE.value] = RepoType.UPSTREAM.value
os.environ[BranchComparatorEnvVar.FEATURE_BRANCH.value] = "origin/branch-3.3"
os.environ[BranchComparatorEnvVar.MASTER_BRANCH.value] = "origin/trunk"
def run_test_reporter(self,
job_name: str,
recipients=MAIL_ADDR_YARN_ENG_BP,
testcase_filter: str = TC_FILTER_ALL):
if not job_name:
raise ValueError("Jenkins job name should be specified")
process_builds: int = OsUtils.get_env_value(
JenkinsTestReporterEnvVar.BUILD_PROCESSING_LIMIT.value, 1)
LOG.info(f"Processing {process_builds} builds...")
sender = "YARN jenkins test reporter"
tc_filter_param = f"--testcase-filter {testcase_filter}"
self.execute_yarndevtools_script(
f"--debug jenkins_test_reporter "
f"--job-name {job_name} "
f"{self.common_mail_config.as_arguments()}"
f'--sender "{sender}" '
f'--recipients "{recipients}" '
f"{tc_filter_param} "
f"--request-limit {process_builds}")
def __init__(self, parser, args, output_dir: str):
self._validate_args(parser, args)
self.console_mode = getattr(args, "console mode", False)
self.gmail_query = args.gmail_query
self.smart_subject_query = args.smart_subject_query
self.request_limit = getattr(args, "request_limit", 1000000)
self.account_email: str = args.account_email
self.testcase_filters = TestCaseFilters(
TestCaseFilters.convert_raw_match_expressions_to_objs(
getattr(args, "match_expression", None)),
getattr(args, "aggregate_filters", []),
)
self.skip_lines_starting_with: List[str] = getattr(
args, "skip_lines_starting_with", [])
self.email_content_line_sep = getattr(args,
"email_content_line_separator",
DEFAULT_LINE_SEP)
self.truncate_subject_with: str = getattr(args, "truncate_subject",
None)
self.abbrev_tc_package: str = getattr(args, "abbrev_testcase_package",
None)
self.summary_mode = args.summary_mode
self.output_dir = output_dir
self.email_cache_dir = FileUtils.join_path(output_dir, "email_cache")
self.session_dir = ProjectUtils.get_session_dir_under_child_dir(
FileUtils.basename(output_dir))
self.full_cmd: str = OsUtils.determine_full_command_filtered(
filter_password=True)
if self.operation_mode == OperationMode.GSHEET:
worksheet_names: List[str] = [
self.get_worksheet_name(tcf)
for tcf in self.testcase_filters.ALL_VALID_FILTERS
]
LOG.info(
f"Adding worksheets to {self.gsheet_options.__class__.__name__}. "
f"Generated worksheet names: {worksheet_names}")
for worksheet_name in worksheet_names:
self.gsheet_options.add_worksheet(worksheet_name)