def extract_zip_file(file: str, path: str):
# Apparently, ZipFile does not resolve symlinks so let's do it manually
if os.path.islink(file):
file = os.path.realpath(file)
FileUtils.ensure_file_exists(file)
zip_file = zipfile.ZipFile(file, "r")
zip_file.extractall(path)
def _validate_args(self, parser, args):
if args.gsheet and (args.gsheet_client_secret is None
or args.gsheet_spreadsheet is None
or args.gsheet_worksheet is None):
parser.error(
"--gsheet requires the following arguments: "
"--gsheet-client-secret, --gsheet-spreadsheet and --gsheet-worksheet."
)
if args.do_print:
self.operation_mode = OperationMode.PRINT
elif args.gsheet:
self.operation_mode = OperationMode.GSHEET
self.gsheet_options = GSheetOptions(args.gsheet_client_secret,
args.gsheet_spreadsheet,
worksheet=None)
self.gsheet_jira_table = getattr(args,
"gsheet_compare_with_jira_table",
None)
if self.operation_mode not in VALID_OPERATION_MODES:
raise ValueError(
f"Unknown state! "
f"Operation mode should be any of {VALID_OPERATION_MODES}, but it is set to: {self.operation_mode}"
)
if hasattr(args, "gmail_credentials_file"):
FileUtils.ensure_file_exists(args.gmail_credentials_file)
def run(self):
LOG.info(
f"Starting sending latest command data in email.\n Config: {str(self.config)}"
)
zip_extract_dest = FileUtils.join_path(os.sep, "tmp", "extracted_zip")
ZipFileUtils.extract_zip_file(self.config.email.attachment_file,
zip_extract_dest)
# Pick file from zip that will be the email's body
email_body_file = FileUtils.join_path(os.sep, zip_extract_dest,
self.config.email_body_file)
FileUtils.ensure_file_exists(email_body_file)
email_body_contents: str = FileUtils.read_file(email_body_file)
body_mimetype: EmailMimeType = self._determine_body_mimetype_by_attachment(
email_body_file)
email_service = EmailService(self.config.email.email_conf)
try:
email_service.send_mail(
self.config.email.sender,
self.config.email.subject,
email_body_contents,
self.config.email.recipients,
self.config.email.attachment_file,
body_mimetype=body_mimetype,
override_attachment_filename=self.config.email.
attachment_filename,
)
except SMTPAuthenticationError as smtpe:
ignore_smpth_auth_env: str = OsUtils.get_env_value(
EnvVar.IGNORE_SMTP_AUTH_ERROR.value, "")
LOG.info(
f"Recognized env var '{EnvVar.IGNORE_SMTP_AUTH_ERROR.value}': {ignore_smpth_auth_env}"
)
if not ignore_smpth_auth_env:
raise smtpe
else:
# Swallow exeption
LOG.exception(
f"SMTP auth error occurred but env var "
f"'{EnvVar.IGNORE_SMTP_AUTH_ERROR.value}' was set",
exc_info=True,
)
LOG.info("Finished sending email to recipients")
def send_mail(self, sender: str, subject: str, body: str, recipients: List[str],
attachment_file=None,
override_attachment_filename: str = None,
body_mimetype: EmailMimeType = EmailMimeType.PLAIN):
self._validate_config(recipients)
email_msg = None
if attachment_file:
FileUtils.ensure_file_exists(attachment_file)
# https://stackoverflow.com/a/169406/1106893
email_msg = MIMEMultipart()
email_msg.attach(MIMEText(str(body), body_mimetype.value))
if override_attachment_filename:
attachment = self._create_attachment(attachment_file, attachment_name=override_attachment_filename)
else:
attachment = self._create_attachment(attachment_file)
email_msg.attach(attachment)
else:
email_msg = MIMEText(str(body))
recipients_comma_separated = self._add_common_email_data(email_msg, recipients, sender, subject)
self._connect_to_server_and_send(email_msg, recipients, recipients_comma_separated, sender)
def write_all_changes_files(self):
"""
Iterate over changed files, print all matching changes to the particular file
Create changes file for each touched file
:return:
"""
LOG.info("Recording changes of individual files...")
for idx, changed_file in enumerate(self.data.list_of_changed_files):
target_file = FileUtils.join_path(
self.config.umbrella_result_basedir, "changes",
os.path.basename(changed_file))
FileUtils.ensure_file_exists(target_file, create=True)
# NOTE: It seems impossible to call the following command with gitpython:
# git log --follow --oneline -- <file>
# Use a simple CLI command instead
# TODO check if change file exists - It can happen that it was deleted
cli_command = (
f"cd {self.upstream_repo.repo_path} && "
f"git log {ORIGIN_TRUNK} --follow --oneline -- {changed_file} | "
f'egrep "{self.data.piped_jira_ids}"')
LOG.info("[%d / %d] CLI command: %s", idx + 1,
len(self.data.list_of_changed_files), cli_command)
cmd, output = CommandRunner.run_cli_command(
cli_command,
fail_on_empty_output=False,
print_command=False,
fail_on_error=False)
if output:
LOG.info("Saving changes result to file: %s", target_file)
FileUtils.save_to_file(target_file, output)
else:
LOG.error(
f"Failed to detect changes of file: {changed_file}. CLI command was: {cli_command}. "
f"This seems to be a programming error. Exiting...")
FileUtils.save_to_file(target_file, "")
sys.exit(1)
def run(self):
patch_file = self.args.patch_file
FileUtils.ensure_file_exists(patch_file, create=False)
patch_file_name = FileUtils.path_basename(patch_file)
matches = RegexUtils.ensure_matches_pattern(patch_file_name, YARN_PATCH_FILENAME_REGEX)
if not matches:
raise ValueError(
f"Filename '{patch_file_name}' (full path: {patch_file}) "
f"does not match usual patch file pattern: '{YARN_PATCH_FILENAME_REGEX}'!"
)
orig_branch = self.upstream_repo.get_current_branch_name()
LOG.info("Current branch: %s", orig_branch)
target_branch = BRANCH_PREFIX + RegexUtils.get_matched_group(patch_file, YARN_PATCH_FILENAME_REGEX, 1)
LOG.info("Target branch: %s", target_branch)
clean = self.upstream_repo.is_working_directory_clean()
if not clean:
raise ValueError("git working directory is not clean, please stash or drop your changes")
self.upstream_repo.checkout_branch(self.base_branch)
self.upstream_repo.pull(ORIGIN)
diff = self.upstream_repo.diff_between_refs(self.remote_base_branch, self.base_branch)
if diff:
raise ValueError(
f"There is a diff between local {self.base_branch} and {self.remote_base_branch}! "
f"Run 'git reset {self.remote_base_branch} --hard' and re-run the script!"
)
apply_result = self.upstream_repo.apply_check(patch_file, raise_exception=False)
if not apply_result:
self.upstream_repo.checkout_previous_branch()
cmd = "git apply " + patch_file
raise ValueError(
f"Patch does not apply to {self.base_branch}, please resolve the conflicts manually. "
f"Run this command to apply the patch again: {cmd}"
)
LOG.info("Patch %s applies cleanly to %s", patch_file, self.base_branch)
branch_exists = self.upstream_repo.is_branch_exist(target_branch)
base_ref = self.base_branch
if not branch_exists:
success = self.upstream_repo.checkout_new_branch(target_branch, base_ref)
if not success:
raise ValueError(f"Cannot checkout new branch {target_branch} based on ref {base_ref}")
LOG.info("Checked out branch %s based on ref %s", target_branch, base_ref)
else:
branch_pattern = target_branch + "*"
branches = self.upstream_repo.list_branches(branch_pattern)
LOG.info("Found existing review branches for this patch: %s", branches)
target_branch = PatchUtils.get_next_review_branch_name(branches)
LOG.info("Creating new version of review branch as: %s", target_branch)
success = self.upstream_repo.checkout_new_branch(target_branch, base_ref)
if not success:
raise ValueError(f"Cannot checkout new branch {target_branch} based on ref {base_ref}")
self.upstream_repo.apply_patch(patch_file, include_check=False)
LOG.info("Successfully applied patch: %s", patch_file)
commit_msg = f"patch file: {patch_file}"
self.upstream_repo.add_all_and_commit(commit_msg)
LOG.info("Committed changes of patch: %s with message: %s", patch_file, commit_msg)