def generate_pr_thanks(gitdir, jsondata, branch):
config = b4.get_main_config()
jsondata, config = set_branch_details(gitdir, branch, jsondata, config)
thanks_template = DEFAULT_PR_TEMPLATE
if config['thanks-pr-template']:
# Try to load this template instead
try:
thanks_template = read_template(config['thanks-pr-template'])
except FileNotFoundError:
logger.critical('ERROR: thanks-pr-template says to use %s, but it does not exist',
config['thanks-pr-template'])
sys.exit(2)
if 'merge_commit_id' not in jsondata:
merge_commit_id = git_get_merge_id(gitdir, jsondata['pr_commit_id'])
if not merge_commit_id:
logger.critical('Could not get merge commit id for %s', jsondata['subject'])
logger.critical('Was it actually merged?')
sys.exit(1)
jsondata['merge_commit_id'] = merge_commit_id
# Make a summary
cidmask = config['thanks-commit-url-mask']
if not cidmask:
cidmask = 'merge commit: %s'
jsondata['summary'] = cidmask % jsondata['merge_commit_id']
msg = make_reply(thanks_template, jsondata)
return msg
def main(cmdargs):
if cmdargs.checknewer:
# Force nocache mode
cmdargs.nocache = True
config = b4.get_main_config()
if not cmdargs.localmbox:
if not cmdargs.msgid:
logger.debug('Getting Message-ID from stdin')
msgid = get_msgid_from_stdin()
if msgid is None:
logger.error('Unable to find a valid message-id in stdin.')
sys.exit(1)
else:
msgid = cmdargs.msgid
msgid = msgid.strip('<>')
# Handle the case when someone pastes a full URL to the message
matches = re.search(r'^https?://[^/]+/([^/]+)/([^/]+@[^/]+)', msgid,
re.IGNORECASE)
if matches:
chunks = matches.groups()
msgid = chunks[1]
# Infer the project name from the URL, if possible
if chunks[0] != 'r':
cmdargs.useproject = chunks[0]
# Handle special case when msgid is prepended by id: or rfc822msgid:
if msgid.find('id:') >= 0:
msgid = re.sub(r'^\w*id:', '', msgid)
mboxfile = get_pi_thread_by_msgid(msgid, config, cmdargs)
if mboxfile is None:
return
# Move it into -thread
threadmbox = '%s-thread' % mboxfile
os.rename(mboxfile, threadmbox)
else:
if os.path.exists(cmdargs.localmbox):
threadmbox = cmdargs.localmbox
else:
logger.critical('Mailbox %s does not exist', cmdargs.localmbox)
sys.exit(1)
if threadmbox and cmdargs.checknewer:
get_newest_series(threadmbox)
if cmdargs.subcmd == 'am':
mbox_to_am(threadmbox, config, cmdargs)
if not cmdargs.localmbox:
os.unlink(threadmbox)
else:
mbx = mailbox.mbox(threadmbox)
logger.critical('Saved %s', threadmbox)
logger.critical('%s messages in the thread', len(mbx))
def write_tracked(tracked):
counter = 1
config = b4.get_main_config()
logger.info('Currently tracking:')
for entry in tracked:
logger.info('%3d: %s', counter, entry['subject'])
logger.info(' From: %s <%s>', entry['fromname'], entry['fromemail'])
logger.info(' Date: %s', entry['sentdate'])
logger.info(' Link: %s', config['linkmask'] % entry['msgid'])
counter += 1
def attest_fetch_head(gitdir, lmsg):
config = b4.get_main_config()
attpolicy = config['attestation-policy']
if config['attestation-checkmarks'] == 'fancy':
attpass = b4.PASS_FANCY
attfail = b4.FAIL_FANCY
else:
attpass = b4.PASS_SIMPLE
attfail = b4.FAIL_SIMPLE
# Is FETCH_HEAD a tag or a commit?
htype = b4.git_get_command_lines(gitdir, ['cat-file', '-t', 'FETCH_HEAD'])
passing = False
out = ''
otype = 'unknown'
if len(htype):
otype = htype[0]
if otype == 'tag':
ecode, out = b4.git_run_command(gitdir, ['verify-tag', '--raw', 'FETCH_HEAD'], logstderr=True)
elif otype == 'commit':
ecode, out = b4.git_run_command(gitdir, ['verify-commit', '--raw', 'FETCH_HEAD'], logstderr=True)
lsig = b4.LoreAttestationSignature(out, 'git')
if lsig.good and lsig.valid and lsig.trusted:
passing = True
out = out.strip()
if not len(out) and attpolicy != 'check':
lsig.errors.add('Remote %s is not signed!' % otype)
if passing:
trailer = lsig.attestor.get_trailer(lmsg.fromemail)
logger.info(' ---')
logger.info(' %s %s', attpass, trailer)
return
if lsig.errors:
logger.critical(' ---')
if len(out):
logger.critical(' Pull request is signed, but verification did not succeed:')
else:
logger.critical(' Pull request verification did not succeed:')
for error in lsig.errors:
logger.critical(' %s %s', attfail, error)
if attpolicy == 'hardfail':
import sys
sys.exit(128)
def generate_am_thanks(gitdir, jsondata, branch, since):
config = b4.get_main_config()
jsondata, config = set_branch_details(gitdir, branch, jsondata, config)
thanks_template = DEFAULT_AM_TEMPLATE
if config['thanks-am-template']:
# Try to load this template instead
try:
thanks_template = read_template(config['thanks-am-template'])
except FileNotFoundError:
logger.critical('ERROR: thanks-am-template says to use %s, but it does not exist',
config['thanks-am-template'])
sys.exit(2)
if 'commits' not in jsondata:
commits = auto_locate_series(gitdir, jsondata, branch, since)
else:
commits = jsondata['commits']
cidmask = config['thanks-commit-url-mask']
if not cidmask:
cidmask = 'commit: %s'
slines = list()
counter = 0
nomatch = 0
padlen = len(str(len(commits)))
for commit in commits:
counter += 1
prefix = '[%s/%s] ' % (str(counter).zfill(padlen), len(commits))
slines.append('%s%s' % (prefix, commit[1]))
if commit[0] is None:
slines.append('%s(no commit info)' % (' ' * len(prefix)))
nomatch += 1
else:
slines.append('%s%s' % (' ' * len(prefix), cidmask % commit[0]))
jsondata['summary'] = '\n'.join(slines)
if nomatch == counter:
logger.critical(' WARNING: None of the patches matched for: %s', jsondata['subject'])
logger.critical(' Please review the resulting message')
elif nomatch > 0:
logger.critical(' WARNING: Could not match %s of %s patches in: %s', nomatch, counter, jsondata['subject'])
logger.critical(' Please review the resulting message')
msg = make_reply(thanks_template, jsondata)
return msg
def fetch_remote(gitdir, lmsg, branch=None):
# Do we know anything about this base commit?
if lmsg.pr_base_commit and not b4.git_commit_exists(gitdir, lmsg.pr_base_commit):
logger.critical('ERROR: git knows nothing about commit %s', lmsg.pr_base_commit)
logger.critical(' Are you running inside a git checkout and is it up-to-date?')
return 1
if lmsg.pr_tip_commit != lmsg.pr_remote_tip_commit:
logger.critical('ERROR: commit-id mismatch between pull request and remote')
logger.critical(' msg=%s, remote=%s', lmsg.pr_tip_commit, lmsg.pr_remote_tip_commit)
return 1
# Fetch it now
logger.info(' Fetching %s %s', lmsg.pr_repo, lmsg.pr_ref)
gitargs = ['fetch', lmsg.pr_repo, lmsg.pr_ref]
ecode, out = b4.git_run_command(gitdir, gitargs, logstderr=True)
if ecode > 0:
logger.critical('ERROR: Could not fetch remote:')
logger.critical(out)
return ecode
config = b4.get_main_config()
if config['attestation-policy'] != 'off':
attest_fetch_head(gitdir, lmsg)
logger.info('---')
if branch:
gitargs = ['checkout', '-b', branch, 'FETCH_HEAD']
logger.info('Fetched into branch %s', branch)
ecode, out = b4.git_run_command(gitdir, gitargs)
if ecode > 0:
logger.critical('ERROR: Failed to create branch')
logger.critical(out)
return ecode
else:
logger.info('Successfully fetched into FETCH_HEAD')
thanks_record_pr(lmsg)
return 0
def main(cmdargs):
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
formatter = logging.Formatter('%(message)s')
ch.setFormatter(formatter)
if cmdargs.quiet:
ch.setLevel(logging.CRITICAL)
elif cmdargs.debug:
ch.setLevel(logging.DEBUG)
else:
ch.setLevel(logging.INFO)
logger.addHandler(ch)
config = b4.get_main_config()
if not cmdargs.localmbox:
if not cmdargs.msgid:
logger.debug('Getting Message-ID from stdin')
msgid = get_msgid_from_stdin()
if msgid is None:
logger.error('Unable to find a valid message-id in stdin.')
sys.exit(1)
else:
msgid = cmdargs.msgid
msgid = msgid.strip('<>')
# Handle the case when someone pastes a full URL to the message
matches = re.search(r'^https?://[^/]+/([^/]+)/([^/]+@[^/]+)', msgid,
re.IGNORECASE)
if matches:
chunks = matches.groups()
msgid = chunks[1]
# Infer the project name from the URL, if possible
if chunks[0] != 'r':
cmdargs.useproject = chunks[0]
mboxfile = get_pi_thread_by_msgid(msgid, config, cmdargs)
if mboxfile is None:
return
# Move it into -thread
threadmbox = '%s-thread' % mboxfile
os.rename(mboxfile, threadmbox)
else:
if os.path.exists(cmdargs.localmbox):
threadmbox = cmdargs.localmbox
else:
logger.critical('Mailbox %s does not exist', cmdargs.localmbox)
sys.exit(1)
if threadmbox and cmdargs.checknewer:
get_newest_series(threadmbox)
if cmdargs.amready:
mbox_to_am(threadmbox, config, cmdargs)
if not cmdargs.localmbox:
os.unlink(threadmbox)
else:
mbx = mailbox.mbox(threadmbox)
logger.critical('Saved %s', threadmbox)
logger.critical('%s messages in the thread', len(mbx))
def mbox_to_am(mboxfile, cmdargs):
config = b4.get_main_config()
outdir = cmdargs.outdir
if outdir == '-':
cmdargs.nocover = True
wantver = cmdargs.wantver
wantname = cmdargs.wantname
covertrailers = cmdargs.covertrailers
if os.path.isdir(mboxfile):
mbx = mailbox.Maildir(mboxfile)
else:
mbx = mailbox.mbox(mboxfile)
count = len(mbx)
logger.info('Analyzing %s messages in the thread', count)
lmbx = b4.LoreMailbox()
# Go through the mbox once to populate base series
for key, msg in mbx.items():
lmbx.add_message(msg)
lser = lmbx.get_series(revision=wantver,
sloppytrailers=cmdargs.sloppytrailers)
if lser is None and wantver is None:
logger.critical('No patches found.')
return
if lser is None:
logger.critical('Unable to find revision %s', wantver)
return
if len(lmbx.series) > 1 and not wantver:
logger.info('Will use the latest revision: v%s', lser.revision)
logger.info('You can pick other revisions using the -vN flag')
if wantname:
slug = wantname
if wantname.find('.') > -1:
slug = '.'.join(wantname.split('.')[:-1])
gitbranch = slug
else:
slug = lser.get_slug(extended=True)
gitbranch = lser.get_slug(extended=False)
if outdir != '-':
am_filename = os.path.join(outdir, '%s.mbx' % slug)
am_cover = os.path.join(outdir, '%s.cover' % slug)
if os.path.exists(am_filename):
os.unlink(am_filename)
else:
# Create a temporary file that we will remove later
am_filename = mkstemp('b4-am-stdout')[1]
am_cover = None
logger.info('---')
if cmdargs.cherrypick:
cherrypick = list()
if cmdargs.cherrypick == '_':
msgid = b4.get_msgid(cmdargs)
# Only grab the exact msgid provided
at = 0
for lmsg in lser.patches[1:]:
at += 1
if lmsg and lmsg.msgid == msgid:
cherrypick = [at]
cmdargs.cherrypick = f'<{msgid}>'
break
if not len(cherrypick):
logger.critical(
'Specified msgid is not present in the series, cannot cherrypick'
)
sys.exit(1)
elif cmdargs.cherrypick.find('*') >= 0:
# Globbing on subject
at = 0
for lmsg in lser.patches[1:]:
at += 1
if fnmatch.fnmatch(lmsg.subject, cmdargs.cherrypick):
cherrypick.append(at)
if not len(cherrypick):
logger.critical(
'Could not match "%s" to any subjects in the series',
cmdargs.cherrypick)
sys.exit(1)
else:
cherrypick = list(
b4.parse_int_range(cmdargs.cherrypick,
upper=len(lser.patches) - 1))
else:
cherrypick = None
logger.critical('Writing %s', am_filename)
mbx = mailbox.mbox(am_filename)
try:
am_mbx = lser.save_am_mbox(mbx,
noaddtrailers=cmdargs.noaddtrailers,
covertrailers=covertrailers,
trailer_order=config['trailer-order'],
addmysob=cmdargs.addmysob,
addlink=cmdargs.addlink,
linkmask=config['linkmask'],
cherrypick=cherrypick)
except KeyError:
sys.exit(1)
logger.info('---')
if cherrypick is None:
logger.critical('Total patches: %s', len(am_mbx))
else:
logger.info('Total patches: %s (cherrypicked: %s)', len(am_mbx),
cmdargs.cherrypick)
if lser.has_cover and lser.patches[
0].followup_trailers and not covertrailers:
# Warn that some trailers were sent to the cover letter
logger.critical('---')
logger.critical('NOTE: Some trailers were sent to the cover letter:')
for trailer in lser.patches[0].followup_trailers:
logger.critical(' %s: %s', trailer[0], trailer[1])
logger.critical('NOTE: Rerun with -t to apply them to all patches')
if len(lser.trailer_mismatches):
logger.critical('---')
logger.critical(
'NOTE: some trailers ignored due to from/email mismatches:')
for tname, tvalue, fname, femail in lser.trailer_mismatches:
logger.critical(' ! Trailer: %s: %s', tname, tvalue)
logger.critical(' Msg From: %s <%s>', fname, femail)
logger.critical('NOTE: Rerun with -S to apply them anyway')
topdir = None
# Are we in a git tree and if so, what is our toplevel?
gitargs = ['rev-parse', '--show-toplevel']
lines = b4.git_get_command_lines(None, gitargs)
if len(lines) == 1:
topdir = lines[0]
if cmdargs.threeway:
if not topdir:
logger.critical('WARNING: cannot prepare 3-way (not in a git dir)')
elif not lser.complete:
logger.critical(
'WARNING: cannot prepare 3-way (series incomplete)')
else:
rstart, rend = lser.make_fake_am_range(gitdir=None)
if rstart and rend:
logger.info(
'Prepared a fake commit range for 3-way merge (%.12s..%.12s)',
rstart, rend)
logger.critical('---')
if not lser.complete and not cmdargs.cherrypick:
logger.critical('WARNING: Thread incomplete!')
if lser.has_cover and not cmdargs.nocover:
lser.save_cover(am_cover)
top_msgid = None
first_body = None
for lmsg in lser.patches:
if lmsg is not None:
first_body = lmsg.body
top_msgid = lmsg.msgid
break
if top_msgid is None:
logger.critical('Could not find any patches in the series.')
return
linkurl = config['linkmask'] % top_msgid
if cmdargs.quiltready:
q_dirname = os.path.join(outdir, '%s.patches' % slug)
am_mbox_to_quilt(am_mbx, q_dirname)
logger.critical('Quilt: %s', q_dirname)
logger.critical(' Link: %s', linkurl)
base_commit = None
matches = re.search(r'base-commit: .*?([0-9a-f]+)', first_body,
re.MULTILINE)
if matches:
base_commit = matches.groups()[0]
else:
# Try a more relaxed search
matches = re.search(r'based on .*?([0-9a-f]{40})', first_body,
re.MULTILINE)
if matches:
base_commit = matches.groups()[0]
if base_commit:
logger.critical(' Base: %s', base_commit)
logger.critical(' git checkout -b %s %s', gitbranch, base_commit)
if cmdargs.outdir != '-':
logger.critical(' git am %s', am_filename)
else:
cleanmsg = ''
if topdir is not None:
checked, mismatches = lser.check_applies_clean(topdir)
if mismatches == 0 and checked != mismatches:
cleanmsg = ' (applies clean to current tree)'
elif cmdargs.guessbase:
# Look at the last 10 tags and see if it applies cleanly to
# any of them. I'm not sure how useful this is, but I'm going
# to put it in for now and maybe remove later if it causes
# problems or slowness
if checked != mismatches:
best_matches = mismatches
cleanmsg = ' (best guess: current tree)'
else:
best_matches = None
# sort the tags by authordate
gitargs = ['tag', '-l', '--sort=-taggerdate']
lines = b4.git_get_command_lines(None, gitargs)
if lines:
# Check last 10 tags
for tag in lines[:10]:
logger.debug('Checking base-commit possibility for %s',
tag)
checked, mismatches = lser.check_applies_clean(
topdir, tag)
if mismatches == 0 and checked != mismatches:
cleanmsg = ' (applies clean to: %s)' % tag
break
# did they all mismatch?
if checked == mismatches:
continue
if best_matches is None or mismatches < best_matches:
best_matches = mismatches
cleanmsg = ' (best guess: %s)' % tag
logger.critical(' Base: not found%s', cleanmsg)
if cmdargs.outdir != '-':
logger.critical(' git am %s', am_filename)
am_mbx.close()
if cmdargs.outdir == '-':
logger.info('---')
with open(am_filename, 'rb') as fh:
shutil.copyfileobj(fh, sys.stdout.buffer)
os.unlink(am_filename)
thanks_record_am(lser, cherrypick=cherrypick)
def create_attestation(cmdargs):
attlines = list()
subject = 'Patch attestation'
for patchfile in cmdargs.patchfile:
with open(patchfile, 'r', encoding='utf-8') as fh:
content = fh.read()
if content.find('From') != 0:
logger.info('SKIP | %s', os.path.basename(patchfile))
continue
msg = email.message_from_string(content)
lmsg = b4.LoreMessage(msg)
lmsg.load_hashes()
att = lmsg.attestation
if att is None:
logger.info('SKIP | %s', os.path.basename(patchfile))
# See if it's a cover letter
if lmsg.counters_inferred or lmsg.counter > 0:
# No
continue
newprefs = list()
for prefix in lmsg.lsubject.prefixes:
if prefix.lower() == 'patch':
newprefs.append('PSIGN')
elif prefix == '%s/%s' % (lmsg.counter, lmsg.expected):
newprefs.append('X/%s' % lmsg.expected)
else:
newprefs.append(prefix)
subject = '[%s] %s' % (' '.join(newprefs), lmsg.subject)
continue
logger.info('HASH | %s', os.path.basename(patchfile))
attlines.append('%s:' % att.attid)
attlines.append(' i: %s' % att.i)
attlines.append(' m: %s' % att.m)
attlines.append(' p: %s' % att.p)
payload = '\n'.join(attlines)
usercfg = b4.get_user_config()
gpgargs = list()
if 'signingkey' in usercfg:
gpgargs += ['-u', usercfg['signingkey']]
gpgargs += ['--clearsign',
'--comment', 'att-fmt-ver: %s' % b4.ATTESTATION_FORMAT_VER,
'--comment', 'att-hash: sha256',
]
ecode, signed = b4.gpg_run_command(gpgargs, stdin=payload.encode('utf-8'))
if ecode > 0:
config = b4.get_main_config()
logger.critical('ERROR: Unable to sign using %s', config['gpgbin'])
sys.exit(1)
att_msg = email.message.EmailMessage()
att_msg.set_payload(signed.encode('utf-8'))
sender = cmdargs.sender
if '>' not in sender:
sender = '<%s>' % sender
att_msg['From'] = sender
att_msg['To'] = '<*****@*****.**>'
att_msg['Message-Id'] = email.utils.make_msgid(domain='kernel.org')
att_msg['Subject'] = subject
logger.info('---')
if not cmdargs.nosubmit:
# Try to deliver it via mail.kernel.org
try:
mailserver = smtplib.SMTP('mail.kernel.org', 587)
# identify ourselves to smtp gmail client
mailserver.ehlo()
# secure our email with tls encryption
mailserver.starttls()
# re-identify ourselves as an encrypted connection
mailserver.ehlo()
logger.info('Delivering via mail.kernel.org')
mailserver.sendmail('*****@*****.**', '*****@*****.**', att_msg.as_string())
mailserver.quit()
sys.exit(0)
except Exception as ex:
logger.info('Could not deliver: %s', ex)
# Future iterations will also be able to submit this to a RESTful URL
# at git.kernel.org, in order not to depend on avaialbility of SMTP gateways
with open(cmdargs.output, 'wb') as fh:
fh.write(att_msg.as_bytes())
logger.info('Wrote %s', cmdargs.output)
logger.info('You can send it using:')
logger.info(' sendmail -oi [email protected] < %s', cmdargs.output)
logger.info(' mutt -H %s', cmdargs.output)
def verify_attestation(cmdargs):
config = b4.get_main_config()
if cmdargs.tofu:
config['attestation-trust-model'] = 'tofu'
exact_from_match = True
if cmdargs.ignorefrom:
exact_from_match = False
mbx = mailbox.mbox(cmdargs.mbox[0])
if cmdargs.attfile:
b4.LoreAttestationDocument.load_from_file(cmdargs.attfile)
eligible = list()
for msg in mbx:
lmsg = b4.LoreMessage(msg)
if lmsg.has_diff:
eligible.append(lmsg)
continue
# See if body has "att-fmt-ver
if re.search(r'^Comment: att-fmt-ver:', lmsg.body, re.I | re.M):
logger.debug('Found attestation message')
b4.LoreAttestationDocument.load_from_string(lmsg.msgid, lmsg.body)
logger.debug('SKIP | %s', msg['Subject'])
if not len(eligible):
logger.error('No patches found in %s', cmdargs.mbox[0])
sys.exit(1)
logger.info('---')
attrailers = set()
ecode = 1
if config['attestation-checkmarks'] == 'fancy':
attpass = b4.PASS_FANCY
attfail = b4.FAIL_FANCY
else:
attpass = b4.PASS_SIMPLE
attfail = b4.FAIL_SIMPLE
for lmsg in eligible:
attdoc = lmsg.get_attestation(lore_lookup=True, exact_from_match=exact_from_match)
if not attdoc:
logger.critical('%s %s', attfail, lmsg.full_subject)
if not cmdargs.nofast:
logger.critical('Aborting due to failure.')
ecode = 1
break
else:
ecode = 128
continue
if ecode != 128:
ecode = 0
logger.critical('%s %s', attpass, lmsg.full_subject)
attrailers.add(attdoc.attestor.get_trailer(lmsg.fromemail))
logger.critical('---')
if ecode > 0:
logger.critical('Attestation verification failed.')
errors = set()
for attdoc in b4.ATTESTATIONS:
errors.update(attdoc.errors)
if len(errors):
logger.critical('---')
logger.critical('The validation process reported the following errors:')
for error in errors:
logger.critical(' %s %s', attfail, error)
else:
logger.critical('All patches passed attestation:')
for attrailer in attrailers:
logger.critical(' %s %s', attpass, attrailer)
sys.exit(ecode)
