def generate_pr_thanks(gitdir, jsondata, branch): config = b4.get_main_config() jsondata, config = set_branch_details(gitdir, branch, jsondata, config) thanks_template = DEFAULT_PR_TEMPLATE if config['thanks-pr-template']: # Try to load this template instead try: thanks_template = read_template(config['thanks-pr-template']) except FileNotFoundError: logger.critical('ERROR: thanks-pr-template says to use %s, but it does not exist', config['thanks-pr-template']) sys.exit(2) if 'merge_commit_id' not in jsondata: merge_commit_id = git_get_merge_id(gitdir, jsondata['pr_commit_id']) if not merge_commit_id: logger.critical('Could not get merge commit id for %s', jsondata['subject']) logger.critical('Was it actually merged?') sys.exit(1) jsondata['merge_commit_id'] = merge_commit_id # Make a summary cidmask = config['thanks-commit-url-mask'] if not cidmask: cidmask = 'merge commit: %s' jsondata['summary'] = cidmask % jsondata['merge_commit_id'] msg = make_reply(thanks_template, jsondata) return msg
def main(cmdargs): if cmdargs.checknewer: # Force nocache mode cmdargs.nocache = True config = b4.get_main_config() if not cmdargs.localmbox: if not cmdargs.msgid: logger.debug('Getting Message-ID from stdin') msgid = get_msgid_from_stdin() if msgid is None: logger.error('Unable to find a valid message-id in stdin.') sys.exit(1) else: msgid = cmdargs.msgid msgid = msgid.strip('<>') # Handle the case when someone pastes a full URL to the message matches = re.search(r'^https?://[^/]+/([^/]+)/([^/]+@[^/]+)', msgid, re.IGNORECASE) if matches: chunks = matches.groups() msgid = chunks[1] # Infer the project name from the URL, if possible if chunks[0] != 'r': cmdargs.useproject = chunks[0] # Handle special case when msgid is prepended by id: or rfc822msgid: if msgid.find('id:') >= 0: msgid = re.sub(r'^\w*id:', '', msgid) mboxfile = get_pi_thread_by_msgid(msgid, config, cmdargs) if mboxfile is None: return # Move it into -thread threadmbox = '%s-thread' % mboxfile os.rename(mboxfile, threadmbox) else: if os.path.exists(cmdargs.localmbox): threadmbox = cmdargs.localmbox else: logger.critical('Mailbox %s does not exist', cmdargs.localmbox) sys.exit(1) if threadmbox and cmdargs.checknewer: get_newest_series(threadmbox) if cmdargs.subcmd == 'am': mbox_to_am(threadmbox, config, cmdargs) if not cmdargs.localmbox: os.unlink(threadmbox) else: mbx = mailbox.mbox(threadmbox) logger.critical('Saved %s', threadmbox) logger.critical('%s messages in the thread', len(mbx))
def write_tracked(tracked): counter = 1 config = b4.get_main_config() logger.info('Currently tracking:') for entry in tracked: logger.info('%3d: %s', counter, entry['subject']) logger.info(' From: %s <%s>', entry['fromname'], entry['fromemail']) logger.info(' Date: %s', entry['sentdate']) logger.info(' Link: %s', config['linkmask'] % entry['msgid']) counter += 1
def attest_fetch_head(gitdir, lmsg): config = b4.get_main_config() attpolicy = config['attestation-policy'] if config['attestation-checkmarks'] == 'fancy': attpass = b4.PASS_FANCY attfail = b4.FAIL_FANCY else: attpass = b4.PASS_SIMPLE attfail = b4.FAIL_SIMPLE # Is FETCH_HEAD a tag or a commit? htype = b4.git_get_command_lines(gitdir, ['cat-file', '-t', 'FETCH_HEAD']) passing = False out = '' otype = 'unknown' if len(htype): otype = htype[0] if otype == 'tag': ecode, out = b4.git_run_command(gitdir, ['verify-tag', '--raw', 'FETCH_HEAD'], logstderr=True) elif otype == 'commit': ecode, out = b4.git_run_command(gitdir, ['verify-commit', '--raw', 'FETCH_HEAD'], logstderr=True) lsig = b4.LoreAttestationSignature(out, 'git') if lsig.good and lsig.valid and lsig.trusted: passing = True out = out.strip() if not len(out) and attpolicy != 'check': lsig.errors.add('Remote %s is not signed!' % otype) if passing: trailer = lsig.attestor.get_trailer(lmsg.fromemail) logger.info(' ---') logger.info(' %s %s', attpass, trailer) return if lsig.errors: logger.critical(' ---') if len(out): logger.critical(' Pull request is signed, but verification did not succeed:') else: logger.critical(' Pull request verification did not succeed:') for error in lsig.errors: logger.critical(' %s %s', attfail, error) if attpolicy == 'hardfail': import sys sys.exit(128)
def generate_am_thanks(gitdir, jsondata, branch, since): config = b4.get_main_config() jsondata, config = set_branch_details(gitdir, branch, jsondata, config) thanks_template = DEFAULT_AM_TEMPLATE if config['thanks-am-template']: # Try to load this template instead try: thanks_template = read_template(config['thanks-am-template']) except FileNotFoundError: logger.critical('ERROR: thanks-am-template says to use %s, but it does not exist', config['thanks-am-template']) sys.exit(2) if 'commits' not in jsondata: commits = auto_locate_series(gitdir, jsondata, branch, since) else: commits = jsondata['commits'] cidmask = config['thanks-commit-url-mask'] if not cidmask: cidmask = 'commit: %s' slines = list() counter = 0 nomatch = 0 padlen = len(str(len(commits))) for commit in commits: counter += 1 prefix = '[%s/%s] ' % (str(counter).zfill(padlen), len(commits)) slines.append('%s%s' % (prefix, commit[1])) if commit[0] is None: slines.append('%s(no commit info)' % (' ' * len(prefix))) nomatch += 1 else: slines.append('%s%s' % (' ' * len(prefix), cidmask % commit[0])) jsondata['summary'] = '\n'.join(slines) if nomatch == counter: logger.critical(' WARNING: None of the patches matched for: %s', jsondata['subject']) logger.critical(' Please review the resulting message') elif nomatch > 0: logger.critical(' WARNING: Could not match %s of %s patches in: %s', nomatch, counter, jsondata['subject']) logger.critical(' Please review the resulting message') msg = make_reply(thanks_template, jsondata) return msg
def fetch_remote(gitdir, lmsg, branch=None): # Do we know anything about this base commit? if lmsg.pr_base_commit and not b4.git_commit_exists(gitdir, lmsg.pr_base_commit): logger.critical('ERROR: git knows nothing about commit %s', lmsg.pr_base_commit) logger.critical(' Are you running inside a git checkout and is it up-to-date?') return 1 if lmsg.pr_tip_commit != lmsg.pr_remote_tip_commit: logger.critical('ERROR: commit-id mismatch between pull request and remote') logger.critical(' msg=%s, remote=%s', lmsg.pr_tip_commit, lmsg.pr_remote_tip_commit) return 1 # Fetch it now logger.info(' Fetching %s %s', lmsg.pr_repo, lmsg.pr_ref) gitargs = ['fetch', lmsg.pr_repo, lmsg.pr_ref] ecode, out = b4.git_run_command(gitdir, gitargs, logstderr=True) if ecode > 0: logger.critical('ERROR: Could not fetch remote:') logger.critical(out) return ecode config = b4.get_main_config() if config['attestation-policy'] != 'off': attest_fetch_head(gitdir, lmsg) logger.info('---') if branch: gitargs = ['checkout', '-b', branch, 'FETCH_HEAD'] logger.info('Fetched into branch %s', branch) ecode, out = b4.git_run_command(gitdir, gitargs) if ecode > 0: logger.critical('ERROR: Failed to create branch') logger.critical(out) return ecode else: logger.info('Successfully fetched into FETCH_HEAD') thanks_record_pr(lmsg) return 0
def main(cmdargs): logger.setLevel(logging.DEBUG) ch = logging.StreamHandler() formatter = logging.Formatter('%(message)s') ch.setFormatter(formatter) if cmdargs.quiet: ch.setLevel(logging.CRITICAL) elif cmdargs.debug: ch.setLevel(logging.DEBUG) else: ch.setLevel(logging.INFO) logger.addHandler(ch) config = b4.get_main_config() if not cmdargs.localmbox: if not cmdargs.msgid: logger.debug('Getting Message-ID from stdin') msgid = get_msgid_from_stdin() if msgid is None: logger.error('Unable to find a valid message-id in stdin.') sys.exit(1) else: msgid = cmdargs.msgid msgid = msgid.strip('<>') # Handle the case when someone pastes a full URL to the message matches = re.search(r'^https?://[^/]+/([^/]+)/([^/]+@[^/]+)', msgid, re.IGNORECASE) if matches: chunks = matches.groups() msgid = chunks[1] # Infer the project name from the URL, if possible if chunks[0] != 'r': cmdargs.useproject = chunks[0] mboxfile = get_pi_thread_by_msgid(msgid, config, cmdargs) if mboxfile is None: return # Move it into -thread threadmbox = '%s-thread' % mboxfile os.rename(mboxfile, threadmbox) else: if os.path.exists(cmdargs.localmbox): threadmbox = cmdargs.localmbox else: logger.critical('Mailbox %s does not exist', cmdargs.localmbox) sys.exit(1) if threadmbox and cmdargs.checknewer: get_newest_series(threadmbox) if cmdargs.amready: mbox_to_am(threadmbox, config, cmdargs) if not cmdargs.localmbox: os.unlink(threadmbox) else: mbx = mailbox.mbox(threadmbox) logger.critical('Saved %s', threadmbox) logger.critical('%s messages in the thread', len(mbx))
def mbox_to_am(mboxfile, cmdargs): config = b4.get_main_config() outdir = cmdargs.outdir if outdir == '-': cmdargs.nocover = True wantver = cmdargs.wantver wantname = cmdargs.wantname covertrailers = cmdargs.covertrailers if os.path.isdir(mboxfile): mbx = mailbox.Maildir(mboxfile) else: mbx = mailbox.mbox(mboxfile) count = len(mbx) logger.info('Analyzing %s messages in the thread', count) lmbx = b4.LoreMailbox() # Go through the mbox once to populate base series for key, msg in mbx.items(): lmbx.add_message(msg) lser = lmbx.get_series(revision=wantver, sloppytrailers=cmdargs.sloppytrailers) if lser is None and wantver is None: logger.critical('No patches found.') return if lser is None: logger.critical('Unable to find revision %s', wantver) return if len(lmbx.series) > 1 and not wantver: logger.info('Will use the latest revision: v%s', lser.revision) logger.info('You can pick other revisions using the -vN flag') if wantname: slug = wantname if wantname.find('.') > -1: slug = '.'.join(wantname.split('.')[:-1]) gitbranch = slug else: slug = lser.get_slug(extended=True) gitbranch = lser.get_slug(extended=False) if outdir != '-': am_filename = os.path.join(outdir, '%s.mbx' % slug) am_cover = os.path.join(outdir, '%s.cover' % slug) if os.path.exists(am_filename): os.unlink(am_filename) else: # Create a temporary file that we will remove later am_filename = mkstemp('b4-am-stdout')[1] am_cover = None logger.info('---') if cmdargs.cherrypick: cherrypick = list() if cmdargs.cherrypick == '_': msgid = b4.get_msgid(cmdargs) # Only grab the exact msgid provided at = 0 for lmsg in lser.patches[1:]: at += 1 if lmsg and lmsg.msgid == msgid: cherrypick = [at] cmdargs.cherrypick = f'<{msgid}>' break if not len(cherrypick): logger.critical( 'Specified msgid is not present in the series, cannot cherrypick' ) sys.exit(1) elif cmdargs.cherrypick.find('*') >= 0: # Globbing on subject at = 0 for lmsg in lser.patches[1:]: at += 1 if fnmatch.fnmatch(lmsg.subject, cmdargs.cherrypick): cherrypick.append(at) if not len(cherrypick): logger.critical( 'Could not match "%s" to any subjects in the series', cmdargs.cherrypick) sys.exit(1) else: cherrypick = list( b4.parse_int_range(cmdargs.cherrypick, upper=len(lser.patches) - 1)) else: cherrypick = None logger.critical('Writing %s', am_filename) mbx = mailbox.mbox(am_filename) try: am_mbx = lser.save_am_mbox(mbx, noaddtrailers=cmdargs.noaddtrailers, covertrailers=covertrailers, trailer_order=config['trailer-order'], addmysob=cmdargs.addmysob, addlink=cmdargs.addlink, linkmask=config['linkmask'], cherrypick=cherrypick) except KeyError: sys.exit(1) logger.info('---') if cherrypick is None: logger.critical('Total patches: %s', len(am_mbx)) else: logger.info('Total patches: %s (cherrypicked: %s)', len(am_mbx), cmdargs.cherrypick) if lser.has_cover and lser.patches[ 0].followup_trailers and not covertrailers: # Warn that some trailers were sent to the cover letter logger.critical('---') logger.critical('NOTE: Some trailers were sent to the cover letter:') for trailer in lser.patches[0].followup_trailers: logger.critical(' %s: %s', trailer[0], trailer[1]) logger.critical('NOTE: Rerun with -t to apply them to all patches') if len(lser.trailer_mismatches): logger.critical('---') logger.critical( 'NOTE: some trailers ignored due to from/email mismatches:') for tname, tvalue, fname, femail in lser.trailer_mismatches: logger.critical(' ! Trailer: %s: %s', tname, tvalue) logger.critical(' Msg From: %s <%s>', fname, femail) logger.critical('NOTE: Rerun with -S to apply them anyway') topdir = None # Are we in a git tree and if so, what is our toplevel? gitargs = ['rev-parse', '--show-toplevel'] lines = b4.git_get_command_lines(None, gitargs) if len(lines) == 1: topdir = lines[0] if cmdargs.threeway: if not topdir: logger.critical('WARNING: cannot prepare 3-way (not in a git dir)') elif not lser.complete: logger.critical( 'WARNING: cannot prepare 3-way (series incomplete)') else: rstart, rend = lser.make_fake_am_range(gitdir=None) if rstart and rend: logger.info( 'Prepared a fake commit range for 3-way merge (%.12s..%.12s)', rstart, rend) logger.critical('---') if not lser.complete and not cmdargs.cherrypick: logger.critical('WARNING: Thread incomplete!') if lser.has_cover and not cmdargs.nocover: lser.save_cover(am_cover) top_msgid = None first_body = None for lmsg in lser.patches: if lmsg is not None: first_body = lmsg.body top_msgid = lmsg.msgid break if top_msgid is None: logger.critical('Could not find any patches in the series.') return linkurl = config['linkmask'] % top_msgid if cmdargs.quiltready: q_dirname = os.path.join(outdir, '%s.patches' % slug) am_mbox_to_quilt(am_mbx, q_dirname) logger.critical('Quilt: %s', q_dirname) logger.critical(' Link: %s', linkurl) base_commit = None matches = re.search(r'base-commit: .*?([0-9a-f]+)', first_body, re.MULTILINE) if matches: base_commit = matches.groups()[0] else: # Try a more relaxed search matches = re.search(r'based on .*?([0-9a-f]{40})', first_body, re.MULTILINE) if matches: base_commit = matches.groups()[0] if base_commit: logger.critical(' Base: %s', base_commit) logger.critical(' git checkout -b %s %s', gitbranch, base_commit) if cmdargs.outdir != '-': logger.critical(' git am %s', am_filename) else: cleanmsg = '' if topdir is not None: checked, mismatches = lser.check_applies_clean(topdir) if mismatches == 0 and checked != mismatches: cleanmsg = ' (applies clean to current tree)' elif cmdargs.guessbase: # Look at the last 10 tags and see if it applies cleanly to # any of them. I'm not sure how useful this is, but I'm going # to put it in for now and maybe remove later if it causes # problems or slowness if checked != mismatches: best_matches = mismatches cleanmsg = ' (best guess: current tree)' else: best_matches = None # sort the tags by authordate gitargs = ['tag', '-l', '--sort=-taggerdate'] lines = b4.git_get_command_lines(None, gitargs) if lines: # Check last 10 tags for tag in lines[:10]: logger.debug('Checking base-commit possibility for %s', tag) checked, mismatches = lser.check_applies_clean( topdir, tag) if mismatches == 0 and checked != mismatches: cleanmsg = ' (applies clean to: %s)' % tag break # did they all mismatch? if checked == mismatches: continue if best_matches is None or mismatches < best_matches: best_matches = mismatches cleanmsg = ' (best guess: %s)' % tag logger.critical(' Base: not found%s', cleanmsg) if cmdargs.outdir != '-': logger.critical(' git am %s', am_filename) am_mbx.close() if cmdargs.outdir == '-': logger.info('---') with open(am_filename, 'rb') as fh: shutil.copyfileobj(fh, sys.stdout.buffer) os.unlink(am_filename) thanks_record_am(lser, cherrypick=cherrypick)
def create_attestation(cmdargs): attlines = list() subject = 'Patch attestation' for patchfile in cmdargs.patchfile: with open(patchfile, 'r', encoding='utf-8') as fh: content = fh.read() if content.find('From') != 0: logger.info('SKIP | %s', os.path.basename(patchfile)) continue msg = email.message_from_string(content) lmsg = b4.LoreMessage(msg) lmsg.load_hashes() att = lmsg.attestation if att is None: logger.info('SKIP | %s', os.path.basename(patchfile)) # See if it's a cover letter if lmsg.counters_inferred or lmsg.counter > 0: # No continue newprefs = list() for prefix in lmsg.lsubject.prefixes: if prefix.lower() == 'patch': newprefs.append('PSIGN') elif prefix == '%s/%s' % (lmsg.counter, lmsg.expected): newprefs.append('X/%s' % lmsg.expected) else: newprefs.append(prefix) subject = '[%s] %s' % (' '.join(newprefs), lmsg.subject) continue logger.info('HASH | %s', os.path.basename(patchfile)) attlines.append('%s:' % att.attid) attlines.append(' i: %s' % att.i) attlines.append(' m: %s' % att.m) attlines.append(' p: %s' % att.p) payload = '\n'.join(attlines) usercfg = b4.get_user_config() gpgargs = list() if 'signingkey' in usercfg: gpgargs += ['-u', usercfg['signingkey']] gpgargs += ['--clearsign', '--comment', 'att-fmt-ver: %s' % b4.ATTESTATION_FORMAT_VER, '--comment', 'att-hash: sha256', ] ecode, signed = b4.gpg_run_command(gpgargs, stdin=payload.encode('utf-8')) if ecode > 0: config = b4.get_main_config() logger.critical('ERROR: Unable to sign using %s', config['gpgbin']) sys.exit(1) att_msg = email.message.EmailMessage() att_msg.set_payload(signed.encode('utf-8')) sender = cmdargs.sender if '>' not in sender: sender = '<%s>' % sender att_msg['From'] = sender att_msg['To'] = '<*****@*****.**>' att_msg['Message-Id'] = email.utils.make_msgid(domain='kernel.org') att_msg['Subject'] = subject logger.info('---') if not cmdargs.nosubmit: # Try to deliver it via mail.kernel.org try: mailserver = smtplib.SMTP('mail.kernel.org', 587) # identify ourselves to smtp gmail client mailserver.ehlo() # secure our email with tls encryption mailserver.starttls() # re-identify ourselves as an encrypted connection mailserver.ehlo() logger.info('Delivering via mail.kernel.org') mailserver.sendmail('*****@*****.**', '*****@*****.**', att_msg.as_string()) mailserver.quit() sys.exit(0) except Exception as ex: logger.info('Could not deliver: %s', ex) # Future iterations will also be able to submit this to a RESTful URL # at git.kernel.org, in order not to depend on avaialbility of SMTP gateways with open(cmdargs.output, 'wb') as fh: fh.write(att_msg.as_bytes()) logger.info('Wrote %s', cmdargs.output) logger.info('You can send it using:') logger.info(' sendmail -oi [email protected] < %s', cmdargs.output) logger.info(' mutt -H %s', cmdargs.output)
def verify_attestation(cmdargs): config = b4.get_main_config() if cmdargs.tofu: config['attestation-trust-model'] = 'tofu' exact_from_match = True if cmdargs.ignorefrom: exact_from_match = False mbx = mailbox.mbox(cmdargs.mbox[0]) if cmdargs.attfile: b4.LoreAttestationDocument.load_from_file(cmdargs.attfile) eligible = list() for msg in mbx: lmsg = b4.LoreMessage(msg) if lmsg.has_diff: eligible.append(lmsg) continue # See if body has "att-fmt-ver if re.search(r'^Comment: att-fmt-ver:', lmsg.body, re.I | re.M): logger.debug('Found attestation message') b4.LoreAttestationDocument.load_from_string(lmsg.msgid, lmsg.body) logger.debug('SKIP | %s', msg['Subject']) if not len(eligible): logger.error('No patches found in %s', cmdargs.mbox[0]) sys.exit(1) logger.info('---') attrailers = set() ecode = 1 if config['attestation-checkmarks'] == 'fancy': attpass = b4.PASS_FANCY attfail = b4.FAIL_FANCY else: attpass = b4.PASS_SIMPLE attfail = b4.FAIL_SIMPLE for lmsg in eligible: attdoc = lmsg.get_attestation(lore_lookup=True, exact_from_match=exact_from_match) if not attdoc: logger.critical('%s %s', attfail, lmsg.full_subject) if not cmdargs.nofast: logger.critical('Aborting due to failure.') ecode = 1 break else: ecode = 128 continue if ecode != 128: ecode = 0 logger.critical('%s %s', attpass, lmsg.full_subject) attrailers.add(attdoc.attestor.get_trailer(lmsg.fromemail)) logger.critical('---') if ecode > 0: logger.critical('Attestation verification failed.') errors = set() for attdoc in b4.ATTESTATIONS: errors.update(attdoc.errors) if len(errors): logger.critical('---') logger.critical('The validation process reported the following errors:') for error in errors: logger.critical(' %s %s', attfail, error) else: logger.critical('All patches passed attestation:') for attrailer in attrailers: logger.critical(' %s %s', attpass, attrailer) sys.exit(ecode)