def check_build_method(self, write_file): """Check the output from fetching using the BUILD method Args: write_file (bool): True to write the output file when 'make' is called Returns: tuple: str: Filename of written file (or missing 'make' output) str: Contents of stdout """ def fake_run(*cmd): if cmd[0] == 'make': # See Bintool.build_from_git() tmpdir = cmd[2] self.fname = os.path.join(tmpdir, 'pathname') if write_file: tools.WriteFile(self.fname, b'hello') btest = Bintool.create('_testing') col = terminal.Color() self.fname = None with unittest.mock.patch.object(bintool, 'DOWNLOAD_DESTDIR', self._indir): with unittest.mock.patch.object(tools, 'Run', side_effect=fake_run): with test_util.capture_sys_output() as (stdout, _): btest.fetch_tool(bintool.FETCH_BUILD, col, False) fname = os.path.join(self._indir, '_testing') return fname if write_file else self.fname, stdout.getvalue()
def test_fetch_present(self): """Test fetching of a tool""" btest = Bintool.create('_testing') btest.present = True col = terminal.Color() self.assertEqual(bintool.PRESENT, btest.fetch_tool(bintool.FETCH_ANY, col, True))
def show_responses(rtags, indent, is_new): """Show rtags collected Args: rtags (dict): review tags to show key: Response tag (e.g. 'Reviewed-by') value: Set of people who gave that response, each a name/email string indent (str): Indentation string to write before each line is_new (bool): True if this output should be highlighted Returns: int: Number of review tags displayed """ col = terminal.Color() count = 0 for tag in sorted(rtags.keys()): people = rtags[tag] for who in sorted(people): terminal.Print(indent + '%s %s: ' % ('+' if is_new else ' ', tag), newline=False, colour=col.GREEN, bright=is_new) terminal.Print(who, colour=col.WHITE, bright=is_new) count += 1 return count
def check_suppress_cc_config(): """Check if sendemail.suppresscc is configured correctly. Returns: True if the option is configured correctly, False otherwise. """ suppresscc = command.output_one_line('git', 'config', 'sendemail.suppresscc', raise_on_error=False) # Other settings should be fine. if suppresscc == 'all' or suppresscc == 'cccmd': col = terminal.Color() print((col.build(col.RED, "error") + ": git config sendemail.suppresscc set to %s\n" % (suppresscc)) + " patman needs --cc-cmd to be run to set the cc list.\n" + " Please run:\n" + " git config --unset sendemail.suppresscc\n" + " Or read the man page:\n" + " git send-email --help\n" + " and set an option that runs --cc-cmd\n") return False return True
def CheckPatches(verbose, args): '''Run the checkpatch.pl script on each patch''' error_count, warning_count, check_count = 0, 0, 0 col = terminal.Color() for fname in args: result = CheckPatch(fname, verbose) if not result.ok: error_count += result.errors warning_count += result.warnings check_count += result.checks print('%d errors, %d warnings, %d checks for %s:' % (result.errors, result.warnings, result.checks, col.Color(col.BLUE, fname))) if (len(result.problems) != result.errors + result.warnings + result.checks): print("Internal error: some problems lost") for item in result.problems: sys.stderr.write( GetWarningMsg(col, item.get('type', '<unknown>'), item.get('file', '<unknown>'), item.get('line', 0), item.get('msg', 'message'))) print #print(stdout) if error_count or warning_count or check_count: str = 'checkpatch.pl found %d error(s), %d warning(s), %d checks(s)' color = col.GREEN if warning_count: color = col.YELLOW if error_count: color = col.RED print(col.Color(color, str % (error_count, warning_count, check_count))) return False return True
def test_no_fetch(self): """Test fetching when there is no method""" btest = Bintool.create('_testing') btest.disable = True col = terminal.Color() with test_util.capture_sys_output() as _: result = btest.fetch_tool(bintool.FETCH_BIN, col, False) self.assertEqual(bintool.FAIL, result)
def test_install(self): """Test fetching using the install method""" btest = Bintool.create('_testing') btest.install = True col = terminal.Color() with unittest.mock.patch.object(tools, 'Run', return_value=None): with test_util.capture_sys_output() as _: result = btest.fetch_tool(bintool.FETCH_BIN, col, False) self.assertEqual(bintool.FETCHED, result)
def MakeCcFile(self, process_tags, cover_fname, raise_on_error, add_maintainers, limit): """Make a cc file for us to use for per-commit Cc automation Also stores in self._generated_cc to make ShowActions() faster. Args: process_tags: Process tags as if they were aliases cover_fname: If non-None the name of the cover letter. raise_on_error: True to raise an error when an alias fails to match, False to just print a message. add_maintainers: Either: True/False to call the get_maintainers to CC maintainers List of maintainers to include (for testing) limit: Limit the length of the Cc list (None if no limit) Return: Filename of temp file created """ col = terminal.Color() # Look for commit tags (of the form 'xxx:' at the start of the subject) fname = '/tmp/patman.%d' % os.getpid() fd = open(fname, 'w', encoding='utf-8') all_ccs = [] for commit in self.commits: cc = [] if process_tags: cc += gitutil.BuildEmailList(commit.tags, raise_on_error=raise_on_error) cc += gitutil.BuildEmailList(commit.cc_list, raise_on_error=raise_on_error) if type(add_maintainers) == type(cc): cc += add_maintainers elif add_maintainers: dir_list = [os.path.join(gitutil.GetTopLevel(), 'scripts')] cc += get_maintainer.GetMaintainer(dir_list, commit.patch) for x in set(cc) & set(settings.bounces): print(col.Color(col.YELLOW, 'Skipping "%s"' % x)) cc = set(cc) - set(settings.bounces) cc = [tools.FromUnicode(m) for m in cc] if limit is not None: cc = cc[:limit] all_ccs += cc print(commit.patch, '\0'.join(sorted(set(cc))), file=fd) self._generated_cc[commit.patch] = cc if cover_fname: cover_cc = gitutil.BuildEmailList(self.get('cover_cc', '')) cover_cc = [tools.FromUnicode(m) for m in cover_cc] cover_cc = list(set(cover_cc + all_ccs)) if limit is not None: cover_cc = cover_cc[:limit] cc_list = '\0'.join([tools.ToUnicode(x) for x in sorted(cover_cc)]) print(cover_fname, cc_list, file=fd) fd.close() return fname
def List(self): """List out the selected toolchains for each architecture""" col = terminal.Color() print(col.Color(col.BLUE, 'List of available toolchains (%d):' % len(self.toolchains))) if len(self.toolchains): for key, value in sorted(self.toolchains.items()): print('%-10s: %s' % (key, value.gcc)) else: print('None')
def test_branch(self): """Test creating patches from a branch""" repo = self.make_git_tree() target = repo.lookup_reference('refs/heads/first') # pylint doesn't seem to find this # pylint: disable=E1101 self.repo.checkout(target, strategy=pygit2.GIT_CHECKOUT_FORCE) control.setup() try: orig_dir = os.getcwd() os.chdir(self.gitdir) # Check that it can detect the current branch self.assertEqual(2, gitutil.count_commits_to_branch(None)) col = terminal.Color() with capture_sys_output() as _: _, cover_fname, patch_files = control.prepare_patches( col, branch=None, count=-1, start=0, end=0, ignore_binary=False, signoff=True) self.assertIsNone(cover_fname) self.assertEqual(2, len(patch_files)) # Check that it can detect a different branch self.assertEqual(3, gitutil.count_commits_to_branch('second')) with capture_sys_output() as _: _, cover_fname, patch_files = control.prepare_patches( col, branch='second', count=-1, start=0, end=0, ignore_binary=False, signoff=True) self.assertIsNotNone(cover_fname) self.assertEqual(3, len(patch_files)) # Check that it can skip patches at the end with capture_sys_output() as _: _, cover_fname, patch_files = control.prepare_patches( col, branch='second', count=-1, start=0, end=1, ignore_binary=False, signoff=True) self.assertIsNotNone(cover_fname) self.assertEqual(2, len(patch_files)) finally: os.chdir(orig_dir)
def FetchAndInstall(self, arch): """Fetch and install a new toolchain arch: Architecture to fetch, or 'list' to list """ # Fist get the URL for this architecture col = terminal.Color() print(col.build(col.BLUE, "Downloading toolchain for arch '%s'" % arch)) url = self.LocateArchUrl(arch) if not url: print(("Cannot find toolchain for arch '%s' - use 'list' to list" % arch)) return 2 home = os.environ['HOME'] dest = os.path.join(home, '.buildman-toolchains') if not os.path.exists(dest): os.mkdir(dest) # Download the tar file for this toolchain and unpack it tarfile, tmpdir = tools.download(url, '.buildman') if not tarfile: return 1 print(col.build(col.GREEN, 'Unpacking to: %s' % dest), end=' ') sys.stdout.flush() path = self.Unpack(tarfile, dest) os.remove(tarfile) os.rmdir(tmpdir) print() # Check that the toolchain works print(col.build(col.GREEN, 'Testing')) dirpath = os.path.join(dest, path) compiler_fname_list = self.ScanPath(dirpath, True) if not compiler_fname_list: print('Could not locate C compiler - fetch failed.') return 1 if len(compiler_fname_list) != 1: print( col.build( col.RED, 'Warning, ambiguous toolchains: %s' % ', '.join(compiler_fname_list))) toolchain = Toolchain(compiler_fname_list[0], True, True) # Make sure that it will be found by buildman if not self.TestSettingsHasPath(dirpath): print(("Adding 'download' to config file '%s'" % bsettings.config_fname)) bsettings.SetItem('toolchain', 'download', '%s/*/*' % dest) return 0
def fetch_tools(method, names_to_fetch): """Fetch bintools from a suitable place This fetches or builds the requested bintools so that they can be used by binman Args: names_to_fetch (list of str): names of bintools to fetch Returns: True on success, False on failure """ def show_status(color, prompt, names): print( col.build( color, f'{prompt}:%s{len(names):2}: %s' % (' ' * (16 - len(prompt)), ' '.join(names)))) col = terminal.Color() skip_present = False name_list = names_to_fetch if len(names_to_fetch) == 1 and names_to_fetch[0] in [ 'all', 'missing' ]: name_list = Bintool.get_tool_list() if names_to_fetch[0] == 'missing': skip_present = True print( col.build(col.YELLOW, 'Fetching tools: %s' % ' '.join(name_list))) status = collections.defaultdict(list) for name in name_list: btool = Bintool.create(name) result = btool.fetch_tool(method, col, skip_present) status[result].append(name) if result == FAIL: if method == FETCH_ANY: print('- failed to fetch with all methods') else: print(f"- method '{FETCH_NAMES[method]}' is not supported") if len(name_list) > 1: if skip_present: show_status(col.GREEN, 'Already present', status[PRESENT]) show_status(col.GREEN, 'Tools fetched', status[FETCHED]) if status[FAIL]: show_status(col.RED, 'Failures', status[FAIL]) return not status[FAIL]
def Init(_verbose=WARNING, stdout=sys.stdout): """Initialize a new output object. Args: verbose: Verbosity level (0-4). stdout: File to use for stdout. """ global verbose, _progress, _color, _stdout, stdout_is_tty verbose = _verbose _progress = '' # Our last progress message _color = terminal.Color() _stdout = stdout # TODO(sjg): Move this into Chromite libraries when we have them stdout_is_tty = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty()
def ShowActions(self, args, cmd, process_tags): """Show what actions we will/would perform Args: args: List of patch files we created cmd: The git command we would have run process_tags: Process tags as if they were aliases """ to_set = set(gitutil.BuildEmailList(self.to)) cc_set = set(gitutil.BuildEmailList(self.cc)) col = terminal.Color() print('Dry run, so not doing much. But I would do this:') print() print('Send a total of %d patch%s with %scover letter.' % (len(args), '' if len(args) == 1 else 'es', self.get('cover') and 'a ' or 'no ')) # TODO: Colour the patches according to whether they passed checks for upto in range(len(args)): commit = self.commits[upto] print(col.Color(col.GREEN, ' %s' % args[upto])) cc_list = list(self._generated_cc[commit.patch]) for email in sorted(set(cc_list) - to_set - cc_set): if email == None: email = col.Color(col.YELLOW, "<alias '%s' not found>" % tag) if email: print(' Cc: ', email) print for item in sorted(to_set): print('To:\t ', item) for item in sorted(cc_set - to_set): print('Cc:\t ', item) print('Version: ', self.get('version')) print('Prefix:\t ', self.get('prefix')) print('Postfix:\t ', self.get('postfix')) if self.cover: print('Cover: %d lines' % len(self.cover)) cover_cc = gitutil.BuildEmailList(self.get('cover_cc', '')) all_ccs = itertools.chain(cover_cc, *self._generated_cc.values()) for email in sorted(set(all_ccs) - to_set - cc_set): print(' Cc: ', email) if cmd: print('Git command: %s' % cmd)
def ShowActions(series, why_selected, boards_selected, builder, options, board_warnings): """Display a list of actions that we would take, if not a dry run. Args: series: Series object why_selected: Dictionary where each key is a buildman argument provided by the user, and the value is the list of boards brought in by that argument. For example, 'arm' might bring in 400 boards, so in this case the key would be 'arm' and the value would be a list of board names. boards_selected: Dict of selected boards, key is target name, value is Board object builder: The builder that will be used to build the commits options: Command line options object board_warnings: List of warnings obtained from board selected """ col = terminal.Color() print('Dry run, so not doing much. But I would do this:') print() if series: commits = series.commits else: commits = None print(GetActionSummary(False, commits, boards_selected, options)) print('Build directory: %s' % builder.base_dir) if commits: for upto in range(0, len(series.commits), options.step): commit = series.commits[upto] print(' ', col.Color(col.YELLOW, commit.hash[:8], bright=False), end=' ') print(commit.subject) print() for arg in why_selected: if arg != 'all': print(arg, ': %d boards' % len(why_selected[arg])) if options.verbose: print(' %s' % ' '.join(why_selected[arg])) print(('Total boards to build for each commit: %d\n' % len(why_selected['all']))) if board_warnings: for warning in board_warnings: print(col.Color(col.YELLOW, warning))
def check_fetch_url(cls, fake_download, method): """Check the output from fetching a tool Args: fake_download (function): Function to call instead of tools.Download() method (bintool.FETCH_...: Fetch method to use Returns: str: Contents of stdout """ btest = Bintool.create('_testing') col = terminal.Color() with unittest.mock.patch.object(tools, 'Download', side_effect=fake_download): with test_util.capture_sys_output() as (stdout, _): btest.fetch_tool(method, col, False) return stdout.getvalue()
def send(args): """Create, check and send patches by email Args: args (argparse.Namespace): Arguments to patman """ setup() col = terminal.Color() series, cover_fname, patch_files = prepare_patches(col, args.branch, args.count, args.start, args.end, args.ignore_binary) ok = check_patches(series, patch_files, args.check_patch, args.verbose) ok = ok and gitutil.CheckSuppressCCConfig() its_a_go = ok or args.ignore_errors email_patches(col, series, cover_fname, patch_files, args.process_tags, its_a_go, args.ignore_bad_tags, args.add_maintainers, args.limit, args.dry_run, args.in_reply_to, args.thread, args.smtp_server)
def DoChecks(self): """Check that each version has a change log Print an error if something is wrong. """ col = terminal.Color() if self.get('version'): changes_copy = dict(self.changes) for version in range(1, int(self.version) + 1): if self.changes.get(version): del changes_copy[version] else: if version > 1: str = 'Change log missing for v%d' % version print(col.Color(col.RED, str)) for version in changes_copy: str = 'Change log for unknown version v%d' % version print(col.Color(col.RED, str)) elif self.changes: str = 'Change log exists, but no version is set' print(col.Color(col.RED, str))
def setUp(self): # Set up commits to build self.commits = [] sequence = 0 for commit_info in commits: comm = commit.Commit(commit_info[0]) comm.subject = commit_info[1] comm.return_code = commit_info[2] comm.error_list = commit_info[3] if sequence < 6: comm.error_list += [migration] comm.sequence = sequence sequence += 1 self.commits.append(comm) # Set up boards to build self.boards = board.Boards() for brd in boards: self.boards.AddBoard(board.Board(*brd)) self.boards.SelectBoards([]) # Add some test settings bsettings.Setup(None) bsettings.AddFile(settings_data) # Set up the toolchains self.toolchains = toolchain.Toolchains() self.toolchains.Add('arm-linux-gcc', test=False) self.toolchains.Add('sparc-linux-gcc', test=False) self.toolchains.Add('powerpc-linux-gcc', test=False) self.toolchains.Add('gcc', test=False) # Avoid sending any output terminal.SetPrintTestMode() self._col = terminal.Color() self.base_dir = tempfile.mkdtemp() if not os.path.isdir(self.base_dir): os.mkdir(self.base_dir)
pager = os.getenv('PAGER') if not pager: pager = 'more' fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), 'README') command.Run(pager, fname) # Process commits, produce patches files, check them, email them else: gitutil.Setup() if options.count == -1: # Work out how many patches to send if we can options.count = gitutil.CountCommitsToBranch() - options.start col = terminal.Color() if not options.count: str = 'No commits found to process - please use -c flag' sys.exit(col.Color(col.RED, str)) # Read the metadata from the commits if options.count: series = patchstream.GetMetaData(options.start, options.count) cover_fname, args = gitutil.CreatePatches(options.start, options.count, series) # Fix up the patch files to our liking, and insert the cover letter patchstream.FixPatches(series, args) if cover_fname and series.get('cover'): patchstream.InsertCoverLetter(cover_fname, series, options.count)
def _CheckOutput(self, lines, list_error_boards=False, filter_dtb_warnings=False, filter_migration_warnings=False): """Check for expected output from the build summary Args: lines: Iterator containing the lines returned from the summary list_error_boards: Adjust the check for output produced with the --list-error-boards flag filter_dtb_warnings: Adjust the check for output produced with the --filter-dtb-warnings flag """ def add_line_prefix(prefix, boards, error_str, colour): """Add a prefix to each line of a string The training \n in error_str is removed before processing Args: prefix: String prefix to add error_str: Error string containing the lines colour: Expected colour for the line. Note that the board list, if present, always appears in magenta Returns: New string where each line has the prefix added """ lines = error_str.strip().splitlines() new_lines = [] for line in lines: if boards: expect = self._col.Color(colour, prefix + '(') expect += self._col.Color(self._col.MAGENTA, boards, bright=False) expect += self._col.Color(colour, ') %s' % line) else: expect = self._col.Color(colour, prefix + line) new_lines.append(expect) return '\n'.join(new_lines) col = terminal.Color() boards01234 = ('board0 board1 board2 board3 board4' if list_error_boards else '') boards1234 = 'board1 board2 board3 board4' if list_error_boards else '' boards234 = 'board2 board3 board4' if list_error_boards else '' boards34 = 'board3 board4' if list_error_boards else '' boards4 = 'board4' if list_error_boards else '' # Upstream commit: migration warnings only self.assertEqual(next(lines).text, '01: %s' % commits[0][1]) if not filter_migration_warnings: self.assertSummary(next(lines).text, 'arm', 'w+', ['board0', 'board1'], outcome=OUTCOME_WARN) self.assertSummary(next(lines).text, 'powerpc', 'w+', ['board2', 'board3'], outcome=OUTCOME_WARN) self.assertSummary(next(lines).text, 'sandbox', 'w+', ['board4'], outcome=OUTCOME_WARN) self.assertEqual( next(lines).text, add_line_prefix('+', boards01234, migration, col.RED)) # Second commit: all archs should fail with warnings self.assertEqual(next(lines).text, '02: %s' % commits[1][1]) if filter_migration_warnings: self.assertSummary(next(lines).text, 'arm', 'w+', ['board1'], outcome=OUTCOME_WARN) self.assertSummary(next(lines).text, 'powerpc', 'w+', ['board2', 'board3'], outcome=OUTCOME_WARN) self.assertSummary(next(lines).text, 'sandbox', 'w+', ['board4'], outcome=OUTCOME_WARN) # Second commit: The warnings should be listed self.assertEqual( next(lines).text, add_line_prefix('w+', boards1234, errors[0], col.YELLOW)) # Third commit: Still fails self.assertEqual(next(lines).text, '03: %s' % commits[2][1]) if filter_migration_warnings: self.assertSummary(next(lines).text, 'arm', '', ['board1'], outcome=OUTCOME_OK) self.assertSummary( next(lines).text, 'powerpc', '+', ['board2', 'board3']) self.assertSummary(next(lines).text, 'sandbox', '+', ['board4']) # Expect a compiler error self.assertEqual( next(lines).text, add_line_prefix('+', boards234, errors[1], col.RED)) # Fourth commit: Compile errors are fixed, just have warning for board3 self.assertEqual(next(lines).text, '04: %s' % commits[3][1]) if filter_migration_warnings: expect = '%10s: ' % 'powerpc' expect += ' ' + col.Color(col.GREEN, '') expect += ' ' expect += col.Color(col.GREEN, ' %s' % 'board2') expect += ' ' + col.Color(col.YELLOW, 'w+') expect += ' ' expect += col.Color(col.YELLOW, ' %s' % 'board3') self.assertEqual(next(lines).text, expect) else: self.assertSummary(next(lines).text, 'powerpc', 'w+', ['board2', 'board3'], outcome=OUTCOME_WARN) self.assertSummary(next(lines).text, 'sandbox', 'w+', ['board4'], outcome=OUTCOME_WARN) # Compile error fixed self.assertEqual( next(lines).text, add_line_prefix('-', boards234, errors[1], col.GREEN)) if not filter_dtb_warnings: self.assertEqual( next(lines).text, add_line_prefix('w+', boards34, errors[2], col.YELLOW)) # Fifth commit self.assertEqual(next(lines).text, '05: %s' % commits[4][1]) if filter_migration_warnings: self.assertSummary(next(lines).text, 'powerpc', '', ['board3'], outcome=OUTCOME_OK) self.assertSummary(next(lines).text, 'sandbox', '+', ['board4']) # The second line of errors[3] is a duplicate, so buildman will drop it expect = errors[3].rstrip().split('\n') expect = [expect[0]] + expect[2:] expect = '\n'.join(expect) self.assertEqual( next(lines).text, add_line_prefix('+', boards4, expect, col.RED)) if not filter_dtb_warnings: self.assertEqual( next(lines).text, add_line_prefix('w-', boards34, errors[2], col.CYAN)) # Sixth commit self.assertEqual(next(lines).text, '06: %s' % commits[5][1]) if filter_migration_warnings: self.assertSummary(next(lines).text, 'sandbox', '', ['board4'], outcome=OUTCOME_OK) else: self.assertSummary(next(lines).text, 'sandbox', 'w+', ['board4'], outcome=OUTCOME_WARN) # The second line of errors[3] is a duplicate, so buildman will drop it expect = errors[3].rstrip().split('\n') expect = [expect[0]] + expect[2:] expect = '\n'.join(expect) self.assertEqual( next(lines).text, add_line_prefix('-', boards4, expect, col.GREEN)) self.assertEqual( next(lines).text, add_line_prefix('w-', boards4, errors[0], col.CYAN)) # Seventh commit self.assertEqual(next(lines).text, '07: %s' % commits[6][1]) if filter_migration_warnings: self.assertSummary(next(lines).text, 'sandbox', '+', ['board4']) else: self.assertSummary(next(lines).text, 'arm', '', ['board0', 'board1'], outcome=OUTCOME_OK) self.assertSummary(next(lines).text, 'powerpc', '', ['board2', 'board3'], outcome=OUTCOME_OK) self.assertSummary(next(lines).text, 'sandbox', '+', ['board4']) # Pick out the correct error lines expect_str = errors[4].rstrip().replace('%(basedir)s', '').split('\n') expect = expect_str[3:8] + [expect_str[-1]] expect = '\n'.join(expect) if not filter_migration_warnings: self.assertEqual( next(lines).text, add_line_prefix('-', boards01234, migration, col.GREEN)) self.assertEqual( next(lines).text, add_line_prefix('+', boards4, expect, col.RED)) # Now the warnings lines expect = [expect_str[0]] + expect_str[10:12] + [expect_str[9]] expect = '\n'.join(expect) self.assertEqual( next(lines).text, add_line_prefix('w+', boards4, expect, col.YELLOW))
def LookupEmail(lookup_name, alias=None, raise_on_error=True, level=0): """If an email address is an alias, look it up and return the full name TODO: Why not just use git's own alias feature? Args: lookup_name: Alias or email address to look up alias: Dictionary containing aliases (None to use settings default) raise_on_error: True to raise an error when an alias fails to match, False to just print a message. Returns: tuple: list containing a list of email addresses Raises: OSError if a recursive alias reference was found ValueError if an alias was not found >>> alias = {} >>> alias['fred'] = ['*****@*****.**'] >>> alias['john'] = ['*****@*****.**'] >>> alias['mary'] = ['*****@*****.**'] >>> alias['boys'] = ['fred', ' john', '*****@*****.**'] >>> alias['all'] = ['fred ', 'john', ' mary '] >>> alias['loop'] = ['other', 'john', ' mary '] >>> alias['other'] = ['loop', 'john', ' mary '] >>> LookupEmail('mary', alias) ['*****@*****.**'] >>> LookupEmail('*****@*****.**', alias) ['*****@*****.**'] >>> LookupEmail('boys', alias) ['*****@*****.**', '*****@*****.**'] >>> LookupEmail('all', alias) ['*****@*****.**', '*****@*****.**', '*****@*****.**'] >>> LookupEmail('odd', alias) Traceback (most recent call last): ... ValueError: Alias 'odd' not found >>> LookupEmail('loop', alias) Traceback (most recent call last): ... OSError: Recursive email alias at 'other' >>> LookupEmail('odd', alias, raise_on_error=False) Alias 'odd' not found [] >>> # In this case the loop part will effectively be ignored. >>> LookupEmail('loop', alias, raise_on_error=False) Recursive email alias at 'other' Recursive email alias at 'john' Recursive email alias at 'mary' ['*****@*****.**', '*****@*****.**'] """ if not alias: alias = settings.alias lookup_name = lookup_name.strip() if '@' in lookup_name: # Perhaps a real email address return [lookup_name] lookup_name = lookup_name.lower() col = terminal.Color() out_list = [] if level > 10: msg = "Recursive email alias at '%s'" % lookup_name if raise_on_error: raise OSError(msg) else: print(col.Color(col.RED, msg)) return out_list if lookup_name: if not lookup_name in alias: msg = "Alias '%s' not found" % lookup_name if raise_on_error: raise ValueError(msg) else: print(col.Color(col.RED, msg)) return out_list for item in alias[lookup_name]: todo = LookupEmail(item, alias, raise_on_error, level + 1) for new_item in todo: if not new_item in out_list: out_list.append(new_item) #print("No match for alias '%s'" % lookup_name) return out_list
def check_patchwork_status(series, series_id, branch, dest_branch, force, show_comments, url, rest_api=call_rest_api, test_repo=None): """Check the status of a series on Patchwork This finds review tags and comments for a series in Patchwork, displaying them to show what is new compared to the local series. Args: series (Series): Series object for the existing branch series_id (str): Patch series ID number branch (str): Existing branch to update, or None dest_branch (str): Name of new branch to create, or None force (bool): True to force overwriting dest_branch if it exists show_comments (bool): True to show the comments on each patch url (str): URL of patchwork server, e.g. 'https://patchwork.ozlabs.org' rest_api (function): API function to call to access Patchwork, for testing test_repo (pygit2.Repository): Repo to use (use None unless testing) """ patches = collect_patches(series, series_id, url, rest_api) col = terminal.Color() count = len(series.commits) new_rtag_list = [None] * count review_list = [None] * count patch_for_commit, _, warnings = compare_with_series(series, patches) for warn in warnings: tout.Warning(warn) patch_list = [patch_for_commit.get(c) for c in range(len(series.commits))] with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor: futures = executor.map(find_new_responses, repeat(new_rtag_list), repeat(review_list), range(count), series.commits, patch_list, repeat(url), repeat(rest_api)) for fresponse in futures: if fresponse: raise fresponse.exception() num_to_add = 0 for seq, cmt in enumerate(series.commits): patch = patch_for_commit.get(seq) if not patch: continue terminal.Print('%3d %s' % (patch.seq, patch.subject[:50]), colour=col.BLUE) cmt = series.commits[seq] base_rtags = cmt.rtags new_rtags = new_rtag_list[seq] indent = ' ' * 2 show_responses(base_rtags, indent, False) num_to_add += show_responses(new_rtags, indent, True) if show_comments: for review in review_list[seq]: terminal.Print('Review: %s' % review.meta, colour=col.RED) for snippet in review.snippets: for line in snippet: quoted = line.startswith('>') terminal.Print(' %s' % line, colour=col.MAGENTA if quoted else None) terminal.Print() terminal.Print( "%d new response%s available in patchwork%s" % (num_to_add, 's' if num_to_add != 1 else '', '' if dest_branch else ' (use -d to write them to a new branch)')) if dest_branch: num_added = create_branch(series, new_rtag_list, branch, dest_branch, force, test_repo) terminal.Print( "%d response%s added from patchwork into new branch '%s'" % (num_added, 's' if num_added != 1 else '', dest_branch))
def test_review_snippets(self): """Test showing of review snippets""" def _to_submitter(who): m_who = re.match('(.*) <(.*)>', who) return {'name': m_who.group(1), 'email': m_who.group(2)} commit1 = Commit('abcd') commit1.subject = 'Subject 1' commit2 = Commit('ef12') commit2.subject = 'Subject 2' patch1 = status.Patch('1') patch1.parse_subject('[1/2] Subject 1') patch1.name = patch1.raw_subject patch1.content = 'This is my patch content' comment1a = { 'submitter': _to_submitter(self.joe), 'content': '''Hi Fred, On some date Fred wrote: > diff --git a/file.c b/file.c > Some code > and more code Here is my comment above the above... Reviewed-by: %s ''' % self.joe } patch1.comments = [comment1a] patch2 = status.Patch('2') patch2.parse_subject('[2/2] Subject 2') patch2.name = patch2.raw_subject patch2.content = 'Some other patch content' comment2a = { 'content': 'Reviewed-by: %s\nTested-by: %s\n' % (self.mary, self.leb) } comment2b = { 'submitter': _to_submitter(self.fred), 'content': '''Hi Fred, On some date Fred wrote: > diff --git a/tools/patman/commit.py b/tools/patman/commit.py > @@ -41,6 +41,9 @@ class Commit: > self.rtags = collections.defaultdict(set) > self.warn = [] > > + def __str__(self): > + return self.subject > + > def add_change(self, version, info): > """Add a new change line to the change list for a version. > A comment Reviewed-by: %s ''' % self.fred } patch2.comments = [comment2a, comment2b] # This test works by setting up commits and patch for use by the fake # Rest API function _fake_patchwork2(). It calls various functions in # the status module after setting up tags in the commits, checking that # things behaves as expected self.commits = [commit1, commit2] self.patches = [patch1, patch2] # Check that the output patches expectations: # 1 Subject 1 # Reviewed-by: Joe Bloggs <*****@*****.**> # 2 Subject 2 # Tested-by: Lord Edmund Blackaddër <*****@*****.**> # Reviewed-by: Fred Bloggs <*****@*****.**> # + Reviewed-by: Mary Bloggs <*****@*****.**> # 1 new response available in patchwork series = Series() series.commits = [commit1, commit2] terminal.set_print_test_mode() status.check_patchwork_status(series, '1234', None, None, False, True, None, self._fake_patchwork2) lines = iter(terminal.get_print_test_lines()) col = terminal.Color() self.assertEqual(terminal.PrintLine(' 1 Subject 1', col.BLUE), next(lines)) self.assertEqual( terminal.PrintLine(' + Reviewed-by: ', col.GREEN, newline=False), next(lines)) self.assertEqual(terminal.PrintLine(self.joe, col.WHITE), next(lines)) self.assertEqual(terminal.PrintLine('Review: %s' % self.joe, col.RED), next(lines)) self.assertEqual(terminal.PrintLine(' Hi Fred,', None), next(lines)) self.assertEqual(terminal.PrintLine('', None), next(lines)) self.assertEqual(terminal.PrintLine(' > File: file.c', col.MAGENTA), next(lines)) self.assertEqual(terminal.PrintLine(' > Some code', col.MAGENTA), next(lines)) self.assertEqual( terminal.PrintLine(' > and more code', col.MAGENTA), next(lines)) self.assertEqual( terminal.PrintLine(' Here is my comment above the above...', None), next(lines)) self.assertEqual(terminal.PrintLine('', None), next(lines)) self.assertEqual(terminal.PrintLine(' 2 Subject 2', col.BLUE), next(lines)) self.assertEqual( terminal.PrintLine(' + Reviewed-by: ', col.GREEN, newline=False), next(lines)) self.assertEqual(terminal.PrintLine(self.fred, col.WHITE), next(lines)) self.assertEqual( terminal.PrintLine(' + Reviewed-by: ', col.GREEN, newline=False), next(lines)) self.assertEqual(terminal.PrintLine(self.mary, col.WHITE), next(lines)) self.assertEqual( terminal.PrintLine(' + Tested-by: ', col.GREEN, newline=False), next(lines)) self.assertEqual(terminal.PrintLine(self.leb, col.WHITE), next(lines)) self.assertEqual(terminal.PrintLine('Review: %s' % self.fred, col.RED), next(lines)) self.assertEqual(terminal.PrintLine(' Hi Fred,', None), next(lines)) self.assertEqual(terminal.PrintLine('', None), next(lines)) self.assertEqual( terminal.PrintLine(' > File: tools/patman/commit.py', col.MAGENTA), next(lines)) self.assertEqual( terminal.PrintLine(' > Line: 41 / 41: class Commit:', col.MAGENTA), next(lines)) self.assertEqual( terminal.PrintLine(' > + return self.subject', col.MAGENTA), next(lines)) self.assertEqual(terminal.PrintLine(' > +', col.MAGENTA), next(lines)) self.assertEqual( terminal.PrintLine( ' > def add_change(self, version, info):', col.MAGENTA), next(lines)) self.assertEqual( terminal.PrintLine( ' > """Add a new change line to the change list for a version.', col.MAGENTA), next(lines)) self.assertEqual(terminal.PrintLine(' >', col.MAGENTA), next(lines)) self.assertEqual(terminal.PrintLine(' A comment', None), next(lines)) self.assertEqual(terminal.PrintLine('', None), next(lines)) self.assertEqual( terminal.PrintLine( '4 new responses available in patchwork (use -d to write them to a new branch)', None), next(lines))
def test_find_new_responses(self): """Test operation of find_new_responses()""" commit1 = Commit('abcd') commit1.subject = 'Subject 1' commit2 = Commit('ef12') commit2.subject = 'Subject 2' patch1 = status.Patch('1') patch1.parse_subject('[1/2] Subject 1') patch1.name = patch1.raw_subject patch1.content = 'This is my patch content' comment1a = {'content': 'Reviewed-by: %s\n' % self.joe} patch1.comments = [comment1a] patch2 = status.Patch('2') patch2.parse_subject('[2/2] Subject 2') patch2.name = patch2.raw_subject patch2.content = 'Some other patch content' comment2a = { 'content': 'Reviewed-by: %s\nTested-by: %s\n' % (self.mary, self.leb) } comment2b = {'content': 'Reviewed-by: %s' % self.fred} patch2.comments = [comment2a, comment2b] # This test works by setting up commits and patch for use by the fake # Rest API function _fake_patchwork2(). It calls various functions in # the status module after setting up tags in the commits, checking that # things behaves as expected self.commits = [commit1, commit2] self.patches = [patch1, patch2] count = 2 new_rtag_list = [None] * count review_list = [None, None] # Check that the tags are picked up on the first patch status.find_new_responses(new_rtag_list, review_list, 0, commit1, patch1, None, self._fake_patchwork2) self.assertEqual(new_rtag_list[0], {'Reviewed-by': {self.joe}}) # Now the second patch status.find_new_responses(new_rtag_list, review_list, 1, commit2, patch2, None, self._fake_patchwork2) self.assertEqual(new_rtag_list[1], { 'Reviewed-by': {self.mary, self.fred}, 'Tested-by': {self.leb} }) # Now add some tags to the commit, which means they should not appear as # 'new' tags when scanning comments new_rtag_list = [None] * count commit1.rtags = {'Reviewed-by': {self.joe}} status.find_new_responses(new_rtag_list, review_list, 0, commit1, patch1, None, self._fake_patchwork2) self.assertEqual(new_rtag_list[0], {}) # For the second commit, add Ed and Fred, so only Mary should be left commit2.rtags = {'Tested-by': {self.leb}, 'Reviewed-by': {self.fred}} status.find_new_responses(new_rtag_list, review_list, 1, commit2, patch2, None, self._fake_patchwork2) self.assertEqual(new_rtag_list[1], {'Reviewed-by': {self.mary}}) # Check that the output patches expectations: # 1 Subject 1 # Reviewed-by: Joe Bloggs <*****@*****.**> # 2 Subject 2 # Tested-by: Lord Edmund Blackaddër <*****@*****.**> # Reviewed-by: Fred Bloggs <*****@*****.**> # + Reviewed-by: Mary Bloggs <*****@*****.**> # 1 new response available in patchwork series = Series() series.commits = [commit1, commit2] terminal.set_print_test_mode() status.check_patchwork_status(series, '1234', None, None, False, False, None, self._fake_patchwork2) lines = iter(terminal.get_print_test_lines()) col = terminal.Color() self.assertEqual(terminal.PrintLine(' 1 Subject 1', col.BLUE), next(lines)) self.assertEqual( terminal.PrintLine(' Reviewed-by: ', col.GREEN, newline=False, bright=False), next(lines)) self.assertEqual(terminal.PrintLine(self.joe, col.WHITE, bright=False), next(lines)) self.assertEqual(terminal.PrintLine(' 2 Subject 2', col.BLUE), next(lines)) self.assertEqual( terminal.PrintLine(' Reviewed-by: ', col.GREEN, newline=False, bright=False), next(lines)) self.assertEqual( terminal.PrintLine(self.fred, col.WHITE, bright=False), next(lines)) self.assertEqual( terminal.PrintLine(' Tested-by: ', col.GREEN, newline=False, bright=False), next(lines)) self.assertEqual(terminal.PrintLine(self.leb, col.WHITE, bright=False), next(lines)) self.assertEqual( terminal.PrintLine(' + Reviewed-by: ', col.GREEN, newline=False), next(lines)) self.assertEqual(terminal.PrintLine(self.mary, col.WHITE), next(lines)) self.assertEqual( terminal.PrintLine( '1 new response available in patchwork (use -d to write them to a new branch)', None), next(lines))
def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, clean_dir=False): """The main control code for buildman Args: options: Command line options object args: Command line arguments (list of strings) toolchains: Toolchains to use - this should be a Toolchains() object. If None, then it will be created and scanned make_func: Make function to use for the builder. This is called to execute 'make'. If this is None, the normal function will be used, which calls the 'make' tool with suitable arguments. This setting is useful for tests. board: Boards() object to use, containing a list of available boards. If this is None it will be created and scanned. """ global builder if options.full_help: pager = os.getenv('PAGER') if not pager: pager = 'more' fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), 'README') command.Run(pager, fname) return 0 gitutil.Setup() col = terminal.Color() options.git_dir = os.path.join(options.git, '.git') no_toolchains = toolchains is None if no_toolchains: toolchains = toolchain.Toolchains(options.override_toolchain) if options.fetch_arch: if options.fetch_arch == 'list': sorted_list = toolchains.ListArchs() print( col.Color( col.BLUE, 'Available architectures: %s\n' % ' '.join(sorted_list))) return 0 else: fetch_arch = options.fetch_arch if fetch_arch == 'all': fetch_arch = ','.join(toolchains.ListArchs()) print( col.Color(col.CYAN, '\nDownloading toolchains: %s' % fetch_arch)) for arch in fetch_arch.split(','): print() ret = toolchains.FetchAndInstall(arch) if ret: return ret return 0 if no_toolchains: toolchains.GetSettings() toolchains.Scan(options.list_tool_chains and options.verbose) if options.list_tool_chains: toolchains.List() print() return 0 if options.incremental: print( col.Color(col.RED, 'Warning: -I has been removed. See documentation')) if not options.output_dir: if options.work_in_output: sys.exit(col.Color(col.RED, '-w requires that you specify -o')) options.output_dir = '..' # Work out what subset of the boards we are building if not boards: if not os.path.exists(options.output_dir): os.makedirs(options.output_dir) board_file = os.path.join(options.output_dir, 'boards.cfg') genboardscfg = os.path.join(options.git, 'tools/genboardscfg.py') status = subprocess.call([genboardscfg, '-q', '-o', board_file]) if status != 0: sys.exit("Failed to generate boards.cfg") boards = board.Boards() boards.ReadBoards(board_file) exclude = [] if options.exclude: for arg in options.exclude: exclude += arg.split(',') if options.boards: requested_boards = [] for b in options.boards: requested_boards += b.split(',') else: requested_boards = None why_selected, board_warnings = boards.SelectBoards(args, exclude, requested_boards) selected = boards.GetSelected() if not len(selected): sys.exit(col.Color(col.RED, 'No matching boards found')) if options.print_prefix: err = ShowToolchainPrefix(boards, toolchains) if err: sys.exit(col.Color(col.RED, err)) return 0 # Work out how many commits to build. We want to build everything on the # branch. We also build the upstream commit as a control so we can see # problems introduced by the first commit on the branch. count = options.count has_range = options.branch and '..' in options.branch if count == -1: if not options.branch: count = 1 else: if has_range: count, msg = gitutil.CountCommitsInRange( options.git_dir, options.branch) else: count, msg = gitutil.CountCommitsInBranch( options.git_dir, options.branch) if count is None: sys.exit(col.Color(col.RED, msg)) elif count == 0: sys.exit( col.Color(col.RED, "Range '%s' has no commits" % options.branch)) if msg: print(col.Color(col.YELLOW, msg)) count += 1 # Build upstream commit also if not count: str = ("No commits found to process in branch '%s': " "set branch's upstream or use -c flag" % options.branch) sys.exit(col.Color(col.RED, str)) if options.work_in_output: if len(selected) != 1: sys.exit( col.Color(col.RED, '-w can only be used with a single board')) if count != 1: sys.exit( col.Color(col.RED, '-w can only be used with a single commit')) # Read the metadata from the commits. First look at the upstream commit, # then the ones in the branch. We would like to do something like # upstream/master~..branch but that isn't possible if upstream/master is # a merge commit (it will list all the commits that form part of the # merge) # Conflicting tags are not a problem for buildman, since it does not use # them. For example, Series-version is not useful for buildman. On the # other hand conflicting tags will cause an error. So allow later tags # to overwrite earlier ones by setting allow_overwrite=True if options.branch: if count == -1: if has_range: range_expr = options.branch else: range_expr = gitutil.GetRangeInBranch(options.git_dir, options.branch) upstream_commit = gitutil.GetUpstream(options.git_dir, options.branch) series = patchstream.GetMetaDataForList(upstream_commit, options.git_dir, 1, series=None, allow_overwrite=True) series = patchstream.GetMetaDataForList(range_expr, options.git_dir, None, series, allow_overwrite=True) else: # Honour the count series = patchstream.GetMetaDataForList(options.branch, options.git_dir, count, series=None, allow_overwrite=True) else: series = None if not options.dry_run: options.verbose = True if not options.summary: options.show_errors = True # By default we have one thread per CPU. But if there are not enough jobs # we can have fewer threads and use a high '-j' value for make. if not options.threads: options.threads = min(multiprocessing.cpu_count(), len(selected)) if not options.jobs: options.jobs = max(1, (multiprocessing.cpu_count() + len(selected) - 1) // len(selected)) if not options.step: options.step = len(series.commits) - 1 gnu_make = command.Output(os.path.join(options.git, 'scripts/show-gnu-make'), raise_on_error=False).rstrip() if not gnu_make: sys.exit('GNU Make not found') # Create a new builder with the selected options. output_dir = options.output_dir if options.branch: dirname = options.branch.replace('/', '_') # As a special case allow the board directory to be placed in the # output directory itself rather than any subdirectory. if not options.no_subdirs: output_dir = os.path.join(options.output_dir, dirname) if clean_dir and os.path.exists(output_dir): shutil.rmtree(output_dir) builder = Builder(toolchains, output_dir, options.git_dir, options.threads, options.jobs, gnu_make=gnu_make, checkout=True, show_unknown=options.show_unknown, step=options.step, no_subdirs=options.no_subdirs, full_path=options.full_path, verbose_build=options.verbose_build, mrproper=options.mrproper, per_board_out_dir=options.per_board_out_dir, config_only=options.config_only, squash_config_y=not options.preserve_config_y, warnings_as_errors=options.warnings_as_errors, work_in_output=options.work_in_output) builder.force_config_on_failure = not options.quick if make_func: builder.do_make = make_func # For a dry run, just show our actions as a sanity check if options.dry_run: ShowActions(series, why_selected, selected, builder, options, board_warnings) else: builder.force_build = options.force_build builder.force_build_failures = options.force_build_failures builder.force_reconfig = options.force_reconfig builder.in_tree = options.in_tree # Work out which boards to build board_selected = boards.GetSelectedDict() if series: commits = series.commits # Number the commits for test purposes for commit in range(len(commits)): commits[commit].sequence = commit else: commits = None Print( GetActionSummary(options.summary, commits, board_selected, options)) # We can't show function sizes without board details at present if options.show_bloat: options.show_detail = True builder.SetDisplayOptions( options.show_errors, options.show_sizes, options.show_detail, options.show_bloat, options.list_error_boards, options.show_config, options.show_environment, options.filter_dtb_warnings, options.filter_migration_warnings) if options.summary: builder.ShowSummary(commits, board_selected) else: fail, warned = builder.BuildBoards(commits, board_selected, options.keep_outputs, options.verbose) if fail: return 100 elif warned and not options.ignore_warnings: return 101 return 0