def save(self, **pins): for k, v in pins.iteritems(): assert isinstance(k, basestring) assert isinstance(v, basestring) LOGGER.debug('Writing pin file [%s]: %s', self._path, pins) with open(self._path, 'w') as fd: json.dump(pins, fd, indent=2, sort_keys=True)
def save(self, **pins): for k, v in pins.iteritems(): assert isinstance(k, basestring) assert isinstance(v, basestring) LOGGER.debug('Writing pin file [%s]: %s', self._path, pins) with open(self._path, 'w') as fd: json.dump(pins, fd, indent=2, sort_keys=True)
def _destroy_directory(d): LOGGER.debug('Destorying directory: %s', d) def log_failure(_function, path, excinfo): LOGGER.warning('Failed when destroying [%s]: %s', path, excinfo[1].message) shutil.rmtree(d, onerror=log_failure)
def fetch(cls, path): LOGGER.info("Fetching => %s (This can take a while.)", path) gclient_path = os.path.join(path, '.gclient') with open(gclient_path, 'w') as fd: fd.write(cls.GCLIENT_TEMPLATE) execute.check_call(['gclient', 'sync', '--nohooks', '--noprehooks'], cwd=path)
def use(cls, *args, **kwargs): c = None try: c = cls.create(*args, **kwargs) LOGGER.debug('Using checkout at: %s', c.path) yield c finally: if c: c.teardown()
def fetch(cls, build_revision, path): LOGGER.info("Fetching => %s (This can take a while.)", path) gclient_path = os.path.join(path, '.gclient') with open(gclient_path, 'w') as fd: fd.write(cls.GCLIENT_TEMPLATE) cmd = ['gclient', 'sync', '--nohooks', '--noprehooks'] if build_revision: LOGGER.info("Using build.git revision %s", build_revision) cmd.extend(['--revision', build_revision]) execute.check_call(cmd, cwd=path)
def checkout_for_args(args): """A contextmanager that supplies the Checkout configured in args. The Checkout's teardown() method will be invoked on cleanup. Args: args (argparse.Options): Parsed option list. """ LOGGER.warning('Checking out temporary repositories. This may take a few ' 'minutes.') return checkout.Checkout.use( build_revision=args.build_revision, path=args.checkout_path)
def checkout_for_args(args): """A contextmanager that supplies the Checkout configured in args. The Checkout's teardown() method will be invoked on cleanup. Args: args (argparse.Options): Parsed option list. """ LOGGER.warning('Checking out temporary repositories. This may take a few ' 'minutes.') return checkout.Checkout.use( path=args.checkout_path)
def _regenerate_slave_pool(self, master): LOGGER.debug('Regenerating slave pool for: %s', master) cmd = [ os.path.join(*self.RUNIT_PY), os.path.join(*self.SLAVE_ALLOC_UPDATE), ] cmd += logging_verbosity() cmd.append(master) rv, stdout = execute.call(cmd, cwd=self._c.path) if rv != 0: LOGGER.exception('Failed to update slaves for master [%s] (%d):\n%s', master, rv, stdout) raise SlavePoolUpdateError()
def _regenerate_slave_pool(self, master): LOGGER.debug('Regenerating slave pool for: %s', master) cmd = [ os.path.join(*self.RUNIT_PY), os.path.join(*self.SLAVE_ALLOC_UPDATE), ] cmd += logging_verbosity() cmd.append(master) rv, stdout = execute.call(cmd, cwd=self._c.path) if rv != 0: LOGGER.exception('Failed to update slaves for master [%s] (%d):\n%s', master, rv, stdout) raise SlavePoolUpdateError()
def subcommand_add_release(args): """Add a new release branch to the list of pins.""" with checkout_for_args(args) as c: pfe = pinfile_editor_from_args(args, c) tracker = UpdateTracker.from_args(args, c) add_release = (get_release_version(args.branch), args.branch) if add_release[0] is None: raise ValueError("Invalid release branch: [%s]" % (args.branch,)) # Build a list of releases and their versions. pf = pfe.load(pinfile.INTERNAL) releases = [add_release] for name, _ in pf.iterpins(): v = get_release_version(name) if v == add_release[0]: LOGGER.error('Release [%s] (%d) is already pinned.', add_release[1], add_release[0]) return 1 if v is not None: releases.append((v, name)) releases.sort(reverse=True) # Shave off the top [stable_count+1] releases. count = args.stable_count+1 releases, deleted = releases[:count], releases[count:] # Check if LTS release in deleted releases. for v, name in deleted: if ((v - BASE_LTS_MILESTONE) % NEXT_LTS_MILESTONE_MODULO) == 0: releases.append((v, name)) deleted.remove((v, name)) # We allow one milestone to exist in LTS window. # LTS window starts after stable window. break if add_release not in releases: raise ValueError("Updated releases do not include added (%s):\n%s" % ( add_release[1], '\n'.join(r[1] for r in releases))) # Set the new releases. tracker.add(pinfile.INTERNAL, pf.update(add_release[1], create=True)) for _, r in deleted: tracker.add(pinfile.INTERNAL, pf.remove(r)) if not tracker: LOGGER.error('No pins were updated.') return 1 # Regenerate slave pools for affected masters. try: tracker.update() except SlavePoolUpdateError as e: LOGGER.error('Failed to generate pin bump CLs: %s', e) return 1 LOGGER.info('Created issues:\n%s', '\n'.join(tracker.issues)) return 0
def subcommand_update(args): """Update a single Chromite pin.""" require = (args.target != 'existing') target_pins = [] if args.target in ('external', 'both', 'existing'): target_pins.append(pinfile.EXTERNAL) if args.target in ('internal', 'both', 'existing'): target_pins.append(pinfile.INTERNAL) with checkout_for_args(args) as c: pfe = pinfile_editor_from_args(args, c) tracker = UpdateTracker.from_args(args, c) for pin in target_pins: logging.debug('Updating target pin [%s]', pin) # Update the pin. pf = pfe.load(pin) if not (require or pf.has_pin(args.name)): LOGGER.debug('Pin not found in [%s]. Only updating existing pins.', pin) continue update = pf.update(args.name, version=args.version, create=require) if not update: LOGGER.debug('Did not update pins for [%s]', pin) continue tracker.add(pin, update) LOGGER.debug('Updated pin set: %s', update) if not tracker: LOGGER.error('No pins were updated.') return 1 # Regenerate slave pools for affected masters. tracker.update() for i in tracker.issues: LOGGER.warning('Created Issue: %s', i) return 0
def subcommand_update(args): """Update a single Chromite pin.""" require = (args.target != 'existing') target_pins = [] if args.target in ('external', 'both', 'existing'): target_pins.append(pinfile.EXTERNAL) if args.target in ('internal', 'both', 'existing'): target_pins.append(pinfile.INTERNAL) with checkout_for_args(args) as c: pfe = pinfile_editor_from_args(args, c) tracker = UpdateTracker.from_args(args, c) for pin in target_pins: logging.debug('Updating target pin [%s]', pin) # Update the pin. pf = pfe.load(pin) if not (require or pf.has_pin(args.name)): LOGGER.debug('Pin not found in [%s]. Only updating existing pins.', pin) continue update = pf.update(args.name, version=args.version, create=require) if not update: LOGGER.debug('Did not update pins for [%s]', pin) continue tracker.add(pin, update) LOGGER.debug('Updated pin set: %s', update) if not tracker: LOGGER.error('No pins were updated.') return 1 # Regenerate slave pools for affected masters. tracker.update() for i in tracker.issues: LOGGER.warning('Created Issue: %s', i) return 0
def update(self): LOGGER.info('Updating repositories: %s', self._updated) affected_masters = set() for pin in self._updated.iterkeys(): affected_masters.update(pin.masters) failed_slave_pool_masters = [] for m in sorted(affected_masters): try: self._regenerate_slave_pool(m) except SlavePoolUpdateError: failed_slave_pool_masters.append(m) if failed_slave_pool_masters: LOGGER.error('Failed to update slave pools for %s. You may need to ' 'add additional slaves the pool(s).', failed_slave_pool_masters) raise SlavePoolUpdateError("Failed to update slave pools.") # Upload CLs for the affected repositories. for pin, updates in self._updated.iteritems(): self._upload_patch( self._c.subpath(*pin.base), self._generate_commit_message(updates))
def update(self, pin_name, create=False, version=None): """Updates a single pin value.""" if not version: LOGGER.debug('Resolving version for pin [%s]', pin_name) version = self._editor.get_commit(pin_name) elif self._editor._validate: LOGGER.debug('Validating pin [%s]', pin_name) self._editor.validate_pin(version) with self.edit() as pins: current = pins.get(pin_name) if current == version: LOGGER.warning('Pin [%s.%s] is already at version [%s]', self._pin.name, pin_name, current) return None LOGGER.info('Updating pin [%s.%s]: [%s] => [%s]', self._pin.name, pin_name, current, version) if not (current or create): raise ReadOnlyError("Pin does not exist [%s]" % (pin_name,)) pins[pin_name] = version return PinUpdate(pin_name, current, version)
def update(self, pin_name, create=False, version=None): """Updates a single pin value.""" if not version: LOGGER.debug('Resolving version for pin [%s]', pin_name) version = self._editor.get_commit(pin_name) elif self._editor._validate: LOGGER.debug('Validating pin [%s]', pin_name) self._editor.validate_pin(version) with self.edit() as pins: current = pins.get(pin_name) if current == version: LOGGER.warning('Pin [%s.%s] is already at version [%s]', self._pin.name, pin_name, current) return None LOGGER.info('Updating pin [%s.%s]: [%s] => [%s]', self._pin.name, pin_name, current, version) if not (current or create): raise ReadOnlyError("Pin does not exist [%s]" % (pin_name, )) pins[pin_name] = version return PinUpdate(pin_name, current, version)
def subcommand_add_release(args): """Add a new release branch to the list of pins.""" with checkout_for_args(args) as c: pfe = pinfile_editor_from_args(args, c) tracker = UpdateTracker.from_args(args, c) add_release = (get_release_version(args.branch), args.branch) if add_release[0] is None: raise ValueError("Invalid release branch: [%s]" % (args.branch,)) # Build a list of releases and their versions. pf = pfe.load(pinfile.INTERNAL) releases = [add_release] for name, _ in pf.iterpins(): v = get_release_version(name) if v == add_release[0]: LOGGER.error('Release [%s] (%d) is already pinned.', add_release[1], add_release[0]) return 1 if v is not None: releases.append((v, name)) releases.sort(reverse=True) # Shave off the top [stable_count+1] releases. count = args.stable_count+1 releases, deleted = releases[:count], releases[count:] if add_release not in releases: raise ValueError("Updated releases do not include added (%s):\n%s" % ( add_release[1], '\n'.join(r[1] for r in releases))) # Set the new releases. tracker.add(pinfile.INTERNAL, pf.update(add_release[1], create=True)) for _, r in deleted: tracker.add(pinfile.INTERNAL, pf.remove(r)) if not tracker: LOGGER.error('No pins were updated.') return 1 # Regenerate slave pools for affected masters. tracker.update() LOGGER.info('Created issues:\n%s', '\n'.join(tracker.issues)) return 0
def call(cmd, cwd=None, dry_run=False): LOGGER.info("Executing command %s (cwd=%s)", cmd, (cwd or os.getcwd())) if dry_run: LOGGER.info('Dry Run: Not actually executing.') return (0, "") output = [] proc = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd) for line in iter(proc.stdout.readline, b''): LOGGER.debug('[%s]: %s', cmd[0], line.rstrip()) output.append(line) proc.wait() return proc.returncode, ''.join(output)
def update(self): LOGGER.info('Updating repositories: %s', self._updated) affected_masters = set() for pin in self._updated.iterkeys(): affected_masters.update(pin.masters) failed_slave_pool_masters = [] for m in sorted(affected_masters): try: self._regenerate_slave_pool(m) except SlavePoolUpdateError: LOGGER.exception("Failed to update slave pools for [%s].", m) failed_slave_pool_masters.append(m) if failed_slave_pool_masters: LOGGER.error('Failed to update slave pools for: %s. You may need to ' 'add additional slaves the pool(s).', failed_slave_pool_masters) raise SlavePoolUpdateError("Failed to update slave pools.") # Upload CLs for the affected repositories. for pin, updates in self._updated.iteritems(): self._upload_patch( self._c.subpath(*pin.base), self._generate_commit_message(updates))
def _upload_patch(self, repo_path, commit_msg): # Check if the Git repository actually has changes. diff_args = ['git', 'diff', '--no-ext-diff', '--exit-code'] if not LOGGER.isEnabledFor(logging.DEBUG): diff_args.append('--quiet') rv, diff = execute.call(diff_args, cwd=repo_path) LOGGER.debug('Diff for [%s]:\n%s', repo_path, diff) if rv == 0: LOGGER.warning('No changes in repository; refusing to commit.') return commit_msg_file = self._c.mktempfile(commit_msg) LOGGER.warning('Creating commit in [%s] with message:\n%s', repo_path, commit_msg) execute.check_call( ['git', 'checkout', '-b', '_cros_pin'], cwd=repo_path) execute.check_call( ['git', 'commit', '--all', '-F', commit_msg_file], cwd=repo_path) LOGGER.debug('Uploading CL!') args = [ 'git', 'cl', 'upload', '--bypass-hooks', # The CQ will take care of them! '--send-mail', '--message-file', commit_msg_file, '-f', ] if self._cq: print 'Commit? [Y/n]:', input_string = raw_input() if input_string != '' and not distutils.util.strtobool(input_string): LOGGER.warning('User opted not to commit; aborting.') return args.append('--use-commit-queue') if not self._reviewers: args.append('--tbr-owners') output = execute.check_call(args, cwd=repo_path, dry_run=self._dry_run) issue = None for line in output.splitlines(): for rx in self.RE_ISSUE_CREATED: match = rx.match(line) if match: issue = match.group(1) LOGGER.debug('Extracted issue from output: %s', issue) self._issues.add(issue) break else: LOGGER.warning("Unable to extract issue from patch submission from:\n%s", output)
def log_failure(_function, path, excinfo): LOGGER.warning('Failed when destroying [%s]: %s', path, excinfo[1].message)
def _upload_patch(self, repo_path, commit_msg): # Check if the Git repository actually has changes. diff_args = ['git', 'diff', '--no-ext-diff', '--exit-code'] if not LOGGER.isEnabledFor(logging.DEBUG): diff_args.append('--quiet') rv, diff = execute.call(diff_args, cwd=repo_path) LOGGER.debug('Diff for [%s]:\n%s', repo_path, diff) if rv == 0: LOGGER.warning('No changes in repository; refusing to commit.') return LOGGER.warning('Creating commit in [%s] with message:\n%s', repo_path, commit_msg) execute.check_call( ['git', 'checkout', '-b', '_cros_pin'], cwd=repo_path) execute.check_call( ['git', 'commit', '--all', '--message', commit_msg], cwd=repo_path) LOGGER.debug('Uploading CL!') args = [ 'git', 'cl', 'upload', '--bypass-hooks', # The CQ will take care of them! '-t', commit_msg, '-m', 'Auto-generated by `%s`' % (__name__,), '-f', ] if self._cq: print 'Commit? [Y/n]:', input_string = raw_input() if input_string != '' and not distutils.util.strtobool(input_string): LOGGER.warning('User opted not to commit; aborting.') return args.append('--use-commit-queue') if not self._reviewers: args.append('--tbr-owners') output = execute.check_call(args, cwd=repo_path, dry_run=self._dry_run) issue = None for line in output.splitlines(): match = self.RE_ISSUE_CREATED.match(line) if match: issue = match.group(1) LOGGER.debug('Extracted issue from output: %s', issue) self._issues.add(issue) break else: LOGGER.warning("Unable to extract issue from patch submission.")