def parse_args(self): transport = Transport(self.argv, check_help=False) transport.catch_help = self.__doc__ if len(self.argv) <= 1: transport.print_help() transport.parse_args() for action in self.actions: if transport.has(action): return self.actions.get(action)() # If nothing matches, print the help transport.print_help()
def main(self): options = ['--ignore'] config_ignores = ceph_medic.config.file.get_list('check', '--ignore') parser = Transport(self.argv, options=options, check_version=False) parser.catch_help = self._help() parser.parse_args() ignored_codes = as_list(parser.get('--ignore', '')) # fallback to the configuration if nothing is defined in the CLI if not ignored_codes: ignored_codes = config_ignores if len(self.argv) < 1: return parser.print_help() # populate the nodes metadata with the configured nodes for daemon in ceph_medic.config.nodes.keys(): ceph_medic.metadata['nodes'][daemon] = [] for daemon, nodes in ceph_medic.config.nodes.items(): for node in nodes: node_metadata = {'host': node['host']} if 'container' in node: node_metadata['container'] = node['container'] ceph_medic.metadata['nodes'][daemon].append(node_metadata) collector.collect() test = runner.Runner() test.ignore = ignored_codes results = test.run() runner.report(results) #XXX might want to make this configurable to not bark on warnings for # example, setting forcefully for now, but the results object doesn't # make a distinction between error and warning (!) if results.errors or results.warnings: sys.exit(1)
def main(self): options = ['--ignore'] parser = Transport(self.argv, options=options, check_version=False) parser.catch_help = self._help() parser.parse_args() if len(self.argv) < 1: return parser.print_help() # populate the nodes metadata with the configured nodes for daemon in ceph_medic.config.nodes.keys(): ceph_medic.metadata['nodes'][daemon] = [] for daemon, nodes in ceph_medic.config.nodes.items(): for node in nodes: ceph_medic.metadata['nodes'][daemon].append( {'host': node['host']}) collector.collect() test = runner.Runner() results = test.run() runner.report(results) #XXX might want to make this configurable to not bark on warnings for # example, setting forcefully for now, but the results object doesn't # make a distinction between error and warning (!) if results.errors or results.warnings: sys.exit(1)
def main(self, argv): # Console Logger sh = logging.StreamHandler() sh.setFormatter(log.color_format()) sh.setLevel(logging.DEBUG) root_logger = logging.getLogger() root_logger.setLevel(logging.DEBUG) root_logger.addHandler(sh) self.api_credentials() # TODO: Need to implement `--filename` and make it available options = [['--log', '--logging']] parser = Transport(argv, mapper=self.mapper, options=options, check_help=False, check_version=False) parser.parse_args() chacractl.config['verbosity'] = parser.get('--log', 'info') parser.catch_help = self.help() parser.catch_version = chacractl.__version__ parser.mapper = self.mapper if len(argv) <= 1: return parser.print_help() parser.dispatch() parser.catches_help() parser.catches_version()
def main(self, argv): options = [ '--cluster', '--ssh-config', '--inventory', '--config', ] parser = Transport(argv, options=options, check_help=False, check_version=False) parser.parse_args() self.config_path = parser.get('--config', configuration.location()) # load medic configuration loaded_config = configuration.load( path=parser.get('--config', self.config_path)) # this is the earliest we can have enough config to setup logging log.setup(loaded_config) # update the module-wide configuration object ceph_medic.config.update(configuration.get_overrides(loaded_config)) # SSH config ceph_medic.config['ssh_config'] = parser.get('--ssh-config') if ceph_medic.config['ssh_config']: ssh_config_path = ceph_medic.config['ssh_config'] if not os.path.exists(ssh_config_path): terminal.error("the given ssh config path does not exist: %s" % ssh_config_path) sys.exit() ceph_medic.config['cluster_name'] = parser.get('--cluster') ceph_medic.metadata['cluster_name'] = 'ceph' # Hosts file self.hosts_file = parser.get('--inventory', configuration.get_host_file()) # find the hosts files, by the CLI first, fallback to the configuration # file, and lastly if none of those are found or defined, try to load # from well known locations (cwd, and /etc/ansible/) loaded_hosts = configuration.load_hosts( parser.get('--inventory', ceph_medic.config.get('--inventory', self.hosts_file))) ceph_medic.config['nodes'] = loaded_hosts.nodes ceph_medic.config['hosts_file'] = loaded_hosts.filename self.hosts_file = loaded_hosts.filename parser.catch_version = ceph_medic.__version__ parser.mapper = self.mapper parser.catch_help = self.help(parser.subhelp()) if len(argv) <= 1: return parser.print_help() ceph_medic.config['config_path'] = self.config_path parser.dispatch() parser.catches_help() parser.catches_version()
def parse_args(self, argv): """ Main method for parsing arguments, it uses whatever `argv` is although it should always be a list. Once it goes through the ``Transport`` class it tries to generate the help from the mapped classes and the current docstring for this module. If nothing matches it will return the help. """ self.get_extend_commands() transport = Transport(argv, self.mapper) transport.catch_help = "%s \n%s" % (__doc__, transport.subhelp()) transport.catch_version = "dozo version {0}".format(__version__) if len(self.argv) <= 1: transport.print_help() transport.dispatch()
def main(self, argv): parser = Transport(argv, mapper=self.mapper, check_help=False, check_version=False) parser.catch_help = self.help() parser.catch_version = github_status.__version__ parser.mapper = self.mapper if len(argv) <= 1: return parser.print_help() parser.dispatch(with_exit=True) parser.catches_help() parser.catches_version()
def main(self, argv): parser = Transport(argv, mapper=self.mapper, options=[], check_help=False, check_version=False) parser.parse_args() parser.catch_help = self.help(parser.subhelp()) parser.catch_version = ceph_installer.__version__ parser.mapper = self.mapper if len(argv) <= 1: return parser.print_help() parser.dispatch() parser.catches_help() parser.catches_version()
def main(self, argv): options = [] parser = Transport(argv, mapper=self.mapper, options=options, check_help=False, check_version=False) parser.parse_args() parser.catch_help = self.help() parser.catch_version = rhcephpkg.__version__ parser.mapper = self.mapper if len(argv) <= 1: return parser.print_help() parser.dispatch() parser.catches_help() parser.catches_version()
def main(self, argv): options = [['--log', '--logging']] parser = Transport(argv, mapper=self.mapper, options=options, check_help=False, check_version=False) parser.parse_args() merfi.config['verbosity'] = parser.get('--log', 'info') merfi.config['check'] = parser.has('--check') parser.catch_help = self.help() parser.catch_version = merfi.__version__ parser.mapper = self.mapper if len(argv) <= 1: return parser.print_help() parser.dispatch() parser.catches_help() parser.catches_version()
class Clone(object): help_menu = 'clone a package from dist-git' _help = """ Clone a package from dist-git. Your SSH key must be set up in Gerrit. Positional Arguments: [package] The name of the package to clone. """ name = 'clone' def __init__(self, argv): self.argv = argv self.options = [] def main(self): self.parser = Transport(self.argv, options=self.options) self.parser.catch_help = self.help() self.parser.parse_args() try: pkg = self.parser.unknown_commands[0] except IndexError: return self.parser.print_help() self._run(pkg) def help(self): return self._help def _run(self, pkg): """ Clone a package from dist-git. """ if os.path.exists(pkg): raise SystemExit('%s already exists in current working directory.', pkg) configp = util.config() try: user = configp.get('rhcephpkg', 'user') gitbaseurl = configp.get('rhcephpkg', 'gitbaseurl') except configparser.Error as err: raise SystemExit('Problem parsing .rhcephpkg.conf: %s', err.message) # TODO: SafeConfigParser might make the "user" interpolation here # unnecessary? Need to test, particularly what it does to %(module). pkg_url = gitbaseurl % {'user': user, 'module': pkg} cmd = ['git', 'clone', pkg_url] subprocess.check_call(cmd)
def main(self, argv): parser = Transport(argv, mapper=self.mapper, check_help=False, check_version=False) parser.parse_args() parser.catch_help = self.help() parser.catch_version = wari.__version__ parser.mapper = self.mapper if len(argv) <= 1: return parser.print_help() # create the connection and set the collection conn = wari.db.get_connection() wari.db.connection = conn wari.db.collection = conn['wari'] parser.dispatch() parser.catches_help() parser.catches_version() conn.close()
def main(self): options = ['--ignore'] parser = Transport(self.argv, options=options, check_version=False) parser.catch_help = self._help() parser.parse_args() if len(self.argv) < 1: return parser.print_help() # populate the nodes metadata with the configured nodes for daemon in ceph_medic.config['nodes'].keys(): ceph_medic.metadata['nodes'][daemon] = [] for daemon, nodes in ceph_medic.config['nodes'].items(): for node in nodes: ceph_medic.metadata['nodes'][daemon].append( {'host': node['host']}) collector.collect() test = runner.Runner() results = test.run() runner.report(results)
class Clone(object): help_menu = 'clone a package from dist-git' _help = """ Clone a package from dist-git. Your SSH key must be set up in Gerrit. Positional Arguments: [package] The name of the package to clone. Python packages are named slightly differently between RHEL and Debian. If you pass a package name "python-foo" to this command, rhcephpkg will strip off the "python-" prefix and operate on a Debian package name "foo". """ name = 'clone' def __init__(self, argv): self.argv = argv self.options = [] def main(self): self.parser = Transport(self.argv, options=self.options) self.parser.catch_help = self.help() self.parser.parse_args() try: pkg = self.parser.unknown_commands[0] except IndexError: return self.parser.print_help() self._run(pkg) def help(self): return self._help def _run(self, pkg): """ Clone a package from dist-git. """ if os.path.exists(pkg): err = '%s already exists in current working directory.' % pkg raise SystemExit(err) configp = util.config() try: user = configp.get('rhcephpkg', 'user') gitbaseurl = configp.get('rhcephpkg', 'gitbaseurl') except configparser.Error as err: raise SystemExit('Problem parsing .rhcephpkg.conf: %s', err.message) # If we were given an RPM pkg name, switch to the Debian one: if pkg.startswith('python-'): pkg = pkg[7:] # TODO: SafeConfigParser might make the "user" interpolation here # unnecessary? Need to test, particularly what it does to %(module). pkg_url = gitbaseurl % {'user': user, 'module': pkg} cmd = ['git', 'clone', pkg_url] subprocess.check_call(cmd) os.chdir(pkg) patches_url = find_patches_url(configp, user, pkg) if patches_url: cmd = ['git', 'remote', 'add', '-f', 'patches', patches_url] subprocess.check_call(cmd) util.setup_pristine_tar_branch()
def main(self, argv): options = [ '--cluster', '--ssh-config', '--inventory', '--config', '--verbosity', ] parser = Transport(argv, options=options, check_help=False, check_version=False) parser.parse_args() self.config_path = parser.get('--config', configuration.location()) # load medic configuration loaded_config = configuration.load( path=parser.get('--config', self.config_path)) # this is the earliest we can have enough config to setup logging log.setup(loaded_config) ceph_medic.config.file = loaded_config global_options = dict(ceph_medic.config.file._sections['global']) # SSH config ceph_medic.config.ssh_config = parser.get( '--ssh-config', global_options.get('--ssh-config')) if ceph_medic.config.ssh_config: ssh_config_path = ceph_medic.config.ssh_config if not os.path.exists(ssh_config_path): terminal.error("the given ssh config path does not exist: %s" % ssh_config_path) sys.exit() ceph_medic.config.cluster_name = parser.get('--cluster', 'ceph') ceph_medic.metadata['cluster_name'] = 'ceph' # Deployment Type deployment_type = ceph_medic.config.file.get_safe( 'global', 'deployment_type', 'baremetal') if deployment_type in ['kubernetes', 'openshift', 'k8s', 'oc']: pod_hosts = hosts.container_platform(deployment_type) ceph_medic.config.nodes = pod_hosts ceph_medic.config.hosts_file = ':memory:' self.hosts_file = ':memory:' else: # Hosts file self.hosts_file = parser.get('--inventory', configuration.get_host_file()) # find the hosts files, by the CLI first, fallback to the configuration # file, and lastly if none of those are found or defined, try to load # from well known locations (cwd, and /etc/ansible/) loaded_hosts = configuration.load_hosts( parser.get('--inventory', global_options.get('--inventory', self.hosts_file))) ceph_medic.config.nodes = loaded_hosts.nodes ceph_medic.config.hosts_file = loaded_hosts.filename self.hosts_file = loaded_hosts.filename parser.catch_version = ceph_medic.__version__ parser.mapper = self.mapper parser.catch_help = self.help(parser.subhelp()) if len(argv) <= 1: return parser.print_help() ceph_medic.config.config_path = self.config_path parser.dispatch() parser.catches_help() parser.catches_version() # Verbosity verbosity = parser.get('--verbosity', 'debug') ceph_medic.config.verbosity = verbosity.lowercase()
class MergePatches(object): help_menu = 'Merge patches from RHEL -patches branch to patch-queue branch' _help = """ Fetch the latest patches branch that rdopkg uses, and then fast-forward merge that into our local patch-queue branch, so that both branches align. This command helps to align the patch series between our RHEL packages and our Ubuntu packages. Options: --force Do a hard reset, rather than restricting to fast-forward merges only. Use this option if the RHEL patches branch was amended or rebased for some reason. """ name = 'merge-patches' def __init__(self, argv): self.argv = argv self.options = ['--force', '--hard-reset'] def main(self): self.parser = Transport(self.argv, options=self.options) self.parser.catch_help = self.help() self.parser.parse_args() force = False if self.parser.has(['--force', '--hard-reset']): force = True if self.parser.unknown_commands: log.error('unknown option %s', ' '.join(self.parser.unknown_commands)) return self.parser.print_help() self._run(force) def help(self): return self._help def _run(self, force=False): # Determine the names of the relevant branches current_branch = util.current_branch() debian_branch = util.current_debian_branch() patch_queue_branch = util.current_patch_queue_branch() rhel_patches_branch = self.get_rhel_patches_branch(debian_branch) # Do the merge if current_branch == patch_queue_branch: # HEAD is our patch-queue branch. Use "git pull" directly. # For example: "git pull --ff-only patches/ceph-2-rhel-patches" cmd = ['git', 'pull', '--ff-only', 'patches/' + rhel_patches_branch] if force: # Do a hard reset on HEAD instead. cmd = ['git', 'reset', '--hard', 'patches/' + rhel_patches_branch] else: # HEAD is our debian branch. Use "git fetch" to update the # patch-queue ref. For example: # "git fetch . \ # patches/ceph-2-rhel-patches:patch-queue/ceph-2-ubuntu" util.ensure_patch_queue_branch() cmd = ['git', 'fetch', '.', 'patches/%s:%s' % (rhel_patches_branch, patch_queue_branch)] if force: # Do a hard push (with "+") instead. cmd = ['git', 'push', '.', '+patches/%s:%s' % (rhel_patches_branch, patch_queue_branch)] log.info(' '.join(cmd)) subprocess.check_call(cmd) def get_rhel_patches_branch(self, debian_branch): """ Get the RHEL -patches branch corresponding to this debian branch. Examples: ceph-2-ubuntu -> ceph-2-rhel-patches ceph-2-trusty -> ceph-2-rhel-patches ceph-2-xenial -> ceph-2-rhel-patches ceph-1.3-ubuntu -> ceph-1.3-rhel-patches ceph-2-ubuntu-hotfix-bz123 -> ceph-2-rhel-patches-hotfix-bz123 """ (product, version, distro) = debian_branch.split('-', 2) suffix = None if '-' in distro: (distro, suffix) = distro.split('-', 1) rhel = '%s-%s-rhel-patches' % (product, version) if suffix is not None: rhel = '%s-%s' % (rhel, suffix) return rhel
class CheckoutFromPatches(object): help_menu = 'Choose a Debian branch based on a RHEL -patches branch' _help = """ Check out the Git branch that corresponds to a given RHEL (rdopkg-style) -patches branch. If you are starting from a RHEL -patches branch name (say, from a trigger in Jenkins), this will automatically choose the right Debian branch that goes with your -patches branch. Example: rhcephpkg checkout-from-patches ceph-3.0-rhel-patches ... this will checkout the "ceph-3.0-xenial" Git branch if it exists in the "origin" remote, or fall back to the "ceph-3.0-ubuntu" branch, or error if neither exist. Positional Arguments: [branch] The name of the -patches branch. """ name = 'checkout-from-patches' def __init__(self, argv): self.argv = argv self.options = [] def main(self): self.parser = Transport(self.argv, options=self.options) self.parser.catch_help = self.help() self.parser.parse_args() try: patches_branch = self.parser.unknown_commands[0] except IndexError: return self.parser.print_help() self._run(patches_branch) def help(self): return self._help def _run(self, patches_branch): debian_branch = self.get_debian_branch(patches_branch) if not debian_branch: err = 'could not find debian branch for %s' % patches_branch raise SystemExit(err) cmd = ['git', 'checkout', debian_branch] log.info(' '.join(cmd)) subprocess.check_call(cmd) def get_debian_branch(self, patches_branch): """ Get the debian branch corresponding to this RHEL -patches branch. Examples: ceph-2-rhel-patches -> ceph-2-xenial or ceph-2-ubuntu ceph-2-rhel-patches-hotfix-bz123 -> ceph-2-ubuntu-hotfix-bz123 :returns: name of debian branch, or None if none was found. """ patches_re = re.compile('-rhel-patches') debian_re = patches_re.sub('-([a-z]+)', patches_branch) ubuntu_branch = None for branch in self.get_origin_branches(): m = re.match('^%s$' % debian_re, branch) if m: if m.group(1) == 'ubuntu': # Use this only if we could find no other distro branch. ubuntu_branch = branch else: return branch return ubuntu_branch def get_origin_branches(self): """ Return a list of all the branches in the "origin" remote. """ cmd = ['git', 'branch', '-r', '--list', 'origin/*'] output = subprocess.check_output(cmd) if six.PY3: output = output.decode('utf-8') lines = output.split("\n") branches = [line.strip()[7:] for line in lines] return branches
class Download(object): help_menu = 'download a build from chacra' _help = """ Download a build's entire artifacts from chacra. Positional Arguments: [build] The name of the build to download, eg. "ceph_10.2.0-2redhat1trusty" """ name = 'download' def __init__(self, argv): self.argv = argv self.options = [] def main(self): self.parser = Transport(self.argv, options=self.options) self.parser.catch_help = self.help() self.parser.parse_args() try: build = self.parser.unknown_commands[0] except IndexError: return self.parser.print_help() self._run(build) def help(self): return self._help def _run(self, build): configp = util.config() try: base_url = configp.get('rhcephpkg.chacra', 'url') except configparser.Error as err: raise SystemExit('Problem parsing .rhcephpkg.conf: %s', err.message) try: (pkg, version) = build.split('_') except ValueError: log.error('%s is not a valid package build N-V-R' % build) return self.parser.print_help() build_url = posixpath.join(base_url, 'binaries/', pkg, version, 'ubuntu', 'all') log.info('searching %s for builds' % build_url) build_response = urlopen(Request(build_url)) headers = build_response.headers if six.PY2: encoding = headers.getparam('charset') or 'utf-8' # if encoding is None: # encoding = 'utf-8' else: encoding = headers.get_content_charset(failobj='utf-8') payload = json.loads(build_response.read().decode(encoding)) for arch, binaries in six.iteritems(payload): for binary in binaries: if os.path.isfile(binary): # TODO: check the sha256sum of the already-downloaded file # here? log.info('skipping %s' % binary) continue log.info('downloading %s' % binary) binary_url = posixpath.join(build_url, arch, binary) + '/' response = urlopen(Request(binary_url)) with open(binary, 'wb') as fp: shutil.copyfileobj(response, fp)
class MergePatches(object): help_menu = 'Merge patches from RHEL -patches branch to patch-queue branch' _help = """ Fetch the latest patches branch that rdopkg uses, and then fast-forward merge that into our local patch-queue branch, so that both branches align. This command helps to align the patch series between our RHEL packages and our Ubuntu packages. Options: --force Do a hard reset, rather than restricting to fast-forward merges only. Use this option if the RHEL patches branch was amended or rebased for some reason. """ name = 'merge-patches' def __init__(self, argv): self.argv = argv self.options = ['--force', '--hard-reset'] def main(self): self.parser = Transport(self.argv, options=self.options) self.parser.catch_help = self.help() self.parser.parse_args() force = False if self.parser.has(['--force', '--hard-reset']): force = True if self.parser.unknown_commands: log.error('unknown option %s', ' '.join(self.parser.unknown_commands)) return self.parser.print_help() self._run(force) def help(self): return self._help def _run(self, force=False): # Determine the names of the relevant branches current_branch = util.current_branch() debian_branch = util.current_debian_branch() patches_branch = util.current_patches_branch() rhel_patches_branch = self.get_rhel_patches_branch(debian_branch) # Do the merge if current_branch == patches_branch: # HEAD is our patch-queue branch. Use "git pull" directly. # For example: "git pull --ff-only patches/ceph-2-rhel-patches" cmd = ['git', 'pull', '--ff-only', 'patches/' + rhel_patches_branch] if force: # Do a hard reset on HEAD instead. cmd = ['git', 'reset', '--hard', 'patches/' + rhel_patches_branch] else: # HEAD is our debian branch. Use "git fetch" to update the # patch-queue ref. For example: # "git fetch . \ # patches/ceph-2-rhel-patches:patch-queue/ceph-2-ubuntu" cmd = ['git', 'fetch', '.', 'patches/%s:%s' % (rhel_patches_branch, patches_branch)] if force: # Do a hard push (with "+") instead. cmd = ['git', 'push', '.', '+patches/%s:%s' % (rhel_patches_branch, patches_branch)] log.info(' '.join(cmd)) subprocess.check_call(cmd) def get_rhel_patches_branch(self, debian_branch): """ Get the RHEL -patches branch corresponding to this debian branch. Examples: ceph-2-ubuntu -> ceph-2-rhel-patches ceph-2-trusty -> ceph-2-rhel-patches ceph-2-xenial -> ceph-2-rhel-patches ceph-1.3-ubuntu -> ceph-1.3-rhel-patches ceph-2-ubuntu-hotfix-bz123 -> ceph-2-rhel-patches-hotfix-bz123 """ (product, version, distro) = debian_branch.split('-', 2) suffix = None if '-' in distro: (distro, suffix) = distro.split('-', 1) rhel = '%s-%s-rhel-patches' % (product, version) if suffix is not None: rhel = '%s-%s' % (rhel, suffix) return rhel
class WatchBuild(object): help_menu = 'watch a build-package job in Jenkins' _help = """ Watch a particular build-package job in Jenkins. Positional Arguments: [id] The build-package job ID to watch For example: "rhcephpkg watch-build 328" """ name = 'watch-build' def __init__(self, argv): self.argv = argv self.options = [] def main(self): self.parser = Transport(self.argv, options=self.options) self.parser.catch_help = self.help() self.parser.parse_args() try: build_number = int(self.parser.unknown_commands[0]) except (IndexError, ValueError): return self.parser.print_help() self.watch(build_number) def help(self): return self._help def watch(self, build_number): jenkins = util.jenkins_connection() build_info = jenkins.get_build_info('build-package', build_number) job_url = posixpath.join(jenkins.url, 'job', 'build-package', str(build_number)) log.info('Watching %s' % job_url) pkg_name = self.pkg_name(build_info) start_seconds = build_info['timestamp'] / 1000.0 # rcm-jenkins is uses the America/New_York timezone: jenkins_tz = tz.gettz('America/New_York') start = datetime.fromtimestamp(start_seconds, jenkins_tz) # If you want to convert to local time: # start = start.astimezone(tz.tzlocal()) log.info('Started %s' % start.strftime("%F %r %z")) was_building = build_info['building'] while build_info['building']: try: elapsed = datetime.now(jenkins_tz) - start # TODO: Xenial has python-humanize (humanize.naturaldelta() # here) (minutes, seconds) = divmod(elapsed.total_seconds(), 60) # Clear the previous line: msg = '\r%s building for %02d:%02d' % \ (pkg_name, minutes, seconds) sys.stdout.write(msg) sys.stdout.flush() sleep(10) build_info = jenkins.get_build_info('build-package', build_number) except requests.exceptions.ConnectionError as e: print('') log.error('connection error: %s' % e) log.info('Re-try watching with `rhcephpkg watch-build %s`' % build_number) except KeyboardInterrupt: print('') log.info('continue watching with `rhcephpkg watch-build %s`' % build_number) raise SystemExit(1) if was_building: # The above "while" loop will not print a final newline. print('') end_millis = build_info['timestamp'] + build_info['duration'] end_seconds = end_millis / 1000.0 end = datetime.fromtimestamp(end_seconds, jenkins_tz) log.info('Ended %s' % end.strftime("%F %r %z")) # Show the final build result. if build_info['result'] == 'SUCCESS': log.info('result is SUCCESS') else: log.error(build_info['result']) raise SystemExit(1) def pkg_name(self, build_info): """ Return a package name based on this build's information. :param build_info: ``dict`` from python-jenkins' get_build_info() :returns: ``str``, for example "ceph" or "ceph-ansible". """ pkg_name = None for action in build_info['actions']: if action.get('_class', '') == 'hudson.model.ParametersAction': for parameter in action['parameters']: if parameter['name'] == 'PKG_NAME': pkg_name = parameter['value'] if pkg_name is None: # Maybe the Jenkins job was badly mis-configured or something. # This will probably never happen, but let's raise defensively. raise RuntimeError('could not find pkg name in %s' % build_info) return pkg_name
class Localbuild(object): help_menu = 'build a package on the local system' _help = """ Build a package on the local system, using pbuilder. Options: --dist "xenial" or "trusty". If unspecified, rhcephpkg will choose one based on the current branch's name. Rules for automatic distro selection: 1) If the branch suffix is an ubuntu distro name, use that. eg "ceph-3.0-xenial". 2) If a branch has a version number starting with "1.3", return "trusty". eg. "ceph-1.3-ubuntu" 3) If a branch has a version number starting with "2" return "xenial". eg. "ceph-2-ubuntu" 4) If a branch has a version number starting with "3" return "xenial". eg. "ceph-3.0-ubuntu" 5) Otherwise raise, because we need to add more rules. """ name = 'localbuild' def __init__(self, argv): self.argv = argv self.options = ('--dist', ) def main(self): self.parser = Transport(self.argv, options=self.options) self.parser.catch_help = self.help() self.parser.parse_args() # Allow user to override the distro. if self.parser.has('--dist'): if self.parser.get('--dist') is None: raise SystemExit('Specify a distro to --dist') distro = self.parser.get('--dist') else: distro = get_distro() if self.parser.unknown_commands: log.error('unknown option %s', ' '.join(self.parser.unknown_commands)) return self.parser.print_help() self._run(distro) def help(self): return self._help def _run(self, distro): """ Build a package on the local system, using pbuilder. """ pkg_name = util.package_name() os.environ['BUILDER'] = 'pbuilder' j_arg = self._get_j_arg(cpu_count()) pbuilder_cache = '/var/cache/pbuilder/base-%s-amd64.tgz' % distro setup_pbuilder_cache(pbuilder_cache, distro) util.setup_pristine_tar_branch() # TODO: we should also probably check parent dir for leftovers and warn # the user to delete them (or delete them ourselves?) cmd = [ 'gbp', 'buildpackage', '--git-dist=%s' % distro, '--git-arch=amd64', '--git-verbose', '--git-pbuilder', j_arg, '-us', '-uc' ] log.info('building %s with pbuilder', pkg_name) subprocess.check_call(cmd) def _get_j_arg(self, cpus, total_ram_gb=None): """ Returns a string like "-j4" or "-j8". j is the number of processors, with a maximum of x, where x = TOTAL_RAM_GB / 4. We want to use all our processors (a high "j" value), but the build process will fail with an "out of memory" error out if this j value is too high. An 8 GB system would have a maximum of -j2 A 16 GB system would have a maximum of -j4 A 32 GB system would have a maximum of -j8 """ if total_ram_gb is None: page_size = os.sysconf('SC_PAGE_SIZE') mem_bytes = page_size * os.sysconf('SC_PHYS_PAGES') # mem_gib is a decimal, eg. 7.707 on 8GB system mem_gib = mem_bytes / (1024.**3) # Round up to the nearest GB for our purposes. total_ram_gb = math.ceil(mem_gib) number = min(cpus, total_ram_gb / 4) return '-j%d' % max(number, 1)
class WatchBuild(object): help_menu = 'watch a build-package job in Jenkins' _help = """ Watch a particular build-package job in Jenkins. Positional Arguments: [id] The build-package job ID to watch For example: "rhcephpkg watch-build 328" """ name = 'watch-build' def __init__(self, argv): self.argv = argv self.options = [] def main(self): self.parser = Transport(self.argv, options=self.options) self.parser.catch_help = self.help() self.parser.parse_args() try: build_number = int(self.parser.unknown_commands[0]) except (IndexError, ValueError): return self.parser.print_help() self.watch(build_number) def help(self): return self._help def watch(self, build_number): jenkins = util.jenkins_connection() build_info = jenkins.get_build_info('build-package', build_number) job_url = posixpath.join(jenkins.url, 'job', 'build-package', str(build_number)) log.info('Watching %s' % job_url) pkg_name = self.pkg_name(build_info) start_seconds = build_info['timestamp'] / 1000.0 # rcm-jenkins is uses the America/New_York timezone: jenkins_tz = tz.gettz('America/New_York') start = datetime.fromtimestamp(start_seconds, jenkins_tz) # If you want to convert to local time: # start = start.astimezone(tz.tzlocal()) log.info('Started %s' % start.strftime("%F %r %z")) while build_info['building']: elapsed = datetime.now(jenkins_tz) - start # TODO: Xenial has python-humanize (humanize.naturaldelta() here) # Backport the python-humanize package for Trusty? Or drop Trusty? (minutes, seconds) = divmod(elapsed.total_seconds(), 60) # Clear the previous line: msg = '\r%s building for %02d:%02d' % (pkg_name, minutes, seconds) sys.stdout.write(msg) sys.stdout.flush() sleep(10) build_info = jenkins.get_build_info('build-package', build_number) end_millis = build_info['timestamp'] + build_info['duration'] end_seconds = end_millis / 1000.0 end = datetime.fromtimestamp(end_seconds, jenkins_tz) log.info('Ended %s' % end.strftime("%F %r %z")) # Show the final build result. if build_info['result'] == 'SUCCESS': log.info('result is SUCCESS') else: log.error(build_info['result']) raise SystemExit(1) def pkg_name(self, build_info): """ Return a package name based on this build's information. :param build_info: ``dict`` from python-jenkins' get_build_info() :returns: ``str``, for example "ceph" or "ceph-ansible". """ pkg_name = None for action in build_info['actions']: if action.get('_class', '') == 'hudson.model.ParametersAction': for parameter in action['parameters']: if parameter['name'] == 'PKG_NAME': pkg_name = parameter['value'] if pkg_name is None: # Maybe the Jenkins job was badly mis-configured or something. # This will probably never happen, but let's raise defensively. raise RuntimeError('could not find pkg name in %s' % build_info) return pkg_name