def parse_args(self):
transport = Transport(self.argv, check_help=False)
transport.catch_help = self.__doc__
if len(self.argv) <= 1:
transport.print_help()
transport.parse_args()
for action in self.actions:
if transport.has(action):
return self.actions.get(action)()
# If nothing matches, print the help
transport.print_help()
def main(self):
options = ['--ignore']
config_ignores = ceph_medic.config.file.get_list('check', '--ignore')
parser = Transport(self.argv, options=options, check_version=False)
parser.catch_help = self._help()
parser.parse_args()
ignored_codes = as_list(parser.get('--ignore', ''))
# fallback to the configuration if nothing is defined in the CLI
if not ignored_codes:
ignored_codes = config_ignores
if len(self.argv) < 1:
return parser.print_help()
# populate the nodes metadata with the configured nodes
for daemon in ceph_medic.config.nodes.keys():
ceph_medic.metadata['nodes'][daemon] = []
for daemon, nodes in ceph_medic.config.nodes.items():
for node in nodes:
node_metadata = {'host': node['host']}
if 'container' in node:
node_metadata['container'] = node['container']
ceph_medic.metadata['nodes'][daemon].append(node_metadata)
collector.collect()
test = runner.Runner()
test.ignore = ignored_codes
results = test.run()
runner.report(results)
#XXX might want to make this configurable to not bark on warnings for
# example, setting forcefully for now, but the results object doesn't
# make a distinction between error and warning (!)
if results.errors or results.warnings:
sys.exit(1)
def main(self):
options = ['--ignore']
parser = Transport(self.argv, options=options, check_version=False)
parser.catch_help = self._help()
parser.parse_args()
if len(self.argv) < 1:
return parser.print_help()
# populate the nodes metadata with the configured nodes
for daemon in ceph_medic.config.nodes.keys():
ceph_medic.metadata['nodes'][daemon] = []
for daemon, nodes in ceph_medic.config.nodes.items():
for node in nodes:
ceph_medic.metadata['nodes'][daemon].append(
{'host': node['host']})
collector.collect()
test = runner.Runner()
results = test.run()
runner.report(results)
#XXX might want to make this configurable to not bark on warnings for
# example, setting forcefully for now, but the results object doesn't
# make a distinction between error and warning (!)
if results.errors or results.warnings:
sys.exit(1)
def main(self, argv):
# Console Logger
sh = logging.StreamHandler()
sh.setFormatter(log.color_format())
sh.setLevel(logging.DEBUG)
root_logger = logging.getLogger()
root_logger.setLevel(logging.DEBUG)
root_logger.addHandler(sh)
self.api_credentials()
# TODO: Need to implement `--filename` and make it available
options = [['--log', '--logging']]
parser = Transport(argv, mapper=self.mapper,
options=options, check_help=False,
check_version=False)
parser.parse_args()
chacractl.config['verbosity'] = parser.get('--log', 'info')
parser.catch_help = self.help()
parser.catch_version = chacractl.__version__
parser.mapper = self.mapper
if len(argv) <= 1:
return parser.print_help()
parser.dispatch()
parser.catches_help()
parser.catches_version()
def main(self, argv):
# Console Logger
sh = logging.StreamHandler()
sh.setFormatter(log.color_format())
sh.setLevel(logging.DEBUG)
root_logger = logging.getLogger()
root_logger.setLevel(logging.DEBUG)
root_logger.addHandler(sh)
self.api_credentials()
# TODO: Need to implement `--filename` and make it available
options = [['--log', '--logging']]
parser = Transport(argv,
mapper=self.mapper,
options=options,
check_help=False,
check_version=False)
parser.parse_args()
chacractl.config['verbosity'] = parser.get('--log', 'info')
parser.catch_help = self.help()
parser.catch_version = chacractl.__version__
parser.mapper = self.mapper
if len(argv) <= 1:
return parser.print_help()
parser.dispatch()
parser.catches_help()
parser.catches_version()
def main(self, argv):
options = [
'--cluster',
'--ssh-config',
'--inventory',
'--config',
]
parser = Transport(argv,
options=options,
check_help=False,
check_version=False)
parser.parse_args()
self.config_path = parser.get('--config', configuration.location())
# load medic configuration
loaded_config = configuration.load(
path=parser.get('--config', self.config_path))
# this is the earliest we can have enough config to setup logging
log.setup(loaded_config)
# update the module-wide configuration object
ceph_medic.config.update(configuration.get_overrides(loaded_config))
# SSH config
ceph_medic.config['ssh_config'] = parser.get('--ssh-config')
if ceph_medic.config['ssh_config']:
ssh_config_path = ceph_medic.config['ssh_config']
if not os.path.exists(ssh_config_path):
terminal.error("the given ssh config path does not exist: %s" %
ssh_config_path)
sys.exit()
ceph_medic.config['cluster_name'] = parser.get('--cluster')
ceph_medic.metadata['cluster_name'] = 'ceph'
# Hosts file
self.hosts_file = parser.get('--inventory',
configuration.get_host_file())
# find the hosts files, by the CLI first, fallback to the configuration
# file, and lastly if none of those are found or defined, try to load
# from well known locations (cwd, and /etc/ansible/)
loaded_hosts = configuration.load_hosts(
parser.get('--inventory',
ceph_medic.config.get('--inventory', self.hosts_file)))
ceph_medic.config['nodes'] = loaded_hosts.nodes
ceph_medic.config['hosts_file'] = loaded_hosts.filename
self.hosts_file = loaded_hosts.filename
parser.catch_version = ceph_medic.__version__
parser.mapper = self.mapper
parser.catch_help = self.help(parser.subhelp())
if len(argv) <= 1:
return parser.print_help()
ceph_medic.config['config_path'] = self.config_path
parser.dispatch()
parser.catches_help()
parser.catches_version()
def parse_args(self, argv):
"""
Main method for parsing arguments, it uses whatever `argv` is although
it should always be a list. Once it goes through the ``Transport`` class
it tries to generate the help from the mapped classes and the current
docstring for this module.
If nothing matches it will return the help.
"""
self.get_extend_commands()
transport = Transport(argv, self.mapper)
transport.catch_help = "%s \n%s" % (__doc__, transport.subhelp())
transport.catch_version = "dozo version {0}".format(__version__)
if len(self.argv) <= 1:
transport.print_help()
transport.dispatch()
def main(self, argv):
parser = Transport(argv,
mapper=self.mapper,
check_help=False,
check_version=False)
parser.catch_help = self.help()
parser.catch_version = github_status.__version__
parser.mapper = self.mapper
if len(argv) <= 1:
return parser.print_help()
parser.dispatch(with_exit=True)
parser.catches_help()
parser.catches_version()
def main(self, argv):
parser = Transport(argv, mapper=self.mapper,
options=[], check_help=False,
check_version=False)
parser.parse_args()
parser.catch_help = self.help(parser.subhelp())
parser.catch_version = ceph_installer.__version__
parser.mapper = self.mapper
if len(argv) <= 1:
return parser.print_help()
parser.dispatch()
parser.catches_help()
parser.catches_version()
def main(self, argv):
options = []
parser = Transport(argv, mapper=self.mapper,
options=options, check_help=False,
check_version=False)
parser.parse_args()
parser.catch_help = self.help()
parser.catch_version = rhcephpkg.__version__
parser.mapper = self.mapper
if len(argv) <= 1:
return parser.print_help()
parser.dispatch()
parser.catches_help()
parser.catches_version()
def main(self, argv):
options = [['--log', '--logging']]
parser = Transport(argv, mapper=self.mapper,
options=options, check_help=False,
check_version=False)
parser.parse_args()
merfi.config['verbosity'] = parser.get('--log', 'info')
merfi.config['check'] = parser.has('--check')
parser.catch_help = self.help()
parser.catch_version = merfi.__version__
parser.mapper = self.mapper
if len(argv) <= 1:
return parser.print_help()
parser.dispatch()
parser.catches_help()
parser.catches_version()
def main(self, argv):
options = []
parser = Transport(argv,
mapper=self.mapper,
options=options,
check_help=False,
check_version=False)
parser.parse_args()
parser.catch_help = self.help()
parser.catch_version = rhcephpkg.__version__
parser.mapper = self.mapper
if len(argv) <= 1:
return parser.print_help()
parser.dispatch()
parser.catches_help()
parser.catches_version()
class Clone(object):
help_menu = 'clone a package from dist-git'
_help = """
Clone a package from dist-git. Your SSH key must be set up in Gerrit.
Positional Arguments:
[package] The name of the package to clone.
"""
name = 'clone'
def __init__(self, argv):
self.argv = argv
self.options = []
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
try:
pkg = self.parser.unknown_commands[0]
except IndexError:
return self.parser.print_help()
self._run(pkg)
def help(self):
return self._help
def _run(self, pkg):
""" Clone a package from dist-git. """
if os.path.exists(pkg):
raise SystemExit('%s already exists in current working directory.',
pkg)
configp = util.config()
try:
user = configp.get('rhcephpkg', 'user')
gitbaseurl = configp.get('rhcephpkg', 'gitbaseurl')
except configparser.Error as err:
raise SystemExit('Problem parsing .rhcephpkg.conf: %s',
err.message)
# TODO: SafeConfigParser might make the "user" interpolation here
# unnecessary? Need to test, particularly what it does to %(module).
pkg_url = gitbaseurl % {'user': user, 'module': pkg}
cmd = ['git', 'clone', pkg_url]
subprocess.check_call(cmd)
def main(self, argv):
parser = Transport(argv, mapper=self.mapper,
check_help=False,
check_version=False)
parser.parse_args()
parser.catch_help = self.help()
parser.catch_version = wari.__version__
parser.mapper = self.mapper
if len(argv) <= 1:
return parser.print_help()
# create the connection and set the collection
conn = wari.db.get_connection()
wari.db.connection = conn
wari.db.collection = conn['wari']
parser.dispatch()
parser.catches_help()
parser.catches_version()
conn.close()
def main(self):
options = ['--ignore']
parser = Transport(self.argv, options=options, check_version=False)
parser.catch_help = self._help()
parser.parse_args()
if len(self.argv) < 1:
return parser.print_help()
# populate the nodes metadata with the configured nodes
for daemon in ceph_medic.config['nodes'].keys():
ceph_medic.metadata['nodes'][daemon] = []
for daemon, nodes in ceph_medic.config['nodes'].items():
for node in nodes:
ceph_medic.metadata['nodes'][daemon].append(
{'host': node['host']})
collector.collect()
test = runner.Runner()
results = test.run()
runner.report(results)
class Clone(object):
help_menu = 'clone a package from dist-git'
_help = """
Clone a package from dist-git. Your SSH key must be set up in Gerrit.
Positional Arguments:
[package] The name of the package to clone.
Python packages are named slightly differently between RHEL and Debian.
If you pass a package name "python-foo" to this command, rhcephpkg will strip
off the "python-" prefix and operate on a Debian package name "foo".
"""
name = 'clone'
def __init__(self, argv):
self.argv = argv
self.options = []
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
try:
pkg = self.parser.unknown_commands[0]
except IndexError:
return self.parser.print_help()
self._run(pkg)
def help(self):
return self._help
def _run(self, pkg):
""" Clone a package from dist-git. """
if os.path.exists(pkg):
err = '%s already exists in current working directory.' % pkg
raise SystemExit(err)
configp = util.config()
try:
user = configp.get('rhcephpkg', 'user')
gitbaseurl = configp.get('rhcephpkg', 'gitbaseurl')
except configparser.Error as err:
raise SystemExit('Problem parsing .rhcephpkg.conf: %s',
err.message)
# If we were given an RPM pkg name, switch to the Debian one:
if pkg.startswith('python-'):
pkg = pkg[7:]
# TODO: SafeConfigParser might make the "user" interpolation here
# unnecessary? Need to test, particularly what it does to %(module).
pkg_url = gitbaseurl % {'user': user, 'module': pkg}
cmd = ['git', 'clone', pkg_url]
subprocess.check_call(cmd)
os.chdir(pkg)
patches_url = find_patches_url(configp, user, pkg)
if patches_url:
cmd = ['git', 'remote', 'add', '-f', 'patches', patches_url]
subprocess.check_call(cmd)
util.setup_pristine_tar_branch()
def main(self, argv):
options = [
'--cluster',
'--ssh-config',
'--inventory',
'--config',
'--verbosity',
]
parser = Transport(argv,
options=options,
check_help=False,
check_version=False)
parser.parse_args()
self.config_path = parser.get('--config', configuration.location())
# load medic configuration
loaded_config = configuration.load(
path=parser.get('--config', self.config_path))
# this is the earliest we can have enough config to setup logging
log.setup(loaded_config)
ceph_medic.config.file = loaded_config
global_options = dict(ceph_medic.config.file._sections['global'])
# SSH config
ceph_medic.config.ssh_config = parser.get(
'--ssh-config', global_options.get('--ssh-config'))
if ceph_medic.config.ssh_config:
ssh_config_path = ceph_medic.config.ssh_config
if not os.path.exists(ssh_config_path):
terminal.error("the given ssh config path does not exist: %s" %
ssh_config_path)
sys.exit()
ceph_medic.config.cluster_name = parser.get('--cluster', 'ceph')
ceph_medic.metadata['cluster_name'] = 'ceph'
# Deployment Type
deployment_type = ceph_medic.config.file.get_safe(
'global', 'deployment_type', 'baremetal')
if deployment_type in ['kubernetes', 'openshift', 'k8s', 'oc']:
pod_hosts = hosts.container_platform(deployment_type)
ceph_medic.config.nodes = pod_hosts
ceph_medic.config.hosts_file = ':memory:'
self.hosts_file = ':memory:'
else:
# Hosts file
self.hosts_file = parser.get('--inventory',
configuration.get_host_file())
# find the hosts files, by the CLI first, fallback to the configuration
# file, and lastly if none of those are found or defined, try to load
# from well known locations (cwd, and /etc/ansible/)
loaded_hosts = configuration.load_hosts(
parser.get('--inventory',
global_options.get('--inventory', self.hosts_file)))
ceph_medic.config.nodes = loaded_hosts.nodes
ceph_medic.config.hosts_file = loaded_hosts.filename
self.hosts_file = loaded_hosts.filename
parser.catch_version = ceph_medic.__version__
parser.mapper = self.mapper
parser.catch_help = self.help(parser.subhelp())
if len(argv) <= 1:
return parser.print_help()
ceph_medic.config.config_path = self.config_path
parser.dispatch()
parser.catches_help()
parser.catches_version()
# Verbosity
verbosity = parser.get('--verbosity', 'debug')
ceph_medic.config.verbosity = verbosity.lowercase()
class MergePatches(object):
help_menu = 'Merge patches from RHEL -patches branch to patch-queue branch'
_help = """
Fetch the latest patches branch that rdopkg uses, and then fast-forward merge
that into our local patch-queue branch, so that both branches align.
This command helps to align the patch series between our RHEL packages and our
Ubuntu packages.
Options:
--force Do a hard reset, rather than restricting to fast-forward merges
only. Use this option if the RHEL patches branch was amended or
rebased for some reason.
"""
name = 'merge-patches'
def __init__(self, argv):
self.argv = argv
self.options = ['--force', '--hard-reset']
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
force = False
if self.parser.has(['--force', '--hard-reset']):
force = True
if self.parser.unknown_commands:
log.error('unknown option %s',
' '.join(self.parser.unknown_commands))
return self.parser.print_help()
self._run(force)
def help(self):
return self._help
def _run(self, force=False):
# Determine the names of the relevant branches
current_branch = util.current_branch()
debian_branch = util.current_debian_branch()
patch_queue_branch = util.current_patch_queue_branch()
rhel_patches_branch = self.get_rhel_patches_branch(debian_branch)
# Do the merge
if current_branch == patch_queue_branch:
# HEAD is our patch-queue branch. Use "git pull" directly.
# For example: "git pull --ff-only patches/ceph-2-rhel-patches"
cmd = ['git', 'pull', '--ff-only',
'patches/' + rhel_patches_branch]
if force:
# Do a hard reset on HEAD instead.
cmd = ['git', 'reset', '--hard',
'patches/' + rhel_patches_branch]
else:
# HEAD is our debian branch. Use "git fetch" to update the
# patch-queue ref. For example:
# "git fetch . \
# patches/ceph-2-rhel-patches:patch-queue/ceph-2-ubuntu"
util.ensure_patch_queue_branch()
cmd = ['git', 'fetch', '.',
'patches/%s:%s' % (rhel_patches_branch, patch_queue_branch)]
if force:
# Do a hard push (with "+") instead.
cmd = ['git', 'push', '.', '+patches/%s:%s' %
(rhel_patches_branch, patch_queue_branch)]
log.info(' '.join(cmd))
subprocess.check_call(cmd)
def get_rhel_patches_branch(self, debian_branch):
"""
Get the RHEL -patches branch corresponding to this debian branch.
Examples:
ceph-2-ubuntu -> ceph-2-rhel-patches
ceph-2-trusty -> ceph-2-rhel-patches
ceph-2-xenial -> ceph-2-rhel-patches
ceph-1.3-ubuntu -> ceph-1.3-rhel-patches
ceph-2-ubuntu-hotfix-bz123 -> ceph-2-rhel-patches-hotfix-bz123
"""
(product, version, distro) = debian_branch.split('-', 2)
suffix = None
if '-' in distro:
(distro, suffix) = distro.split('-', 1)
rhel = '%s-%s-rhel-patches' % (product, version)
if suffix is not None:
rhel = '%s-%s' % (rhel, suffix)
return rhel
class CheckoutFromPatches(object):
help_menu = 'Choose a Debian branch based on a RHEL -patches branch'
_help = """
Check out the Git branch that corresponds to a given RHEL (rdopkg-style)
-patches branch.
If you are starting from a RHEL -patches branch name (say, from a trigger in
Jenkins), this will automatically choose the right Debian branch that goes
with your -patches branch.
Example:
rhcephpkg checkout-from-patches ceph-3.0-rhel-patches
... this will checkout the "ceph-3.0-xenial" Git branch if it exists in the
"origin" remote, or fall back to the "ceph-3.0-ubuntu" branch, or error if
neither exist.
Positional Arguments:
[branch] The name of the -patches branch.
"""
name = 'checkout-from-patches'
def __init__(self, argv):
self.argv = argv
self.options = []
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
try:
patches_branch = self.parser.unknown_commands[0]
except IndexError:
return self.parser.print_help()
self._run(patches_branch)
def help(self):
return self._help
def _run(self, patches_branch):
debian_branch = self.get_debian_branch(patches_branch)
if not debian_branch:
err = 'could not find debian branch for %s' % patches_branch
raise SystemExit(err)
cmd = ['git', 'checkout', debian_branch]
log.info(' '.join(cmd))
subprocess.check_call(cmd)
def get_debian_branch(self, patches_branch):
"""
Get the debian branch corresponding to this RHEL -patches branch.
Examples:
ceph-2-rhel-patches -> ceph-2-xenial or ceph-2-ubuntu
ceph-2-rhel-patches-hotfix-bz123 -> ceph-2-ubuntu-hotfix-bz123
:returns: name of debian branch, or None if none was found.
"""
patches_re = re.compile('-rhel-patches')
debian_re = patches_re.sub('-([a-z]+)', patches_branch)
ubuntu_branch = None
for branch in self.get_origin_branches():
m = re.match('^%s$' % debian_re, branch)
if m:
if m.group(1) == 'ubuntu':
# Use this only if we could find no other distro branch.
ubuntu_branch = branch
else:
return branch
return ubuntu_branch
def get_origin_branches(self):
""" Return a list of all the branches in the "origin" remote. """
cmd = ['git', 'branch', '-r', '--list', 'origin/*']
output = subprocess.check_output(cmd)
if six.PY3:
output = output.decode('utf-8')
lines = output.split("\n")
branches = [line.strip()[7:] for line in lines]
return branches
class Download(object):
help_menu = 'download a build from chacra'
_help = """
Download a build's entire artifacts from chacra.
Positional Arguments:
[build] The name of the build to download, eg. "ceph_10.2.0-2redhat1trusty"
"""
name = 'download'
def __init__(self, argv):
self.argv = argv
self.options = []
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
try:
build = self.parser.unknown_commands[0]
except IndexError:
return self.parser.print_help()
self._run(build)
def help(self):
return self._help
def _run(self, build):
configp = util.config()
try:
base_url = configp.get('rhcephpkg.chacra', 'url')
except configparser.Error as err:
raise SystemExit('Problem parsing .rhcephpkg.conf: %s',
err.message)
try:
(pkg, version) = build.split('_')
except ValueError:
log.error('%s is not a valid package build N-V-R' % build)
return self.parser.print_help()
build_url = posixpath.join(base_url, 'binaries/', pkg, version,
'ubuntu', 'all')
log.info('searching %s for builds' % build_url)
build_response = urlopen(Request(build_url))
headers = build_response.headers
if six.PY2:
encoding = headers.getparam('charset') or 'utf-8'
# if encoding is None:
# encoding = 'utf-8'
else:
encoding = headers.get_content_charset(failobj='utf-8')
payload = json.loads(build_response.read().decode(encoding))
for arch, binaries in six.iteritems(payload):
for binary in binaries:
if os.path.isfile(binary):
# TODO: check the sha256sum of the already-downloaded file
# here?
log.info('skipping %s' % binary)
continue
log.info('downloading %s' % binary)
binary_url = posixpath.join(build_url, arch, binary) + '/'
response = urlopen(Request(binary_url))
with open(binary, 'wb') as fp:
shutil.copyfileobj(response, fp)
class MergePatches(object):
help_menu = 'Merge patches from RHEL -patches branch to patch-queue branch'
_help = """
Fetch the latest patches branch that rdopkg uses, and then fast-forward merge
that into our local patch-queue branch, so that both branches align.
This command helps to align the patch series between our RHEL packages and our
Ubuntu packages.
Options:
--force Do a hard reset, rather than restricting to fast-forward merges
only. Use this option if the RHEL patches branch was amended or
rebased for some reason.
"""
name = 'merge-patches'
def __init__(self, argv):
self.argv = argv
self.options = ['--force', '--hard-reset']
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
force = False
if self.parser.has(['--force', '--hard-reset']):
force = True
if self.parser.unknown_commands:
log.error('unknown option %s',
' '.join(self.parser.unknown_commands))
return self.parser.print_help()
self._run(force)
def help(self):
return self._help
def _run(self, force=False):
# Determine the names of the relevant branches
current_branch = util.current_branch()
debian_branch = util.current_debian_branch()
patches_branch = util.current_patches_branch()
rhel_patches_branch = self.get_rhel_patches_branch(debian_branch)
# Do the merge
if current_branch == patches_branch:
# HEAD is our patch-queue branch. Use "git pull" directly.
# For example: "git pull --ff-only patches/ceph-2-rhel-patches"
cmd = ['git', 'pull', '--ff-only',
'patches/' + rhel_patches_branch]
if force:
# Do a hard reset on HEAD instead.
cmd = ['git', 'reset', '--hard',
'patches/' + rhel_patches_branch]
else:
# HEAD is our debian branch. Use "git fetch" to update the
# patch-queue ref. For example:
# "git fetch . \
# patches/ceph-2-rhel-patches:patch-queue/ceph-2-ubuntu"
cmd = ['git', 'fetch', '.',
'patches/%s:%s' % (rhel_patches_branch, patches_branch)]
if force:
# Do a hard push (with "+") instead.
cmd = ['git', 'push', '.', '+patches/%s:%s' %
(rhel_patches_branch, patches_branch)]
log.info(' '.join(cmd))
subprocess.check_call(cmd)
def get_rhel_patches_branch(self, debian_branch):
"""
Get the RHEL -patches branch corresponding to this debian branch.
Examples:
ceph-2-ubuntu -> ceph-2-rhel-patches
ceph-2-trusty -> ceph-2-rhel-patches
ceph-2-xenial -> ceph-2-rhel-patches
ceph-1.3-ubuntu -> ceph-1.3-rhel-patches
ceph-2-ubuntu-hotfix-bz123 -> ceph-2-rhel-patches-hotfix-bz123
"""
(product, version, distro) = debian_branch.split('-', 2)
suffix = None
if '-' in distro:
(distro, suffix) = distro.split('-', 1)
rhel = '%s-%s-rhel-patches' % (product, version)
if suffix is not None:
rhel = '%s-%s' % (rhel, suffix)
return rhel
class WatchBuild(object):
help_menu = 'watch a build-package job in Jenkins'
_help = """
Watch a particular build-package job in Jenkins.
Positional Arguments:
[id] The build-package job ID to watch
For example: "rhcephpkg watch-build 328"
"""
name = 'watch-build'
def __init__(self, argv):
self.argv = argv
self.options = []
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
try:
build_number = int(self.parser.unknown_commands[0])
except (IndexError, ValueError):
return self.parser.print_help()
self.watch(build_number)
def help(self):
return self._help
def watch(self, build_number):
jenkins = util.jenkins_connection()
build_info = jenkins.get_build_info('build-package', build_number)
job_url = posixpath.join(jenkins.url, 'job', 'build-package',
str(build_number))
log.info('Watching %s' % job_url)
pkg_name = self.pkg_name(build_info)
start_seconds = build_info['timestamp'] / 1000.0
# rcm-jenkins is uses the America/New_York timezone:
jenkins_tz = tz.gettz('America/New_York')
start = datetime.fromtimestamp(start_seconds, jenkins_tz)
# If you want to convert to local time:
# start = start.astimezone(tz.tzlocal())
log.info('Started %s' % start.strftime("%F %r %z"))
was_building = build_info['building']
while build_info['building']:
try:
elapsed = datetime.now(jenkins_tz) - start
# TODO: Xenial has python-humanize (humanize.naturaldelta()
# here)
(minutes, seconds) = divmod(elapsed.total_seconds(), 60)
# Clear the previous line:
msg = '\r%s building for %02d:%02d' % \
(pkg_name, minutes, seconds)
sys.stdout.write(msg)
sys.stdout.flush()
sleep(10)
build_info = jenkins.get_build_info('build-package',
build_number)
except requests.exceptions.ConnectionError as e:
print('')
log.error('connection error: %s' % e)
log.info('Re-try watching with `rhcephpkg watch-build %s`' %
build_number)
except KeyboardInterrupt:
print('')
log.info('continue watching with `rhcephpkg watch-build %s`' %
build_number)
raise SystemExit(1)
if was_building:
# The above "while" loop will not print a final newline.
print('')
end_millis = build_info['timestamp'] + build_info['duration']
end_seconds = end_millis / 1000.0
end = datetime.fromtimestamp(end_seconds, jenkins_tz)
log.info('Ended %s' % end.strftime("%F %r %z"))
# Show the final build result.
if build_info['result'] == 'SUCCESS':
log.info('result is SUCCESS')
else:
log.error(build_info['result'])
raise SystemExit(1)
def pkg_name(self, build_info):
""" Return a package name based on this build's information.
:param build_info: ``dict`` from python-jenkins' get_build_info()
:returns: ``str``, for example "ceph" or "ceph-ansible".
"""
pkg_name = None
for action in build_info['actions']:
if action.get('_class', '') == 'hudson.model.ParametersAction':
for parameter in action['parameters']:
if parameter['name'] == 'PKG_NAME':
pkg_name = parameter['value']
if pkg_name is None:
# Maybe the Jenkins job was badly mis-configured or something.
# This will probably never happen, but let's raise defensively.
raise RuntimeError('could not find pkg name in %s' % build_info)
return pkg_name
class Localbuild(object):
help_menu = 'build a package on the local system'
_help = """
Build a package on the local system, using pbuilder.
Options:
--dist "xenial" or "trusty". If unspecified, rhcephpkg will choose one
based on the current branch's name.
Rules for automatic distro selection:
1) If the branch suffix is an ubuntu distro name, use that.
eg "ceph-3.0-xenial".
2) If a branch has a version number starting with "1.3", return "trusty".
eg. "ceph-1.3-ubuntu"
3) If a branch has a version number starting with "2" return "xenial".
eg. "ceph-2-ubuntu"
4) If a branch has a version number starting with "3" return "xenial".
eg. "ceph-3.0-ubuntu"
5) Otherwise raise, because we need to add more rules.
"""
name = 'localbuild'
def __init__(self, argv):
self.argv = argv
self.options = ('--dist', )
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
# Allow user to override the distro.
if self.parser.has('--dist'):
if self.parser.get('--dist') is None:
raise SystemExit('Specify a distro to --dist')
distro = self.parser.get('--dist')
else:
distro = get_distro()
if self.parser.unknown_commands:
log.error('unknown option %s',
' '.join(self.parser.unknown_commands))
return self.parser.print_help()
self._run(distro)
def help(self):
return self._help
def _run(self, distro):
""" Build a package on the local system, using pbuilder. """
pkg_name = util.package_name()
os.environ['BUILDER'] = 'pbuilder'
j_arg = self._get_j_arg(cpu_count())
pbuilder_cache = '/var/cache/pbuilder/base-%s-amd64.tgz' % distro
setup_pbuilder_cache(pbuilder_cache, distro)
util.setup_pristine_tar_branch()
# TODO: we should also probably check parent dir for leftovers and warn
# the user to delete them (or delete them ourselves?)
cmd = [
'gbp', 'buildpackage',
'--git-dist=%s' % distro, '--git-arch=amd64', '--git-verbose',
'--git-pbuilder', j_arg, '-us', '-uc'
]
log.info('building %s with pbuilder', pkg_name)
subprocess.check_call(cmd)
def _get_j_arg(self, cpus, total_ram_gb=None):
"""
Returns a string like "-j4" or "-j8". j is the number of processors,
with a maximum of x, where x = TOTAL_RAM_GB / 4.
We want to use all our processors (a high "j" value), but the build
process will fail with an "out of memory" error out if this j value is
too high.
An 8 GB system would have a maximum of -j2
A 16 GB system would have a maximum of -j4
A 32 GB system would have a maximum of -j8
"""
if total_ram_gb is None:
page_size = os.sysconf('SC_PAGE_SIZE')
mem_bytes = page_size * os.sysconf('SC_PHYS_PAGES')
# mem_gib is a decimal, eg. 7.707 on 8GB system
mem_gib = mem_bytes / (1024.**3)
# Round up to the nearest GB for our purposes.
total_ram_gb = math.ceil(mem_gib)
number = min(cpus, total_ram_gb / 4)
return '-j%d' % max(number, 1)
class WatchBuild(object):
help_menu = 'watch a build-package job in Jenkins'
_help = """
Watch a particular build-package job in Jenkins.
Positional Arguments:
[id] The build-package job ID to watch
For example: "rhcephpkg watch-build 328"
"""
name = 'watch-build'
def __init__(self, argv):
self.argv = argv
self.options = []
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
try:
build_number = int(self.parser.unknown_commands[0])
except (IndexError, ValueError):
return self.parser.print_help()
self.watch(build_number)
def help(self):
return self._help
def watch(self, build_number):
jenkins = util.jenkins_connection()
build_info = jenkins.get_build_info('build-package', build_number)
job_url = posixpath.join(jenkins.url, 'job', 'build-package',
str(build_number))
log.info('Watching %s' % job_url)
pkg_name = self.pkg_name(build_info)
start_seconds = build_info['timestamp'] / 1000.0
# rcm-jenkins is uses the America/New_York timezone:
jenkins_tz = tz.gettz('America/New_York')
start = datetime.fromtimestamp(start_seconds, jenkins_tz)
# If you want to convert to local time:
# start = start.astimezone(tz.tzlocal())
log.info('Started %s' % start.strftime("%F %r %z"))
while build_info['building']:
elapsed = datetime.now(jenkins_tz) - start
# TODO: Xenial has python-humanize (humanize.naturaldelta() here)
# Backport the python-humanize package for Trusty? Or drop Trusty?
(minutes, seconds) = divmod(elapsed.total_seconds(), 60)
# Clear the previous line:
msg = '\r%s building for %02d:%02d' % (pkg_name, minutes, seconds)
sys.stdout.write(msg)
sys.stdout.flush()
sleep(10)
build_info = jenkins.get_build_info('build-package', build_number)
end_millis = build_info['timestamp'] + build_info['duration']
end_seconds = end_millis / 1000.0
end = datetime.fromtimestamp(end_seconds, jenkins_tz)
log.info('Ended %s' % end.strftime("%F %r %z"))
# Show the final build result.
if build_info['result'] == 'SUCCESS':
log.info('result is SUCCESS')
else:
log.error(build_info['result'])
raise SystemExit(1)
def pkg_name(self, build_info):
""" Return a package name based on this build's information.
:param build_info: ``dict`` from python-jenkins' get_build_info()
:returns: ``str``, for example "ceph" or "ceph-ansible".
"""
pkg_name = None
for action in build_info['actions']:
if action.get('_class', '') == 'hudson.model.ParametersAction':
for parameter in action['parameters']:
if parameter['name'] == 'PKG_NAME':
pkg_name = parameter['value']
if pkg_name is None:
# Maybe the Jenkins job was badly mis-configured or something.
# This will probably never happen, but let's raise defensively.
raise RuntimeError('could not find pkg name in %s' % build_info)
return pkg_name
