class Hello(object):
help_menu = 'test authentication to Jenkins'
_help = """
Test authentication to Jenkins and return your user's fullName attribute.
"""
name = 'hello'
def __init__(self, argv):
self.argv = argv
self.options = []
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
self._run()
def help(self):
return self._help
def _run(self):
""" Authenticate to Jenkins and print our username to STDOUT.
Useful for checking that our authentication credentials are
correct. """
jenkins = util.jenkins_connection()
data = jenkins.get_whoami()
name = data['fullName'] # Our Jenkins instance gets this from LDAP
jenkins_version = jenkins.get_version()
print('Hello %s from Jenkins %s' % (name, jenkins_version))
print('Logged in to %s' % jenkins.url)
class Source(object):
help_menu = 'build a source package on the local system'
_help = """
Build a source package on the local system.
"""
name = 'source'
def __init__(self, argv):
self.argv = argv
def main(self):
self.parser = Transport(self.argv)
self.parser.catch_help = self.help()
self.parser.parse_args()
self._run()
def help(self):
return self._help
def _run(self):
""" Build a source package on the local system. """
util.setup_pristine_tar_branch()
cmd = ['gbp', 'buildpackage', '--git-tag', '--git-retag', '-S',
'-us', '-uc']
log.info(' '.join(cmd))
subprocess.check_call(cmd)
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self._help
self.parser.parse_args()
self.force = self.parser.has('--force')
# handle posting binaries:
if self.parser.has('create'):
url_part = self.sanitize_url(self.parser.get('create'))
if not sys.stdin.isatty():
# read from stdin
logger.info('reading input from stdin')
for line in sys.stdin.readlines():
filename = self.sanitize_filename(line)
if not filename:
continue
url = os.path.join(self.base_url, url_part)
self.post(url, filename)
else:
filepath = self.sanitize_filename(self.argv[-1])
if not filepath:
logger.warning('provided path does not exist: %s',
self.argv[-1])
return
url = os.path.join(self.base_url, url_part)
self.post(url, filepath)
elif self.parser.has('delete'):
if self.parser.get('delete') is None:
raise SystemExit('Specify a URL to delete a binary.')
url_part = self.sanitize_url(self.parser.get('delete'))
url = os.path.join(self.base_url, url_part)
self.delete(url)
class Source(object):
help_menu = 'build a source package on the local system'
_help = """
Build a source package on the local system.
"""
name = 'source'
def __init__(self, argv):
self.argv = argv
def main(self):
self.parser = Transport(self.argv)
self.parser.catch_help = self.help()
self.parser.parse_args()
self._run()
def help(self):
return self._help
def _run(self):
""" Build a source package on the local system. """
util.setup_pristine_tar_branch()
cmd = [
'gbp', 'buildpackage', '--git-tag', '--git-retag', '-S', '-us',
'-uc'
]
log.info(' '.join(cmd))
subprocess.check_call(cmd)
def main(self):
parser = Transport(self.arguments,
options=self.options,
check_help=True)
parser.catch_help = self._help
parser.parse_args()
parser.catches_help()
branch = parser.get('--branch', 'master')
user = parser.get('--user', 'vagrant')
high_verbosity = '-vvvv' if parser.has('-vvvv') else '-v'
if not parser.unknown_commands:
log.error(
"it is required to pass a host to deploy to, but none was provided"
)
raise SystemExit(1)
command = [
"ansible-playbook",
"-i",
"%s," % parser.unknown_commands[-1],
high_verbosity,
"-u",
user,
"--extra-vars",
'branch=%s' % branch,
"deploy.yml",
]
log.debug("Running command: %s" % ' '.join(command))
out, err, code = process.run(command, cwd=playbook_path)
log.error(err)
log.debug(out)
class Build(object):
help_menu = 'build a package in Jenkins'
_help = """
Build a package in Jenkins.
"""
name = 'build'
def __init__(self, argv):
self.argv = argv
self.options = []
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
self._run()
def help(self):
return self._help
def _run(self):
""" Build a package in Jenkins. """
pkg_name = util.package_name()
branch_name = util.current_branch()
jenkins = util.jenkins_connection()
if branch_name.startswith('patch-queue/'):
log.error('%s is a patch-queue branch' % branch_name)
msg = 'You can switch to the debian branch with "gbp pq switch"'
raise SystemExit(msg)
log.info('building %s branch %s at %s', pkg_name, branch_name,
posixpath.join(jenkins.url, 'job', 'build-package'))
job_params = {'PKG_NAME': pkg_name, 'BRANCH': branch_name}
queue_number = jenkins.build_job('build-package',
parameters=job_params,
token=jenkins.password)
# Job is now queued, not yet running.
log.info('Waiting for build queue #%d' % queue_number)
log.info('This may be safely interrupted...')
queue_item = jenkins.get_queue_item(queue_number)
while 'executable' not in queue_item:
try:
log.info('queue state: %s' % queue_item['why'])
sleep(2)
queue_item = jenkins.get_queue_item(queue_number)
except KeyboardInterrupt:
# We have no build_number, so just print a general message with
# a basic URL for the user to check.
print('')
print('Build is queued for starting at %s' % jenkins.url)
raise SystemExit(1)
# Job is now running.
build_number = queue_item['executable']['number']
# Pass the rest over to the "watch-build" command.
watcher = WatchBuild(['watch'])
watcher.watch(build_number)
def main(self):
parser = Transport(self.argv, check_help=False)
parser.catch_help = self.help()
parser.catches_help()
if util.build_is_triggered():
print(
'Build has been triggered via Github, will skip setting status'
)
return
print('Build has not been triggered via Github')
print('Assuming manual job execution, will set status')
required_env_vars = [
'GITHUB_REPOSITORY', 'GITHUB_SHA', 'GITHUB_OAUTH_TOKEN',
'GITHUB_STATUS_CONTEXT', 'GITHUB_STATUS_STARTED',
'GITHUB_STATUS_SUCCESS', 'GITHUB_STATUS_FAILURE',
'GITHUB_STATUS_ERROR', 'GITHUB_STATUS_STATE', 'BUILD_URL'
]
missing_envs = []
for env in required_env_vars:
if not conf['env'].get(env):
missing_envs.append(env)
if missing_envs:
print('Will skip setting status')
print('Environment variable(s) required but not provided:')
for env in missing_envs:
print('\t ' + env)
return
self.update()
def parse_args(self):
parser = Transport(self.argv, options=['--socket-location'])
parser.catch_help = self._help
parser.parse_args()
location = parser.get('--socket-location') or '/tmp/pytest.sock'
delgado.config['allowed'] = ['py.test']
engine = Engine(socket_location=location)
engine.run_forever()
def parse_args(self):
options = ['--allowed']
parser = Transport(self.argv, options=options)
parser.catch_help = self._help
parser.parse_args()
delgado.config['allowed'] = parser.get('--allowed') or []
engine = Engine(connection=self.connection)
engine.run_forever()
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
try:
pkg = self.parser.unknown_commands[0]
except IndexError:
return self.parser.print_help()
self._run(pkg)
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
try:
build_number = int(self.parser.unknown_commands[0])
except (IndexError, ValueError):
return self.parser.print_help()
self.watch(build_number)
class Repo(object):
_help = dedent("""
Operate on repositories on a remote chacra instance. Both `recreate` and
`update` calls are not immediate. They rely on the async service managing
repos which usually have a delay applied to them.
Options:
recreate Mark a repository to be removed and created from scratch
again.
update Repository will get updated by running the repo tools on
it again.
""")
help_menu = "recreate, delete, or update repositories"
options = ['recreate', 'update']
def __init__(self, argv):
self.argv = argv
@property
def base_url(self):
return os.path.join(
chacractl.config['url'], 'repos'
)
@catches(requests.exceptions.HTTPError, handler=requests_errors)
@retry()
def post(self, url):
exists = requests.head(
url,
auth=chacractl.config['credentials'],
verify=chacractl.config['ssl_verify'])
exists.raise_for_status()
logger.info('POST: %s', url)
response = requests.post(
url,
auth=chacractl.config['credentials'],
verify=chacractl.config['ssl_verify'])
response.raise_for_status()
json = response.json()
for k, v in json.items():
logger.info("%s: %s", k, v)
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self._help
self.parser.parse_args()
recreate = self.parser.get('recreate')
update = self.parser.get('update')
if recreate:
url_part = os.path.join(recreate, 'recreate')
url = os.path.join(self.base_url, url_part)
self.post(url)
elif update:
url_part = os.path.join(update, 'update')
url = os.path.join(self.base_url, url_part)
self.post(url)
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
try:
tarball = self.parser.unknown_commands[0]
except IndexError:
tarball = None
bugstr = self.parser.get('--bug')
self._run(tarball, bugstr)
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self._help
self.parser.parse_args()
if self.parser.unknown_commands:
url_part = self.sanitize_url(self.parser.unknown_commands[-1])
url = os.path.join(self.base_url, url_part)
return self.head(url)
else:
logger.error('no url was passed in')
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
force = False
if self.parser.has(['--force', '--hard-reset']):
force = True
if self.parser.unknown_commands:
log.error('unknown option %s',
' '.join(self.parser.unknown_commands))
return self.parser.print_help()
self._run(force)
class Hello(object):
help_menu = 'test authentication to Jenkins'
_help = """
Test authentication to Jenkins and return your user's fullName attribute.
"""
name = 'hello'
def __init__(self, argv):
self.argv = argv
self.options = []
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
self._run()
def help(self):
return self._help
def _run(self):
""" Authenticate to Jenkins and print our username to STDOUT.
Useful for checking that our authentication credentials are
correct. """
jenkins = util.jenkins_connection()
# python-jenkins does not have syntactic support for "whoami" (the
# "/me/api/json" endpoint), so we have to hit it and parse it
# ourselves.
# https://review.openstack.org/307896
whoami_url = posixpath.join(jenkins.url, 'me/api/json')
try:
response = jenkins.jenkins_open(Request(whoami_url))
data = json.loads(response)
except JenkinsException as err:
raise SystemExit(err)
name = data['fullName'] # Our Jenkins instance gets this from LDAP
try:
jenkins_version = jenkins.get_version()
except AttributeError:
# python-jenkins older than 0.4.1 does not have get_version().
version_url = jenkins.server
try:
response = urlopen(Request(version_url))
if six.PY2:
jenkins_version = response.info().getheader('X-Jenkins')
else:
jenkins_version = response.getheader('X-Jenkins')
except (HTTPError, BadStatusLine) as err:
raise SystemExit(err)
print('Hello %s from Jenkins %s' % (name, jenkins_version))
def parse_args(self, argv=None):
""" pass argv during testing """
if argv is None:
argv = self.argv
options = [['--output', '-o']]
parser = Transport(argv, options=options)
parser.catch_help = self.help()
parser.parse_args()
self.source = util.infer_path(parser.unknown_commands)
self.output = parser.get('--output', self.source + '-dvd.iso')
self.check_dependency()
self.make_iso()
self.make_sha256sum()
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self._help
self.parser.parse_args()
recreate = self.parser.get('recreate')
update = self.parser.get('update')
if recreate:
url_part = os.path.join(recreate, 'recreate')
url = os.path.join(self.base_url, url_part)
self.post(url)
elif update:
url_part = os.path.join(update, 'update')
url = os.path.join(self.base_url, url_part)
self.post(url)
class Exists(object):
_help = dedent("""
Check if a given URL part exists already. Mainly does a HEAD request to the
given endpoint. If the URL does not exist it will return a non-zero exit
status (404).
For example:
chacractl exists binaries/ceph-deploy/master/debian/wheezy
Positional Arguments:
[URL] The endpoint, starting with the full url part (sans fqdn)
""")
help_menu = "check if a given URL part exists already"
options = []
def __init__(self, argv):
self.argv = argv
self.base_url = chacractl.config['url']
def sanitize_url(self, url_part):
# get rid of the leading slash to prevent issues when joining
url = url_part.lstrip('/')
# and add a trailing slash so that the request is done at the correct
# canonical url
if not url.endswith('/'):
url = "%s/" % url
return url
@catches(requests.exceptions.HTTPError, handler=requests_errors)
@retry()
def head(self, url):
logger.info('HEAD: %s', url)
exists = requests.head(url,
auth=chacractl.config['credentials'],
verify=chacractl.config['ssl_verify'])
exists.raise_for_status()
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self._help
self.parser.parse_args()
if self.parser.unknown_commands:
url_part = self.sanitize_url(self.parser.unknown_commands[-1])
url = os.path.join(self.base_url, url_part)
return self.head(url)
else:
logger.error('no url was passed in')
class Repo(object):
_help = dedent(
"""
Operate on repositories on a remote chacra instance. Both `recreate` and
`update` calls are not immediate. They rely on the async service managing
repos which usually have a delay applied to them.
Options:
recreate Mark a repository to be removed and created from scratch
again.
update Repository will get updated by running the repo tools on
it again.
"""
)
help_menu = "recreate, delete, or update repositories"
options = ["recreate", "update"]
def __init__(self, argv):
self.argv = argv
@property
def base_url(self):
return os.path.join(chacractl.config["url"], "repos")
@catches(requests.exceptions.HTTPError, handler=requests_errors)
def post(self, url):
exists = requests.head(url, auth=chacractl.config["credentials"], verify=chacractl.config["ssl_verify"])
exists.raise_for_status()
logger.info("POST: %s", url)
response = requests.post(url, auth=chacractl.config["credentials"], verify=chacractl.config["ssl_verify"])
response.raise_for_status()
json = response.json()
for k, v in json.items():
logger.info("%s: %s", k, v)
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self._help
self.parser.parse_args()
recreate = self.parser.get("recreate")
update = self.parser.get("update")
if recreate:
url_part = os.path.join(recreate, "recreate")
url = os.path.join(self.base_url, url_part)
self.post(url)
elif update:
url_part = os.path.join(update, "update")
url = os.path.join(self.base_url, url_part)
self.post(url)
class Exists(object):
_help = dedent("""
Check if a given URL part exists already. Mainly does a HEAD request to the
given endpoint. If the URL does not exist it will return a non-zero exit
status (404).
For example:
chacractl exists binaries/ceph-deploy/master/debian/wheezy
Positional Arguments:
[URL] The endpoint, starting with the full url part (sans fqdn)
""")
help_menu = "check if a given URL part exists already"
options = []
def __init__(self, argv):
self.argv = argv
self.base_url = chacractl.config['url']
def sanitize_url(self, url_part):
# get rid of the leading slash to prevent issues when joining
url = url_part.lstrip('/')
# and add a trailing slash so that the request is done at the correct
# canonical url
if not url.endswith('/'):
url = "%s/" % url
return url
@catches(requests.exceptions.HTTPError, handler=requests_errors)
def head(self, url):
logger.info('HEAD: %s', url)
exists = requests.head(
url,
auth=chacractl.config['credentials'],
verify=chacractl.config['ssl_verify'])
exists.raise_for_status()
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self._help
self.parser.parse_args()
if self.parser.unknown_commands:
url_part = self.sanitize_url(self.parser.unknown_commands[-1])
url = os.path.join(self.base_url, url_part)
return self.head(url)
else:
logger.error('no url was passed in')
class BaseBackend(base.BaseCommand):
options = []
parser = None
def parse_args(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
self.path = util.infer_path(self.parser.unknown_commands)
self.check_dependency()
self.sign()
def sign(self):
raise NotImplemented()
class Build(object):
help_menu = "build a package in Jenkins"
_help = """
Build a package in Jenkins.
"""
name = "build"
def __init__(self, argv):
self.argv = argv
self.options = []
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
self._run()
def help(self):
return self._help
def _run(self):
""" Build a package in Jenkins. """
pkg_name = util.package_name()
branch_name = util.current_branch()
jenkins = util.jenkins_connection()
if branch_name.startswith("patch-queue/"):
log.error("%s a patch-queue branch" % branch_name)
msg = 'You can switch to the debian branch with "gbp pq switch"'
raise SystemExit(msg)
log.info(
"building %s branch %s at %s", pkg_name, branch_name, posixpath.join(jenkins.url, "job", "build-package")
)
job_params = {"PKG_NAME": pkg_name, "BRANCH": branch_name}
if self._has_broken_build_job():
jenkins.build_job = types.MethodType(_build_job_fixed, jenkins)
jenkins.build_job("build-package", parameters=job_params, token=jenkins.password)
def _has_broken_build_job(self):
# Ubuntu Trusty ships python-jenkins 0.2.1-0ubuntu1, and this version
# has a broken build_job() method. See
# https://bugs.launchpad.net/bugs/1177831 .
# This bug was fixed in python-jenkins v0.3.2 upstream.
v = get_distribution("python_jenkins").version
return parse_version(v) < parse_version("0.3.2")
class Build(object):
help_menu = 'build a package in Jenkins'
_help = """
Build a package in Jenkins.
"""
name = 'build'
def __init__(self, argv):
self.argv = argv
self.options = []
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
self._run()
def help(self):
return self._help
def _run(self):
""" Build a package in Jenkins. """
pkg_name = util.package_name()
branch_name = util.current_branch()
jenkins = util.jenkins_connection()
if branch_name.startswith('patch-queue/'):
log.error('%s a patch-queue branch' % branch_name)
msg = 'You can switch to the debian branch with "gbp pq switch"'
raise SystemExit(msg)
log.info('building %s branch %s at %s', pkg_name, branch_name,
posixpath.join(jenkins.url, 'job', 'build-package'))
job_params = {'PKG_NAME': pkg_name, 'BRANCH': branch_name}
if self._has_broken_build_job():
jenkins.build_job = types.MethodType(_build_job_fixed, jenkins)
jenkins.build_job('build-package', parameters=job_params,
token=jenkins.password)
def _has_broken_build_job(self):
# Ubuntu Trusty ships python-jenkins 0.2.1-0ubuntu1, and this version
# has a broken build_job() method. See
# https://bugs.launchpad.net/bugs/1177831 .
# This bug was fixed in python-jenkins v0.3.2 upstream.
v = get_distribution('python_jenkins').version
return parse_version(v) < parse_version('0.3.2')
def parse_args(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
self.path = util.infer_path(self.parser.unknown_commands)
self.check_dependency()
self.sign()
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self._help
self.parser.parse_args()
self.force = self.parser.has('--force')
# handle posting binaries:
if self.parser.has('create'):
url_part = self.sanitize_url(self.parser.get('create'))
if not sys.stdin.isatty():
# read from stdin
logger.info('reading input from stdin')
for line in sys.stdin.readlines():
filename = self.sanitize_filename(line)
if not filename:
continue
url = os.path.join(self.base_url, url_part)
self.post(url, filename)
else:
filepath = self.sanitize_filename(self.argv[-1])
if not filepath:
logger.warning(
'provided path does not exist: %s', self.argv[-1]
)
return
url = os.path.join(self.base_url, url_part)
self.post(url, filepath)
elif self.parser.has('delete'):
url_part = self.sanitize_url(self.parser.get('delete'))
url = os.path.join(self.base_url, url_part)
self.delete(url)
def parse_args(self):
transport = Transport(self.argv, check_help=False)
transport.catch_help = self.__doc__
if len(self.argv) <= 1:
transport.print_help()
transport.parse_args()
for action in self.actions:
if transport.has(action):
return self.actions.get(action)()
# If nothing matches, print the help
transport.print_help()
def main(self):
options = ['--ignore']
config_ignores = ceph_medic.config.file.get_list('check', '--ignore')
parser = Transport(self.argv, options=options, check_version=False)
parser.catch_help = self._help()
parser.parse_args()
ignored_codes = as_list(parser.get('--ignore', ''))
# fallback to the configuration if nothing is defined in the CLI
if not ignored_codes:
ignored_codes = config_ignores
if len(self.argv) < 1:
return parser.print_help()
# populate the nodes metadata with the configured nodes
for daemon in ceph_medic.config.nodes.keys():
ceph_medic.metadata['nodes'][daemon] = []
for daemon, nodes in ceph_medic.config.nodes.items():
for node in nodes:
node_metadata = {'host': node['host']}
if 'container' in node:
node_metadata['container'] = node['container']
ceph_medic.metadata['nodes'][daemon].append(node_metadata)
collector.collect()
test = runner.Runner()
test.ignore = ignored_codes
results = test.run()
runner.report(results)
#XXX might want to make this configurable to not bark on warnings for
# example, setting forcefully for now, but the results object doesn't
# make a distinction between error and warning (!)
if results.errors or results.warnings:
sys.exit(1)
class Clone(object):
help_menu = 'clone a package from dist-git'
_help = """
Clone a package from dist-git. Your SSH key must be set up in Gerrit.
Positional Arguments:
[package] The name of the package to clone.
"""
name = 'clone'
def __init__(self, argv):
self.argv = argv
self.options = []
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
try:
pkg = self.parser.unknown_commands[0]
except IndexError:
return self.parser.print_help()
self._run(pkg)
def help(self):
return self._help
def _run(self, pkg):
""" Clone a package from dist-git. """
if os.path.exists(pkg):
raise SystemExit('%s already exists in current working directory.',
pkg)
configp = util.config()
try:
user = configp.get('rhcephpkg', 'user')
gitbaseurl = configp.get('rhcephpkg', 'gitbaseurl')
except configparser.Error as err:
raise SystemExit('Problem parsing .rhcephpkg.conf: %s',
err.message)
# TODO: SafeConfigParser might make the "user" interpolation here
# unnecessary? Need to test, particularly what it does to %(module).
pkg_url = gitbaseurl % {'user': user, 'module': pkg}
cmd = ['git', 'clone', pkg_url]
subprocess.check_call(cmd)
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
try:
build_number = int(self.parser.unknown_commands[0])
except (IndexError, ValueError):
return self.parser.print_help()
self.watch(build_number)
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
try:
pkg = self.parser.unknown_commands[0]
except IndexError:
return self.parser.print_help()
self._run(pkg)
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
# Allow user to override the distro.
if self.parser.has('--dist'):
if self.parser.get('--dist') is None:
raise SystemExit('Specify a distro to --dist')
distro = self.parser.get('--dist')
else:
distro = get_distro()
if self.parser.unknown_commands:
log.error('unknown option %s',
' '.join(self.parser.unknown_commands))
return self.parser.print_help()
self._run(distro)
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self._help
self.parser.parse_args()
if self.parser.unknown_commands:
url_part = self.sanitize_url(self.parser.unknown_commands[-1])
url = os.path.join(self.base_url, url_part)
return self.head(url)
else:
logger.error('no url was passed in')
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
force = False
if self.parser.has(['--force', '--hard-reset']):
force = True
if self.parser.unknown_commands:
log.error('unknown option %s',
' '.join(self.parser.unknown_commands))
return self.parser.print_help()
self._run(force)
def main(self):
options = ['--ignore']
parser = Transport(self.argv, options=options, check_version=False)
parser.catch_help = self._help()
parser.parse_args()
if len(self.argv) < 1:
return parser.print_help()
# populate the nodes metadata with the configured nodes
for daemon in ceph_medic.config.nodes.keys():
ceph_medic.metadata['nodes'][daemon] = []
for daemon, nodes in ceph_medic.config.nodes.items():
for node in nodes:
ceph_medic.metadata['nodes'][daemon].append(
{'host': node['host']})
collector.collect()
test = runner.Runner()
results = test.run()
runner.report(results)
#XXX might want to make this configurable to not bark on warnings for
# example, setting forcefully for now, but the results object doesn't
# make a distinction between error and warning (!)
if results.errors or results.warnings:
sys.exit(1)
def main(self):
options = ['--stdout']
parser = Transport(self.argv, options=options, check_version=False)
parser.catch_help = self._help()
parser.parse_args()
if len(self.argv) == 1:
raise SystemExit(
"A monitor hostname or a ceph.conf file is required as an argument"
)
node = self.argv[-1]
inventory = {}
with get_connection(node) as conn:
report = get_mon_report(conn)
try:
mons = report['monmap']['mons']
except KeyError:
raise SystemExit(report)
inventory['mons'] = [i['name'] for i in mons]
osds = report['osd_metadata']
inventory['osds'] = [i['hostname'] for i in osds]
if not inventory:
raise SystemExit(
'no hosts where found from remote monitor node: %s' % node)
generate_inventory(inventory, to_stdout=parser.get('--stdout'))
conn.exit()
return
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self._help
self.parser.parse_args()
# handle posting projects:
if self.parser.has('create'):
url_part = self.sanitize_url(self.parser.get('create'))
if not sys.stdin.isatty():
# read from stdin
logger.info('reading input from stdin')
for line in sys.stdin.readlines():
url = os.path.join(self.base_url, url_part)
self.post(url)
else:
url = os.path.join(self.base_url, url_part)
self.post(url)
# XXX this exists here but it not yet enabled from the CLI
elif self.parser.has('delete'):
url_part = self.sanitize_url(self.parser.get('delete'))
url = os.path.join(self.base_url, url_part)
self.delete(url)
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self._help
self.parser.parse_args()
recreate = self.parser.get("recreate")
update = self.parser.get("update")
if recreate:
url_part = os.path.join(recreate, "recreate")
url = os.path.join(self.base_url, url_part)
self.post(url)
elif update:
url_part = os.path.join(update, "update")
url = os.path.join(self.base_url, url_part)
self.post(url)
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
# FIXME: stop hardcoding trusty. Use the git branch name instead,
# translating "-ubuntu" into this local computer's own distro.
distro = 'trusty'
# Allow user to override the distro.
if self.parser.has('--dist'):
if self.parser.get('--dist') is None:
raise SystemExit('Specify a distro to --dist')
distro = self.parser.get('--dist')
self._run(distro)
def parse_args(self, argv):
"""
Main method for parsing arguments, it uses whatever `argv` is although
it should always be a list. Once it goes through the ``Transport`` class
it tries to generate the help from the mapped classes and the current
docstring for this module.
If nothing matches it will return the help.
"""
self.get_extend_commands()
transport = Transport(argv, self.mapper)
transport.catch_help = "%s \n%s" % (__doc__, transport.subhelp())
transport.catch_version = "dozo version {0}".format(__version__)
if len(self.argv) <= 1:
transport.print_help()
transport.dispatch()
def main(self):
parser = Transport(self.arguments, options=self.options, check_help=True)
parser.catch_help = self._help
parser.parse_args()
parser.catches_help()
if not parser.unknown_commands:
log.error("it is required to pass an identifer, but none was provided")
raise SystemExit(1)
self.identifier = parser.unknown_commands[-1]
if parser.has('--poll'):
return self.poll()
for key in [
'stdout', 'stderr', 'command', 'ended',
'started', 'succeeded', 'exit_code']:
if parser.has(key):
return self.get(key)
# if nothing else matches, just try to give a generic, full summary
self.summary()
def main(self):
parser = Transport(self.arguments,
options=self.options,
check_help=True)
parser.catch_help = self._help
parser.parse_args()
parser.catches_help()
if not parser.unknown_commands:
log.error(
"it is required to pass an identifer, but none was provided")
raise SystemExit(1)
self.identifier = parser.unknown_commands[-1]
if parser.has('--poll'):
return self.poll()
for key in [
'stdout', 'stderr', 'command', 'ended', 'started', 'succeeded',
'exit_code'
]:
if parser.has(key):
return self.get(key)
# if nothing else matches, just try to give a generic, full summary
self.summary()
def main(self):
parser = Transport(self.arguments, options=self.options, check_help=True)
parser.catch_help = self._help
parser.parse_args()
parser.catches_help()
branch = parser.get('--branch', 'master')
user = parser.get('--user', 'vagrant')
high_verbosity = '-vvvv' if parser.has('-vvvv') else '-v'
if not parser.unknown_commands:
log.error("it is required to pass a host to deploy to, but none was provided")
raise SystemExit(1)
command = [
"ansible-playbook",
"-i", "%s," % parser.unknown_commands[-1],
high_verbosity,
"-u", user,
"--extra-vars", 'branch=%s' % branch,
"deploy.yml",
]
log.debug("Running command: %s" % ' '.join(command))
out, err, code = process.run(command, cwd=playbook_path)
log.error(err)
log.debug(out)
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self._help
self.parser.parse_args()
# handle posting projects:
if self.parser.has("create"):
url_part = self.sanitize_url(self.parser.get("create"))
if not sys.stdin.isatty():
# read from stdin
logger.info("reading input from stdin")
for line in sys.stdin.readlines():
url = os.path.join(self.base_url, url_part)
self.post(url)
else:
url = os.path.join(self.base_url, url_part)
self.post(url)
# XXX this exists here but it not yet enabled from the CLI
elif self.parser.has("delete"):
url_part = self.sanitize_url(self.parser.get("delete"))
url = os.path.join(self.base_url, url_part)
self.delete(url)
def main(self):
options = ['--ignore']
parser = Transport(self.argv, options=options, check_version=False)
parser.catch_help = self._help()
parser.parse_args()
if len(self.argv) < 1:
return parser.print_help()
# populate the nodes metadata with the configured nodes
for daemon in ceph_medic.config['nodes'].keys():
ceph_medic.metadata['nodes'][daemon] = []
for daemon, nodes in ceph_medic.config['nodes'].items():
for node in nodes:
ceph_medic.metadata['nodes'][daemon].append(
{'host': node['host']})
collector.collect()
test = runner.Runner()
results = test.run()
runner.report(results)
class Binary(object):
_help = dedent("""
Operate binaries on a remote chacra instance.
Creating a new binary::
chacractl binary create project/ref/distro/distro_version/arch /path/to/binary
Options:
create Creates a new binary at a given distro version architecture
delete Deletes an existing binary from chacra
--force If the resource exists, force the upload
""")
help_menu = "create, update metadata, or delete binaries"
options = ['create', '--force', 'delete']
def __init__(self, argv):
self.argv = argv
@property
def base_url(self):
return os.path.join(
chacractl.config['url'], 'binaries'
)
def sanitize_filename(self, line):
"""
lines may come with newlines and leading slashes make sure
they are clean so that they can be processed
"""
line = line.strip('\n')
if os.path.isfile(line):
return os.path.abspath(line)
def sanitize_url(self, url_part):
# get rid of the leading slash to prevent issues when joining
url = url_part.lstrip('/')
# and add a trailing slash so that the request is done at the correct
# canonical url
if not url.endswith('/'):
url = "%s/" % url
return url
def post(self, url, filepath):
filename = os.path.basename(filepath)
file_url = os.path.join(url, filename) + '/'
exists = requests.head(file_url, verify=chacractl.config['ssl_verify'])
if exists.status_code == 200:
if not self.force:
logger.warning(
'resource exists and --force was not used, will not upload'
)
logger.warning('SKIP %s', file_url)
return
return self.put(file_url, filepath)
elif exists.status_code == 404:
logger.info('POSTing file: %s', filepath)
with open(filepath, 'rb') as binary:
response = requests.post(
url,
files={'file': binary},
auth=chacractl.config['credentials'],
verify=chacractl.config['ssl_verify'])
if response.status_code > 201:
logger.warning("%s -> %s", response.status_code, response.text)
response.raise_for_status()
def put(self, url, filepath):
logger.info('resource exists and --force was used, will re-upload')
logger.info('PUTing file: %s', filepath)
with open(filepath, 'rb') as binary:
response = requests.put(
url,
files={'file': binary},
auth=chacractl.config['credentials'],
verify=chacractl.config['ssl_verify'])
if response.status_code > 201:
logger.warning("%s -> %s", response.status_code, response.text)
def delete(self, url):
exists = requests.head(url, verify=chacractl.config['ssl_verify'])
if exists.status_code == 404:
logger.warning('resource already deleted')
logger.warning('SKIP %s', url)
return
logger.info('DELETE file: %s', url)
response = requests.delete(
url,
auth=chacractl.config['credentials'],
verify=chacractl.config['ssl_verify'])
if response.status_code > 201:
logger.warning("%s -> %s", response.status_code, response.text)
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self._help
self.parser.parse_args()
self.force = self.parser.has('--force')
# handle posting binaries:
if self.parser.has('create'):
url_part = self.sanitize_url(self.parser.get('create'))
if not sys.stdin.isatty():
# read from stdin
logger.info('reading input from stdin')
for line in sys.stdin.readlines():
filename = self.sanitize_filename(line)
if not filename:
continue
url = os.path.join(self.base_url, url_part)
self.post(url, filename)
else:
filepath = self.sanitize_filename(self.argv[-1])
if not filepath:
logger.warning(
'provided path does not exist: %s', self.argv[-1]
)
return
url = os.path.join(self.base_url, url_part)
self.post(url, filepath)
elif self.parser.has('delete'):
url_part = self.sanitize_url(self.parser.get('delete'))
url = os.path.join(self.base_url, url_part)
self.delete(url)
def main(self, argv):
options = [
'--cluster',
'--ssh-config',
'--inventory',
'--config',
'--verbosity',
]
parser = Transport(argv,
options=options,
check_help=False,
check_version=False)
parser.parse_args()
self.config_path = parser.get('--config', configuration.location())
# load medic configuration
loaded_config = configuration.load(
path=parser.get('--config', self.config_path))
# this is the earliest we can have enough config to setup logging
log.setup(loaded_config)
ceph_medic.config.file = loaded_config
global_options = dict(ceph_medic.config.file._sections['global'])
# SSH config
ceph_medic.config.ssh_config = parser.get(
'--ssh-config', global_options.get('--ssh-config'))
if ceph_medic.config.ssh_config:
ssh_config_path = ceph_medic.config.ssh_config
if not os.path.exists(ssh_config_path):
terminal.error("the given ssh config path does not exist: %s" %
ssh_config_path)
sys.exit()
ceph_medic.config.cluster_name = parser.get('--cluster', 'ceph')
ceph_medic.metadata['cluster_name'] = 'ceph'
# Deployment Type
deployment_type = ceph_medic.config.file.get_safe(
'global', 'deployment_type', 'baremetal')
if deployment_type in ['kubernetes', 'openshift', 'k8s', 'oc']:
pod_hosts = hosts.container_platform(deployment_type)
ceph_medic.config.nodes = pod_hosts
ceph_medic.config.hosts_file = ':memory:'
self.hosts_file = ':memory:'
else:
# Hosts file
self.hosts_file = parser.get('--inventory',
configuration.get_host_file())
# find the hosts files, by the CLI first, fallback to the configuration
# file, and lastly if none of those are found or defined, try to load
# from well known locations (cwd, and /etc/ansible/)
loaded_hosts = configuration.load_hosts(
parser.get('--inventory',
global_options.get('--inventory', self.hosts_file)))
ceph_medic.config.nodes = loaded_hosts.nodes
ceph_medic.config.hosts_file = loaded_hosts.filename
self.hosts_file = loaded_hosts.filename
parser.catch_version = ceph_medic.__version__
parser.mapper = self.mapper
parser.catch_help = self.help(parser.subhelp())
if len(argv) <= 1:
return parser.print_help()
ceph_medic.config.config_path = self.config_path
parser.dispatch()
parser.catches_help()
parser.catches_version()
# Verbosity
verbosity = parser.get('--verbosity', 'debug')
ceph_medic.config.verbosity = verbosity.lowercase()
class MergePatches(object):
help_menu = 'Merge patches from RHEL -patches branch to patch-queue branch'
_help = """
Fetch the latest patches branch that rdopkg uses, and then fast-forward merge
that into our local patch-queue branch, so that both branches align.
This command helps to align the patch series between our RHEL packages and our
Ubuntu packages.
Options:
--force Do a hard reset, rather than restricting to fast-forward merges
only. Use this option if the RHEL patches branch was amended or
rebased for some reason.
"""
name = 'merge-patches'
def __init__(self, argv):
self.argv = argv
self.options = ['--force', '--hard-reset']
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
force = False
if self.parser.has(['--force', '--hard-reset']):
force = True
if self.parser.unknown_commands:
log.error('unknown option %s',
' '.join(self.parser.unknown_commands))
return self.parser.print_help()
self._run(force)
def help(self):
return self._help
def _run(self, force=False):
# Determine the names of the relevant branches
current_branch = util.current_branch()
debian_branch = util.current_debian_branch()
patch_queue_branch = util.current_patch_queue_branch()
rhel_patches_branch = self.get_rhel_patches_branch(debian_branch)
# Do the merge
if current_branch == patch_queue_branch:
# HEAD is our patch-queue branch. Use "git pull" directly.
# For example: "git pull --ff-only patches/ceph-2-rhel-patches"
cmd = ['git', 'pull', '--ff-only',
'patches/' + rhel_patches_branch]
if force:
# Do a hard reset on HEAD instead.
cmd = ['git', 'reset', '--hard',
'patches/' + rhel_patches_branch]
else:
# HEAD is our debian branch. Use "git fetch" to update the
# patch-queue ref. For example:
# "git fetch . \
# patches/ceph-2-rhel-patches:patch-queue/ceph-2-ubuntu"
util.ensure_patch_queue_branch()
cmd = ['git', 'fetch', '.',
'patches/%s:%s' % (rhel_patches_branch, patch_queue_branch)]
if force:
# Do a hard push (with "+") instead.
cmd = ['git', 'push', '.', '+patches/%s:%s' %
(rhel_patches_branch, patch_queue_branch)]
log.info(' '.join(cmd))
subprocess.check_call(cmd)
def get_rhel_patches_branch(self, debian_branch):
"""
Get the RHEL -patches branch corresponding to this debian branch.
Examples:
ceph-2-ubuntu -> ceph-2-rhel-patches
ceph-2-trusty -> ceph-2-rhel-patches
ceph-2-xenial -> ceph-2-rhel-patches
ceph-1.3-ubuntu -> ceph-1.3-rhel-patches
ceph-2-ubuntu-hotfix-bz123 -> ceph-2-rhel-patches-hotfix-bz123
"""
(product, version, distro) = debian_branch.split('-', 2)
suffix = None
if '-' in distro:
(distro, suffix) = distro.split('-', 1)
rhel = '%s-%s-rhel-patches' % (product, version)
if suffix is not None:
rhel = '%s-%s' % (rhel, suffix)
return rhel
class Binary(object):
_help = dedent("""
Operate binaries on a remote chacra instance.
Creating a new binary::
chacractl binary create project/ref/distro/distro_version/arch /path/to/binary
Options:
create Creates a new binary at a given distro version architecture
delete Deletes an existing binary from chacra
--force If the resource exists, force the upload
""")
help_menu = "create, update metadata, or delete binaries"
options = ['create', '--force', 'delete']
def __init__(self, argv):
self.argv = argv
@property
def base_url(self):
return os.path.join(
chacractl.config['url'], 'binaries'
)
def sanitize_filename(self, line):
"""
lines may come with newlines and leading slashes make sure
they are clean so that they can be processed
"""
line = line.strip('\n')
if os.path.isfile(line):
return os.path.abspath(line)
def sanitize_url(self, url_part):
# get rid of the leading slash to prevent issues when joining
url = url_part.lstrip('/')
# and add a trailing slash so that the request is done at the correct
# canonical url
if not url.endswith('/'):
url = "%s/" % url
return url
def load_file(self, filepath):
chsum = sha512()
binary = open(filepath, 'rb')
for chunk in iter(lambda: binary.read(4096), b''):
chsum.update(chunk)
binary.seek(0)
return binary, chsum.hexdigest()
def upload_is_verified(self, arch_url, filename, digest):
r = requests.get(arch_url, verify=chacractl.config['ssl_verify'])
r.raise_for_status()
arch_data = r.json()
remote_digest = arch_data[filename]['checksum']
verified = remote_digest == digest
if not verified:
logging.error(
'Checksum mismatch: server has wrong checksum for %s',
filename)
logging.error('local checksum: %s', digest)
logging.error('remote checksum: %s', remote_digest)
return verified
def post(self, url, filepath):
filename = os.path.basename(filepath)
file_url = os.path.join(url, filename) + '/'
exists = requests.head(file_url, verify=chacractl.config['ssl_verify'])
if exists.status_code == 200:
if not self.force:
logger.warning(
'resource exists and --force was not used, will not upload'
)
logger.warning('SKIP %s', file_url)
return
return self.put(file_url, filepath)
elif exists.status_code == 404:
logger.info('POSTing file: %s', filepath)
binary, digest = self.load_file(filepath)
with binary:
response = requests.post(
url,
files={'file': binary},
auth=chacractl.config['credentials'],
verify=chacractl.config['ssl_verify'])
if response.status_code > 201:
logger.warning("%s -> %s", response.status_code, response.text)
response.raise_for_status()
if not self.upload_is_verified(url, filename, digest):
# Since this is a new file, attempt to delete it
logging.error('Deleting corrupted file from server...')
self.delete(file_url)
raise SystemExit(
'Checksum mismatch: remote server has wrong checksum for %s'
% filepath)
def put(self, url, filepath):
filename = os.path.basename(filepath)
logger.info('resource exists and --force was used, will re-upload')
logger.info('PUTing file: %s', filepath)
binary, digest = self.load_file(filepath)
with binary:
response = requests.put(
url,
files={'file': binary},
auth=chacractl.config['credentials'],
verify=chacractl.config['ssl_verify'])
if response.status_code > 201:
logger.warning("%s -> %s", response.status_code, response.text)
# trim off binary filename
url = url.rsplit('/', 2)[0] + "/"
if not self.upload_is_verified(url, filename, digest):
# Maybe the old file with a different digest is still there, so
# don't delete it
raise SystemExit(
'Checksum mismatch: server has wrong checksum for %s!'
% filepath)
def delete(self, url):
exists = requests.head(url, verify=chacractl.config['ssl_verify'])
if exists.status_code == 404:
logger.warning('resource already deleted')
logger.warning('SKIP %s', url)
return
logger.info('DELETE file: %s', url)
response = requests.delete(
url,
auth=chacractl.config['credentials'],
verify=chacractl.config['ssl_verify'])
if response.status_code < 200 or response.status_code > 299:
logger.warning("%s -> %s", response.status_code, response.text)
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self._help
self.parser.parse_args()
self.force = self.parser.has('--force')
# handle posting binaries:
if self.parser.has('create'):
url_part = self.sanitize_url(self.parser.get('create'))
if not sys.stdin.isatty():
# read from stdin
logger.info('reading input from stdin')
for line in sys.stdin.readlines():
filename = self.sanitize_filename(line)
if not filename:
continue
url = os.path.join(self.base_url, url_part)
self.post(url, filename)
else:
filepath = self.sanitize_filename(self.argv[-1])
if not filepath:
logger.warning(
'provided path does not exist: %s', self.argv[-1]
)
return
url = os.path.join(self.base_url, url_part)
self.post(url, filepath)
elif self.parser.has('delete'):
if self.parser.get('delete') is None:
raise SystemExit('Specify a URL to delete a binary.')
url_part = self.sanitize_url(self.parser.get('delete'))
url = os.path.join(self.base_url, url_part)
self.delete(url)
class MergePatches(object):
help_menu = 'Merge patches from RHEL -patches branch to patch-queue branch'
_help = """
Fetch the latest patches branch that rdopkg uses, and then fast-forward merge
that into our local patch-queue branch, so that both branches align.
This command helps to align the patch series between our RHEL packages and our
Ubuntu packages.
Options:
--force Do a hard reset, rather than restricting to fast-forward merges
only. Use this option if the RHEL patches branch was amended or
rebased for some reason.
"""
name = 'merge-patches'
def __init__(self, argv):
self.argv = argv
self.options = ['--force', '--hard-reset']
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
force = False
if self.parser.has(['--force', '--hard-reset']):
force = True
if self.parser.unknown_commands:
log.error('unknown option %s',
' '.join(self.parser.unknown_commands))
return self.parser.print_help()
self._run(force)
def help(self):
return self._help
def _run(self, force=False):
# Determine the names of the relevant branches
current_branch = util.current_branch()
debian_branch = util.current_debian_branch()
patches_branch = util.current_patches_branch()
rhel_patches_branch = self.get_rhel_patches_branch(debian_branch)
# Do the merge
if current_branch == patches_branch:
# HEAD is our patch-queue branch. Use "git pull" directly.
# For example: "git pull --ff-only patches/ceph-2-rhel-patches"
cmd = ['git', 'pull', '--ff-only',
'patches/' + rhel_patches_branch]
if force:
# Do a hard reset on HEAD instead.
cmd = ['git', 'reset', '--hard',
'patches/' + rhel_patches_branch]
else:
# HEAD is our debian branch. Use "git fetch" to update the
# patch-queue ref. For example:
# "git fetch . \
# patches/ceph-2-rhel-patches:patch-queue/ceph-2-ubuntu"
cmd = ['git', 'fetch', '.',
'patches/%s:%s' % (rhel_patches_branch, patches_branch)]
if force:
# Do a hard push (with "+") instead.
cmd = ['git', 'push', '.', '+patches/%s:%s' %
(rhel_patches_branch, patches_branch)]
log.info(' '.join(cmd))
subprocess.check_call(cmd)
def get_rhel_patches_branch(self, debian_branch):
"""
Get the RHEL -patches branch corresponding to this debian branch.
Examples:
ceph-2-ubuntu -> ceph-2-rhel-patches
ceph-2-trusty -> ceph-2-rhel-patches
ceph-2-xenial -> ceph-2-rhel-patches
ceph-1.3-ubuntu -> ceph-1.3-rhel-patches
ceph-2-ubuntu-hotfix-bz123 -> ceph-2-rhel-patches-hotfix-bz123
"""
(product, version, distro) = debian_branch.split('-', 2)
suffix = None
if '-' in distro:
(distro, suffix) = distro.split('-', 1)
rhel = '%s-%s-rhel-patches' % (product, version)
if suffix is not None:
rhel = '%s-%s' % (rhel, suffix)
return rhel
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
self._run()
class Patch(object):
help_menu = 'apply patches from patch-queue branch'
_help = """
Generate patches from a patch-queue branch.
Options:
--nobz Do not require "Resolves: rhbz#" for every patch. The default is to
require them. Use this CLI option to override the default.
"""
name = 'patch'
def __init__(self, argv):
self.argv = argv
self.options = ('--nobz', )
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
self._run()
def help(self):
return self._help
def _run(self):
""" Generate quilt patch series with gbp pq, and update d/rules """
# Determine the names of the patch-queue branch and debian branch
current_branch = util.current_branch()
patch_queue_branch = util.current_patch_queue_branch()
debian_branch = util.current_debian_branch()
# TODO: default to fetching from upstream, the way rdopkg patch does.
# Get the new sha1 to insert into the $COMMIT variable in d/rules
cmd = ['git', 'rev-parse', patch_queue_branch]
output = subprocess.check_output(cmd)
patch_queue_sha1 = output.rstrip()
if six.PY3:
patch_queue_sha1 = output.decode('utf-8').rstrip()
# Switch to "debian" branch if necessary
if current_branch != debian_branch:
cmd = ['git', 'checkout', debian_branch]
subprocess.check_call(cmd)
# Get the original (old) patch series
old_series = self.read_series_file('debian/patches/series')
old_subjects = [patch.subject for patch in old_series]
# Git-buildpackage pq operation
cmd = ['gbp', 'pq', 'export']
subprocess.check_call(cmd)
# Add all patch files to Git's index
cmd = ['git', 'add', '--all', 'debian/patches']
subprocess.check_call(cmd)
# Bail early if gbp pq did nothing.
if not self.read_git_debian_patches_status():
print('No new patches, quitting.')
raise SystemExit(1)
# Replace $COMMIT sha1 in d/rules
old_sha1 = read_commit()
if old_sha1:
rules = read_rules_file()
with open('debian/rules', 'w') as fileh:
fileh.write(rules.replace(old_sha1, patch_queue_sha1))
# Get the new patch series
new_series = self.read_series_file('debian/patches/series')
# Select only the ones that are new (according to commit subjects)
new_series = [p for p in new_series if p.subject not in old_subjects]
if not new_series:
# Maybe we rewrote some patch files in place?
# Check Git itself for changed files:
new_series = self.read_git_debian_patches()
# Add patch entries to d/changelog
changelog = self.generate_changelog(new_series)
try:
ensure_bzs(changelog)
except BzNotFound:
if not self.parser.has('--nobz'):
raise
util.bump_changelog(changelog)
# Assemble a standard commit message string "clog".
clog = "debian: %s\n" % util.get_deb_version()
clog += "\n"
clog += "Add patches from %s\n" % patch_queue_branch
clog += "\n"
clog += util.format_changelog(changelog)
# Commit everything with the standard commit message.
with tempfile.NamedTemporaryFile(mode='w+') as temp:
temp.write(clog)
temp.flush()
cmd = [
'git', 'commit', 'debian/changelog', 'debian/patches',
'debian/rules', '-F', temp.name
]
subprocess.check_call(cmd)
# Summarize this commit on STDOUT for the developer.
# (This matches the behavior of "rdopkg patch".)
cmd = ['git', '--no-pager', 'log', '--name-status', 'HEAD~..HEAD']
subprocess.check_call(cmd)
def generate_changelog(self, series):
"""
Generate a list of changelog entries for this gbp Patch series.
:return: a list of strings
"""
changelog = []
for p in series:
# If there was some in-place Git modification for this patch,
# (.git_action attribute), include that in our log.
action = getattr(p, 'git_action', 'A')
# Make common actions human-readable:
if action == 'M':
action = 'Modified'
if action == 'D':
action = 'Deleted'
if action == 'R':
# We don't log .patch file renames
continue
change = '%s %s' % (action, p.path)
if action == 'A':
# This was a simple patch addition, so just log the patch's
# subject.
change = p.subject
bzs = self.get_rhbzs(p)
bzstr = ' '.join(map(lambda x: 'rhbz#%s' % x, bzs))
if bzstr != '':
change += ' (%s)' % bzstr
changelog.append(change)
return changelog
def get_rhbzs(self, patch):
"""
Return all RHBZ numbers from a Patch's subject and body.
:param patch: gbp.patch_series.Patch``
"""
bzs = re.findall(BZ_REGEX, patch.subject)
body = patch.long_desc
try:
if patch.git_action == 'D':
# patch.long_desc will be empty.
# Read the deleted file's description from Git instead.
body = self.read_deleted_patch_description(patch.path)
except AttributeError:
# This was a simple patch addition, so we'll just search this
# patch's .long_desc.
pass
bzs.extend(re.findall(BZ_REGEX, body))
return bzs
def read_series_file(self, file_):
return gbp.patch_series.PatchSeries.read_series_file(file_)
def read_git_debian_patches_status(self):
"""
Return a list of all edited Debian patch files (from "git status").
:return: a list of actions/filesname pairs. For example:
[
['M', 'debian/patches/0001-foo.patch'],
['D', 'debian/patches/0002-bar.patch'],
]
"""
cmd = ['git', 'status', '-s', 'debian/patches/']
output = subprocess.check_output(cmd)
if six.PY3:
output = output.decode('utf-8')
result = []
for line in output.splitlines():
if line.endswith('.patch'):
result.append(line.split(None, 1))
return result
def read_git_debian_patches(self):
"""
Load all edited Debian patches (from "git status") into Patch objects.
The returned Patch objects have an extra ".git_action" attribute. Use
this to determine what happened to the patch in Git.
:return: a list of gbp.patch_series.Patch objects
"""
patches = []
for (action, filename) in self.read_git_debian_patches_status():
patch = gbp.patch_series.Patch(filename)
# Hack: record what happened to this patch file:
patch.git_action = action
patches.append(patch)
return patches
def read_deleted_patch_description(self, filename):
"""
Parse a deleted .patch file with gbp.patch_series.Patch.
For deleted .patch files, most of the gbp.patch_series.Patch
attributes from read_git_debian_patches() are empty, because the file
no longer exists. More hackery to recover the original .long_desc so
we can recover the original RHBZ number.
:returns: ``str``, the long_desc attribute.
"""
with tempfile.NamedTemporaryFile(mode='w+') as temp:
cmd = ['git', 'show', 'HEAD:%s' % filename]
subprocess.call(cmd, stdout=temp)
temp.flush()
temppatch = gbp.patch_series.Patch(temp.name)
temppatch._read_info() # XXX internal API here :(
return temppatch.long_desc
class Patch(object):
help_menu = 'apply patches from patch-queue branch'
_help = """
Generate patches from a patch-queue branch.
"""
name = 'patch'
def __init__(self, argv):
self.argv = argv
self.options = []
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
self._run()
def help(self):
return self._help
def _run(self):
""" Generate quilt patch series with gbp pq, and update d/rules """
# Determine the names of the patch-queue branch and debian branch
current_branch = util.current_branch()
patches_branch = util.current_patches_branch()
debian_branch = util.current_debian_branch()
# TODO: default to fetching from upstream, the way rdopkg patch does.
# Get the new sha1 to insert into the $COMMIT variable in d/rules
cmd = ['git', 'rev-parse', patches_branch]
patches_sha1 = subprocess.check_output(cmd).rstrip()
# Switch to "debian" branch if necessary
if current_branch != debian_branch:
cmd = ['git', 'checkout', debian_branch]
subprocess.check_call(cmd)
# Get the original (old) patch series
old_series = self.read_series_file('debian/patches/series')
old_subjects = map(lambda x: x.subject, old_series)
# Git-buildpackage pq operation
cmd = ['gbp', 'pq', 'export']
subprocess.check_call(cmd)
# Add all patch files to Git's index
cmd = ['git', 'add', '--all', 'debian/patches']
subprocess.check_call(cmd)
# Replace $COMMIT sha1 in d/rules
with open('debian/rules') as rules:
rules_file = rules.read()
old = r'export COMMIT=[0-9a-f]{40}'
new = 'export COMMIT=%s' % patches_sha1
with open('debian/rules', 'w') as fileh:
fileh.write(re.sub(old, new, rules_file))
# Get the new patch series
new_series = self.read_series_file('debian/patches/series')
# Add patch entries to d/changelog
changelog = []
for p in new_series:
if p.subject in old_subjects:
continue
change = p.subject
bzs = self.get_rhbzs(p)
bzstr = ' '.join(map(lambda x: 'rhbz#%s' % x, bzs))
if bzstr != '':
change += ' (%s)' % bzstr
changelog.append(change)
util.bump_changelog(changelog)
# Assemble a standard commit message string "clog".
clog = "debian: %s\n" % util.get_deb_version()
clog += "\n"
clog += "Add patches from %s\n" % patches_branch
clog += "\n"
clog += util.format_changelog(changelog)
# Commit everything with the standard commit message.
with tempfile.NamedTemporaryFile() as temp:
temp.write(clog)
temp.flush()
cmd = [
'git', 'commit', 'debian/changelog', 'debian/patches',
'debian/rules', '-F', temp.name
]
subprocess.check_call(cmd)
# Summarize this commit on STDOUT for the developer.
# (This matches the behavior of "rdopkg patch".)
cmd = ['git', '--no-pager', 'log', '--name-status', 'HEAD~..HEAD']
subprocess.check_call(cmd)
def get_rhbzs(self, patch):
bzs = re.findall(BZ_REGEX, patch.subject)
bzs.extend(re.findall(BZ_REGEX, patch.long_desc))
return bzs
def read_series_file(self, file_):
try:
from gbp.patch_series import PatchSeries
return PatchSeries.read_series_file(file_)
except ImportError:
log.warning('Please run "sudo apt-get install '
'git-buildpackage" to write the patches to '
'./debian/changelog')
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
self._run()
def parse_args(self):
options = ['create', 'update', 'generate', 'remove', 'get']
parser = Transport(self.argv, options=options)
parser.catch_help = self._help
parser.parse_args()
if parser.has('create'):
return self.create(parser.get('create'))
if parser.has('update'):
optional_args = ['key', 'step', 'secret', 'b32']
items = [i for i in parser.arguments if i in optional_args]
return self.update(parser.get('update'), items)
if parser.has('generate'):
return self.generate()
if parser.has('remove'):
return self.remove(parser.get('remove'))
if parser.has('get'):
items = [i for i in parser.arguments if i in ['pin']]
return self.get(parser.get('get'), items)
def main(self, argv):
parser = Transport(argv, mapper=self.mapper,
check_help=False,
check_version=False)
parser.parse_args()
parser.catch_help = self.help()
parser.catch_version = wari.__version__
parser.mapper = self.mapper
if len(argv) <= 1:
return parser.print_help()
# create the connection and set the collection
conn = wari.db.get_connection()
wari.db.connection = conn
wari.db.collection = conn['wari']
parser.dispatch()
parser.catches_help()
parser.catches_version()
conn.close()
class Localbuild(object):
help_menu = 'build a package on the local system'
_help = """
Build a package on the local system, using pbuilder.
Options:
--dist "xenial" or "trusty". If unspecified, rhcephpkg will choose one
based on the current branch's name.
Rules for automatic distro selection:
1) If the branch suffix is an ubuntu distro name, use that.
eg "ceph-3.0-xenial".
2) If a branch has a version number starting with "1.3", return "trusty".
eg. "ceph-1.3-ubuntu"
3) If a branch has a version number starting with "2" return "xenial".
eg. "ceph-2-ubuntu"
4) If a branch has a version number starting with "3" return "xenial".
eg. "ceph-3.0-ubuntu"
5) Otherwise raise, because we need to add more rules.
"""
name = 'localbuild'
def __init__(self, argv):
self.argv = argv
self.options = ('--dist', )
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
# Allow user to override the distro.
if self.parser.has('--dist'):
if self.parser.get('--dist') is None:
raise SystemExit('Specify a distro to --dist')
distro = self.parser.get('--dist')
else:
distro = get_distro()
if self.parser.unknown_commands:
log.error('unknown option %s',
' '.join(self.parser.unknown_commands))
return self.parser.print_help()
self._run(distro)
def help(self):
return self._help
def _run(self, distro):
""" Build a package on the local system, using pbuilder. """
pkg_name = util.package_name()
os.environ['BUILDER'] = 'pbuilder'
j_arg = self._get_j_arg(cpu_count())
pbuilder_cache = '/var/cache/pbuilder/base-%s-amd64.tgz' % distro
setup_pbuilder_cache(pbuilder_cache, distro)
util.setup_pristine_tar_branch()
# TODO: we should also probably check parent dir for leftovers and warn
# the user to delete them (or delete them ourselves?)
cmd = [
'gbp', 'buildpackage',
'--git-dist=%s' % distro, '--git-arch=amd64', '--git-verbose',
'--git-pbuilder', j_arg, '-us', '-uc'
]
log.info('building %s with pbuilder', pkg_name)
subprocess.check_call(cmd)
def _get_j_arg(self, cpus, total_ram_gb=None):
"""
Returns a string like "-j4" or "-j8". j is the number of processors,
with a maximum of x, where x = TOTAL_RAM_GB / 4.
We want to use all our processors (a high "j" value), but the build
process will fail with an "out of memory" error out if this j value is
too high.
An 8 GB system would have a maximum of -j2
A 16 GB system would have a maximum of -j4
A 32 GB system would have a maximum of -j8
"""
if total_ram_gb is None:
page_size = os.sysconf('SC_PAGE_SIZE')
mem_bytes = page_size * os.sysconf('SC_PHYS_PAGES')
# mem_gib is a decimal, eg. 7.707 on 8GB system
mem_gib = mem_bytes / (1024.**3)
# Round up to the nearest GB for our purposes.
total_ram_gb = math.ceil(mem_gib)
number = min(cpus, total_ram_gb / 4)
return '-j%d' % max(number, 1)
def main(self, argv):
options = []
parser = Transport(argv,
mapper=self.mapper,
options=options,
check_help=False,
check_version=False)
parser.parse_args()
parser.catch_help = self.help()
parser.catch_version = rhcephpkg.__version__
parser.mapper = self.mapper
if len(argv) <= 1:
return parser.print_help()
parser.dispatch()
parser.catches_help()
parser.catches_version()
class Gitbz(object):
help_menu = 'verify each RHBZ in the last Git commit message'
_help = """
Verify that each RHBZ in the last Git commit message is approved for this
release.
If the commit message lacks any RHBZ number, or any RHBZs do not correspond to
this release (dist-git branch), then this command exits with a non-zero exit
code.
Requires a cached login to bugzilla (`bugzilla login` command).
This tool mimics the validation that the internal "gitbz" tool provides for
RHEL dist-git.
"""
name = 'gitbz'
def __init__(self, argv):
self.argv = argv
self.options = []
def main(self):
self.parser = Transport(self.argv, options=self.options)
self.parser.catch_help = self.help()
self.parser.parse_args()
self._run()
def help(self):
return self._help
def _run(self):
debian_branch = util.current_debian_branch()
try:
flag = release_flag(debian_branch)
except ValueError:
raise SystemExit('could not parse debian branch "%s".' %
debian_branch)
msg = last_commit_message()
bzids = find_bzs(msg)
if not bzids:
raise SystemExit('no BZs found')
bzapi = get_bzapi()
bugs = bzapi.getbugs(bzids,
include_fields=['id', 'flags'],
permissive=False)
missing = []
for bug in bugs:
has_release_flag = False
for f in bug.flags:
if f['name'] == flag:
print('rhbz#%s: %s%s' % (bug.id, f['name'], f['status']))
has_release_flag = True
if not has_release_flag:
missing.append(bug.id)
if missing:
print('Missing %s release flag:' % flag)
for m in missing:
print('rhbz#%s' % m)
raise SystemExit(1)
