def check_call(conn, logger, args, *a, **kw):
"""
Wraps ``subprocess.check_call`` for a remote call via ``pushy``
doing all the capturing and logging nicely upon failure/success
The mangling of the traceback when an exception ocurrs, is because the
caller gets eating up by not being executed in the actual function of
a given module (e.g. ``centos/install.py``) but rather here, where the
stack trace is no longer relevant.
:param args: The args to be passed onto ``check_call``
"""
command = ' '.join(args)
patch = kw.pop('patch', True) # Always patch unless explicitly told to
mangle = kw.pop('mangle_exc', False) # Default to not mangle exceptions
stop_on_error = kw.pop('stop_on_error', True) # Halt on remote exceptions
logger.info('Running command: %s' % command)
kw.setdefault(
'env',
{
'PATH':
'/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin'
}
)
def remote_call(args, *a, **kw):
import subprocess
return subprocess.check_call(
args,
*a,
**kw
)
with context.remote(conn, logger, remote_call, mangle_exc=mangle, patch=patch) as call:
try:
return call(args, *a, **kw)
except Exception as err:
import inspect
stack = inspect.getframeinfo(inspect.currentframe().f_back)
if hasattr(err, 'remote_traceback'):
logger.error('Traceback (most recent call last):')
logger.error(' File "%s", line %s, in %s' % (
stack[0],
stack[1],
stack[2])
)
err.remote_traceback.pop(0)
for line in err.remote_traceback:
if line:
logger.error(line)
if stop_on_error:
raise RuntimeError(
'Failed to execute command: %s' % ' '.join(args)
)
else:
if stop_on_error:
raise err
def check_call(conn, logger, args, *a, **kw):
"""
Wraps ``subprocess.check_call`` for a remote call via ``pushy``
doing all the capturing and logging nicely upon failure/success
The mangling of the traceback when an exception ocurrs, is because the
caller gets eating up by not being executed in the actual function of
a given module (e.g. ``centos/install.py``) but rather here, where the
stack trace is no longer relevant.
:param args: The args to be passed onto ``check_call``
"""
command = ' '.join(args)
patch = kw.pop('patch', True) # Always patch unless explicitly told to
mangle = kw.pop('mangle_exc', False) # Default to not mangle exceptions
stop_on_error = kw.pop('stop_on_error', True) # Halt on remote exceptions
logger.info('Running command: %s' % command)
kw.setdefault('env', {
'PATH':
'/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin'
})
def remote_call(args, *a, **kw):
import subprocess
return subprocess.check_call(args, *a, **kw)
with context.remote(conn,
logger,
remote_call,
mangle_exc=mangle,
patch=patch) as call:
try:
return call(args, *a, **kw)
except Exception as err:
import inspect
stack = inspect.getframeinfo(inspect.currentframe().f_back)
if hasattr(err, 'remote_traceback'):
logger.error('Traceback (most recent call last):')
logger.error(' File "%s", line %s, in %s' %
(stack[0], stack[1], stack[2]))
err.remote_traceback.pop(0)
for line in err.remote_traceback:
if line:
logger.error(line)
if stop_on_error:
raise RuntimeError('Failed to execute command: %s' %
' '.join(args))
else:
if stop_on_error:
raise err
def Popen(conn, logger, args, *a, **kw):
"""
Wraps ``subprocess.Popen`` for a remote call via ``pushy``
doing all the capturing and logging nicely upon failure/success
The mangling of the traceback when an exception ocurrs, is because the
caller gets eating up by not being executed in the actual function of
a given module (e.g. ``centos/install.py``) but rather here, where the
stack trace is no longer relevant.
:param args: The args to be passed onto ``Popen``
"""
command = ' '.join(args)
patch = kw.pop('patch', True) # Always patch unless explicitly told to
logger.info('Running command: %s' % command)
def remote_call(args, *a, **kw):
import subprocess
process = subprocess.Popen(args,
*a,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
**kw)
stdout, stderr = process.communicate()
return stdout, stderr, process.wait()
with context.remote(conn,
logger,
remote_call,
mangle_exc=False,
patch=patch) as call:
try:
return call(args, *a, **kw)
except Exception as err:
import inspect
stack = inspect.getframeinfo(inspect.currentframe().f_back)
if hasattr(err, 'remote_traceback'):
logger.error('Traceback (most recent call last):')
logger.error(' File "%s", line %s, in %s' %
(stack[0], stack[1], stack[2]))
err.remote_traceback.pop(0)
for line in err.remote_traceback:
if line:
logger.error(line)
raise RuntimeError('Failed to execute command: %s' %
' '.join(args))
else:
raise err
def Popen(conn, logger, args, *a, **kw):
"""
Wraps ``subprocess.Popen`` for a remote call via ``pushy``
doing all the capturing and logging nicely upon failure/success
The mangling of the traceback when an exception ocurrs, is because the
caller gets eating up by not being executed in the actual function of
a given module (e.g. ``centos/install.py``) but rather here, where the
stack trace is no longer relevant.
:param args: The args to be passed onto ``Popen``
"""
command = ' '.join(args)
patch = kw.pop('patch', True) # Always patch unless explicitly told to
logger.info('Running command: %s' % command)
def remote_call(args, *a, **kw):
import subprocess
process = subprocess.Popen(
args,
*a,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
**kw
)
stdout, stderr = process.communicate()
return stdout, stderr, process.wait()
with context.remote(conn, logger, remote_call, mangle_exc=False, patch=patch) as call:
try:
return call(args, *a, **kw)
except Exception as err:
import inspect
stack = inspect.getframeinfo(inspect.currentframe().f_back)
if hasattr(err, 'remote_traceback'):
logger.error('Traceback (most recent call last):')
logger.error(' File "%s", line %s, in %s' % (
stack[0],
stack[1],
stack[2])
)
err.remote_traceback.pop(0)
for line in err.remote_traceback:
if line:
logger.error(line)
raise RuntimeError('Failed to execute command: %s' % ' '.join(args))
else:
raise err
def install(distro, logger, version_kind, version):
codename = distro.codename
machine = distro.sudo_conn.modules.platform.machine()
if version_kind in ['stable', 'testing']:
key = 'release'
else:
key = 'autobuild'
# Make sure ca-certificates is installed
check_call(distro.sudo_conn, logger, [
'env',
'DEBIAN_FRONTEND=noninteractive',
'apt-get',
'-q',
'install',
'--assume-yes',
'ca-certificates',
])
check_call(
distro.sudo_conn,
logger,
[
'wget -q -O- \'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/{key}.asc\' | apt-key add -'
.format(key=key)
],
shell=True,
)
if version_kind == 'stable':
url = 'http://ceph.com/debian-{version}/'.format(version=version, )
elif version_kind == 'testing':
url = 'http://ceph.com/debian-testing/'
elif version_kind == 'dev':
url = 'http://gitbuilder.ceph.com/ceph-deb-{codename}-{machine}-basic/ref/{version}'.format(
codename=codename,
machine=machine,
version=version,
)
else:
raise RuntimeError('Unknown version kind: %r' % version_kind)
def write_sources_list(url, codename):
"""add ceph deb repo to sources.list"""
with file('/etc/apt/sources.list.d/ceph.list', 'w') as f:
f.write('deb {url} {codename} main\n'.format(
url=url,
codename=codename,
))
with remote(distro.sudo_conn, logger, write_sources_list) as remote_func:
remote_func(url, codename)
check_call(
distro.sudo_conn,
logger,
['apt-get', '-q', 'update'],
)
# TODO this does not downgrade -- should it?
check_call(
distro.sudo_conn,
logger,
[
'env',
'DEBIAN_FRONTEND=noninteractive',
'DEBIAN_PRIORITY=critical',
'apt-get',
'-q',
'-o',
'Dpkg::Options::=--force-confnew',
'install',
'--no-install-recommends',
'--assume-yes',
'--',
'ceph',
'ceph-mds',
'ceph-common',
'ceph-fs-common',
# ceph only recommends gdisk, make sure we actually have
# it; only really needed for osds, but minimal collateral
'gdisk',
],
)
# Check the ceph version
common.ceph_version(distro.sudo_conn, logger)
def mon_create(distro, logger, args, monitor_keyring, hostname):
logger.debug('remote hostname: %s' % hostname)
path = paths.mon.path(args.cluster, hostname)
done_path = paths.mon.done(args.cluster, hostname)
init_path = paths.mon.init(args.cluster, hostname, distro.init)
configuration = conf.load(args)
conf_data = StringIO()
configuration.write(conf_data)
with remote(distro.sudo_conn, logger, conf.write_conf) as remote_func:
remote_func(args.cluster, conf_data.getvalue(), overwrite=args.overwrite_conf)
if not distro.sudo_conn.modules.os.path.exists(path):
logger.info('creating path: %s' % path)
distro.sudo_conn.modules.os.makedirs(path)
logger.debug('checking for done path: %s' % done_path)
if not distro.sudo_conn.modules.os.path.exists(done_path):
logger.debug('done path does not exist: %s' % done_path)
if not distro.sudo_conn.modules.os.path.exists(paths.mon.constants.tmp_path):
logger.info('creating tmp path: %s' % paths.mon.constants.tmp_path)
distro.sudo_conn.modules.os.makedirs(paths.mon.constants.tmp_path)
keyring = paths.mon.keyring(args.cluster, hostname)
def write_monitor_keyring(keyring, monitor_keyring):
"""create the monitor keyring file"""
with file(keyring, 'w') as f:
f.write(monitor_keyring)
logger.info('creating keyring file: %s' % keyring)
with remote(distro.sudo_conn, logger, write_monitor_keyring) as remote_func:
remote_func(keyring, monitor_keyring)
check_call(
distro.sudo_conn,
logger,
[
'ceph-mon',
'--cluster', args.cluster,
'--mkfs',
'-i', hostname,
'--keyring', keyring,
],
)
logger.info('unlinking keyring file %s' % keyring)
distro.sudo_conn.modules.os.unlink(keyring)
def create_done_path(done_path):
"""create a done file to avoid re-doing the mon deployment"""
with file(done_path, 'w'):
pass
with remote(distro.sudo_conn, logger, create_done_path) as remote_func:
remote_func(done_path)
def create_init_path(init_path):
"""create the init path if it does not exist"""
import os
if not os.path.exists(init_path):
with file(init_path, 'w'):
pass
with remote(distro.sudo_conn, logger, create_init_path) as remote_func:
remote_func(init_path)
def install(distro, logger, version_kind, version, adjust_repos):
codename = distro.codename
machine = distro.sudo_conn.modules.platform.machine()
if version_kind in ['stable', 'testing']:
key = 'release'
else:
key = 'autobuild'
# Make sure ca-certificates is installed
check_call(
distro.sudo_conn,
logger,
[
'env',
'DEBIAN_FRONTEND=noninteractive',
'apt-get',
'-q',
'install',
'--assume-yes',
'ca-certificates',
]
)
if adjust_repos:
check_call(
distro.sudo_conn,
logger,
['wget -q -O- \'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/{key}.asc\' | apt-key add -'.format(key=key)],
shell=True,
)
if version_kind == 'stable':
url = 'http://ceph.com/debian-{version}/'.format(
version=version,
)
elif version_kind == 'testing':
url = 'http://ceph.com/debian-testing/'
elif version_kind == 'dev':
url = 'http://gitbuilder.ceph.com/ceph-deb-{codename}-{machine}-basic/ref/{version}'.format(
codename=codename,
machine=machine,
version=version,
)
else:
raise RuntimeError('Unknown version kind: %r' % version_kind)
def write_sources_list(url, codename):
"""add ceph deb repo to sources.list"""
with file('/etc/apt/sources.list.d/ceph.list', 'w') as f:
f.write('deb {url} {codename} main\n'.format(
url=url,
codename=codename,
))
with remote(distro.sudo_conn, logger, write_sources_list) as remote_func:
remote_func(url, codename)
check_call(
distro.sudo_conn,
logger,
['apt-get', '-q', 'update'],
)
# TODO this does not downgrade -- should it?
check_call(
distro.sudo_conn,
logger,
[
'env',
'DEBIAN_FRONTEND=noninteractive',
'DEBIAN_PRIORITY=critical',
'apt-get',
'-q',
'-o', 'Dpkg::Options::=--force-confnew',
'--no-install-recommends',
'--assume-yes',
'install',
'--',
'ceph',
'ceph-mds',
'ceph-common',
'ceph-fs-common',
# ceph only recommends gdisk, make sure we actually have
# it; only really needed for osds, but minimal collateral
'gdisk',
],
)
# Check the ceph version
common.ceph_version(distro.sudo_conn, logger)
def mon_create(distro, logger, args, monitor_keyring, hostname):
logger.debug('remote hostname: %s' % hostname)
path = paths.mon.path(args.cluster, hostname)
done_path = paths.mon.done(args.cluster, hostname)
init_path = paths.mon.init(args.cluster, hostname, distro.init)
configuration = conf.load(args)
conf_data = StringIO()
configuration.write(conf_data)
with remote(distro.sudo_conn, logger, conf.write_conf) as remote_func:
remote_func(args.cluster,
conf_data.getvalue(),
overwrite=args.overwrite_conf)
if not distro.sudo_conn.modules.os.path.exists(path):
logger.info('creating path: %s' % path)
distro.sudo_conn.modules.os.makedirs(path)
logger.debug('checking for done path: %s' % done_path)
if not distro.sudo_conn.modules.os.path.exists(done_path):
logger.debug('done path does not exist: %s' % done_path)
if not distro.sudo_conn.modules.os.path.exists(
paths.mon.constants.tmp_path):
logger.info('creating tmp path: %s' % paths.mon.constants.tmp_path)
distro.sudo_conn.modules.os.makedirs(paths.mon.constants.tmp_path)
keyring = paths.mon.keyring(args.cluster, hostname)
def write_monitor_keyring(keyring, monitor_keyring):
"""create the monitor keyring file"""
with file(keyring, 'w') as f:
f.write(monitor_keyring)
logger.info('creating keyring file: %s' % keyring)
with remote(distro.sudo_conn, logger,
write_monitor_keyring) as remote_func:
remote_func(keyring, monitor_keyring)
check_call(
distro.sudo_conn,
logger,
[
'ceph-mon',
'--cluster',
args.cluster,
'--mkfs',
'-i',
hostname,
'--keyring',
keyring,
],
)
logger.info('unlinking keyring file %s' % keyring)
distro.sudo_conn.modules.os.unlink(keyring)
def create_done_path(done_path):
"""create a done file to avoid re-doing the mon deployment"""
with file(done_path, 'w'):
pass
with remote(distro.sudo_conn, logger, create_done_path) as remote_func:
remote_func(done_path)
def create_init_path(init_path):
"""create the init path if it does not exist"""
import os
if not os.path.exists(init_path):
with file(init_path, 'w'):
pass
with remote(distro.sudo_conn, logger, create_init_path) as remote_func:
remote_func(init_path)