def reset_device_n3000(pci_addr):
# Reset the N3000 FPGA at the specified PCI address.
try:
# Build up the command to perform the reset.
# Note the hack to work around OPAE tool locale issues
cmd = ("docker run -t --privileged -e LC_ALL=en_US.UTF-8 "
"-e LANG=en_US.UTF-8 " + OPAE_IMG +
" rsu bmcimg " + pci_addr)
# Issue the command to perform the firmware update.
subprocess.check_output(shlex.split(cmd),
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as exc:
# "docker run" return code will be:
# 125 if the error is with Docker daemon itself
# 126 if the contained command cannot be invoked
# 127 if the contained command cannot be found
# Exit code of contained command otherwise
msg = ("Failed to reset device %s, "
"return code is %d, command output: %s." %
(pci_addr, exc.returncode,
exc.output.decode('utf-8')))
LOG.error(msg)
LOG.error("Check for intel-max10 kernel logs.")
raise exception.SysinvException(msg)
def write_device_image_n3000(filename, pci_addr):
# Write the firmware image to the FPGA at the specified PCI address.
# We're assuming that the image update tools will catch the scenario
# where the image is not compatible with the device.
try:
# Build up the command to perform the firmware update.
# Note the hack to work around OPAE tool locale issues
cmd = ("docker run -t --privileged -e LC_ALL=en_US.UTF-8 "
"-e LANG=en_US.UTF-8 -v " + DEVICE_IMAGE_CACHE_DIR +
":" + "/mnt/images " + OPAE_IMG +
" fpgasupdate -y --log-level debug /mnt/images/" +
filename + " " + pci_addr)
# Issue the command to perform the firmware update.
subprocess.check_output(shlex.split(cmd),
stderr=subprocess.STDOUT)
# TODO: switch to subprocess.Popen, parse the output and send
# progress updates.
except subprocess.CalledProcessError as exc:
# Check the return code, send completion info to sysinv-conductor.
# "docker run" return code will be:
# 125 if the error is with Docker daemon itself
# 126 if the contained command cannot be invoked
# 127 if the contained command cannot be found
# Exit code of contained command otherwise
msg = ("Failed to update device image %s for device %s, "
"return code is %d, command output: %s." %
(filename, pci_addr, exc.returncode,
exc.output.decode('utf-8')))
LOG.error(msg)
LOG.error("Check for intel-max10 kernel logs.")
raise exception.SysinvException(msg)
def _check_platform_backup_partition(self):
"""Check that the platform-backup partition is the correct size/type"""
args = ['/usr/bin/validate-platform-backup.sh']
try:
subprocess.check_output(args, stderr=subprocess.STDOUT) # pylint: disable=not-callable
except subprocess.CalledProcessError as exc:
LOG.error("Call to %s returned %s and %s" % (args, exc.returncode, exc.output))
return False
return True
def watchdog_action(action):
if action not in ["stop", "start"]:
LOG.warn("watchdog_action called with invalid action: %s", action)
return
try:
# Build up the command to perform the action.
cmd = ["systemctl", action, "hostw"]
# Issue the command to stop/start the watchdog
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as exc:
msg = ("Failed to %s hostw service, "
"return code is %d, command output: %s." %
(action, exc.returncode, exc.output))
LOG.warn(msg)
def get_pci_sriov_vf_driver_name(self, pciaddr, sriov_vfs_pci_address):
vf_driver = None
for addr in sriov_vfs_pci_address:
try:
with open(os.devnull, "w") as fnull:
output = subprocess.check_output(['lspci', '-vmmks', addr],
stderr=fnull)
except Exception as e:
LOG.error(
"Error getting PCI data for SR-IOV "
"VF address %s: %s", addr, e)
continue
for line in output.split('\n'):
pci_attr = shlex.split(line.strip())
if (pci_attr and len(pci_attr) == 2
and 'Module' in pci_attr[0]):
vf_driver = pci_attr[1]
break
# All VFs have the same driver per device.
if vf_driver:
break
return vf_driver
def create_certificate(keyout=None, certout=None):
if not keyout:
keyout, certout = get_certificate_paths()
if not keyout:
raise Exception('Unable to locate TLS certificate path automatically')
shortname = socket.gethostname().split('.')[0]
longname = socket.getfqdn()
subprocess.check_call([
'openssl', 'ecparam', '-name', 'secp384r1', '-genkey', '-out', keyout
])
san = ['IP:{0}'.format(x) for x in get_ip_addresses()]
# It is incorrect to put IP addresses as DNS type. However
# there exists non-compliant clients that fail with them as IP
san.extend(['DNS:{0}'.format(x) for x in get_ip_addresses()])
san.append('DNS:{0}'.format(shortname))
san.append('DNS:{0}'.format(longname))
san = ','.join(san)
sslcfg = get_openssl_conf_location()
tmpconfig = tempfile.mktemp()
shutil.copy2(sslcfg, tmpconfig)
try:
with open(tmpconfig, 'a') as cfgfile:
cfgfile.write(
'\n[SAN]i\nbasicConstraints = CA:true\nsubjectAltName={0}'.
format(san))
subprocess.check_call([
'openssl', 'req', '-new', '-x509', '-key', keyout, '-days', '7300',
'-out', certout, '-subj', '/CN={0}'.format(longname),
'-extensions', 'SAN', '-config', tmpconfig
])
finally:
os.remove(tmpconfig)
# Could restart the webserver now?
fname = '/var/lib/confluent/public/site/tls/{0}.pem'.format(
collective.get_myname())
try:
os.makedirs(os.path.dirname(fname))
except OSError as e:
if e.errno != 17:
raise
shutil.copy2(certout, fname)
hv = subprocess.check_output(
['openssl', 'x509', '-in', certout, '-hash', '-noout'])
if not isinstance(hv, str):
hv = hv.decode('utf8')
hv = hv.strip()
hashname = '/var/lib/confluent/public/site/tls/{0}.0'.format(hv)
certname = '{0}.pem'.format(collective.get_myname())
for currname in os.listdir('/var/lib/confluent/public/site/tls/'):
currname = os.path.join('/var/lib/confluent/public/site/tls/',
currname)
if currname.endswith('.0'):
try:
realname = os.readlink(currname)
if realname == certname:
os.unlink(currname)
except OSError:
pass
os.symlink(certname, hashname)
def _get_kubernetes_join_cmd(self, host):
# The token expires after 24 hours and is needed for a reinstall.
# The puppet manifest handles the case where the node already exists.
try:
join_cmd_additions = ''
if host.personality == constants.CONTROLLER:
# Upload the certificates used during kubeadm join
# The cert key will be printed in the last line of the output
# We will create a temp file with the kubeadm config
# We need this because the kubeadm config could have changed
# since bootstrap. Reading the kubeadm config each time
# it is needed ensures we are not using stale data
fd, temp_kubeadm_config_view = tempfile.mkstemp(
dir='/tmp', suffix='.yaml')
with os.fdopen(fd, 'w') as f:
cmd = ['kubeadm', KUBECONFIG, 'config', 'view']
subprocess.check_call(cmd, stdout=f) # pylint: disable=not-callable
# We will use a custom key to encrypt kubeadm certificates
# to make sure all hosts decrypt using the same key
key = str(keyring.get_password(CERTIFICATE_KEY_SERVICE,
CERTIFICATE_KEY_USER))
with open(temp_kubeadm_config_view, "a") as f:
f.write("---\r\napiVersion: kubeadm.k8s.io/v1beta2\r\n"
"kind: InitConfiguration\r\ncertificateKey: "
"{}".format(key))
cmd = ['kubeadm', 'init', 'phase', 'upload-certs',
'--upload-certs', '--config',
temp_kubeadm_config_view]
subprocess.check_call(cmd) # pylint: disable=not-callable
join_cmd_additions = \
" --control-plane --certificate-key %s" % key
os.unlink(temp_kubeadm_config_view)
# Configure the IP address of the API Server for the controller host.
# If not set the default network interface will be used, which does not
# ensure it will be the Cluster IP address of this host.
host_cluster_ip = self._get_host_cluster_address(host)
join_cmd_additions += \
" --apiserver-advertise-address %s" % host_cluster_ip
cmd = ['kubeadm', KUBECONFIG, 'token', 'create', '--print-join-command',
'--description', 'Bootstrap token for %s' % host.hostname]
join_cmd = subprocess.check_output(cmd) # pylint: disable=not-callable
join_cmd_additions += \
" --cri-socket /var/run/containerd/containerd.sock"
join_cmd = join_cmd.strip() + join_cmd_additions
LOG.info('get_kubernetes_join_cmd join_cmd=%s' % join_cmd)
except Exception:
LOG.exception("Exception generating bootstrap token")
raise exception.SysinvException(
'Failed to generate bootstrap token')
return join_cmd
def get_openstack_pending_install_charts():
try:
return subprocess.check_output(
['helm', '--kubeconfig', '/etc/kubernetes/admin.conf',
'list', '--namespace', 'openstack', '--pending'])
except Exception as e:
raise exception.HelmTillerFailure(
reason="Failed to obtain pending charts list: %s" % e)
def fixed_dylib(dylibs, environ):
for _dylib in dylibs:
_dylib = _dylib.format(**environ)
logger.debug("fixed dylib `%s'" % _dylib)
cmd = ['otool', '-L', _dylib]
for line in subprocess.check_output(cmd).replace('\t', '').split(os.linesep):
if line.startswith('@rpath'):
logger.debug('dylib info: %s' % line)
old_rpath = line.split()[0]
filename = old_rpath.split(os.path.sep)[-1]
new_rpath = os.path.join(environ['DIR_BUNDLES'], 'lib', filename)
logger.debug('old rpath: %s' % old_rpath)
logger.debug('new rpath: %s' % new_rpath)
logger.debug('verify new rpath: %s' % os.path.isfile(new_rpath))
cmd = ['install_name_tool', '-change', old_rpath, new_rpath, _dylib]
logger.debug('run command: %s' % ' '.join(cmd))
logger.debug(subprocess.check_output(cmd))
def assure_tls_ca():
keyout, certout = ('/etc/confluent/tls/cakey.pem',
'/etc/confluent/tls/cacert.pem')
if os.path.exists(certout):
return
try:
os.makedirs('/etc/confluent/tls')
except OSError as e:
if e.errno != 17:
raise
sslcfg = get_openssl_conf_location()
tmpconfig = tempfile.mktemp()
shutil.copy2(sslcfg, tmpconfig)
subprocess.check_call([
'openssl', 'ecparam', '-name', 'secp384r1', '-genkey', '-out', keyout
])
try:
with open(tmpconfig, 'a') as cfgfile:
cfgfile.write('\n[CACert]\nbasicConstraints = CA:true\n')
subprocess.check_call([
'openssl', 'req', '-new', '-x509', '-key', keyout, '-days',
'27300', '-out', certout, '-subj',
'/CN=Confluent TLS Certificate authority ({0})'.format(
socket.gethostname()), '-extensions', 'CACert', '-config',
tmpconfig
])
finally:
os.remove(tmpconfig)
# Could restart the webserver now?
fname = '/var/lib/confluent/public/site/tls/{0}.pem'.format(
collective.get_myname())
try:
os.makedirs(os.path.dirname(fname))
except OSError as e:
if e.errno != 17:
raise
shutil.copy2('/etc/confluent/tls/cacert.pem', fname)
hv = subprocess.check_output([
'openssl', 'x509', '-in', '/etc/confluent/tls/cacert.pem', '-hash',
'-noout'
])
if not isinstance(hv, str):
hv = hv.decode('utf8')
hv = hv.strip()
hashname = '/var/lib/confluent/public/site/tls/{0}.0'.format(hv)
certname = '{0}.pem'.format(collective.get_myname())
for currname in os.listdir('/var/lib/confluent/public/site/tls/'):
currname = os.path.join('/var/lib/confluent/public/site/tls/',
currname)
if currname.endswith('.0'):
try:
realname = os.readlink(currname)
if realname == certname:
os.unlink(currname)
except OSError:
pass
os.symlink(certname, hashname)
def mounted(remote_dir, local_dir):
local_dir = os.path.abspath(local_dir)
try:
subprocess.check_output(["/bin/nfs-mount", remote_dir, local_dir],
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
raise OSError(("mount operation failed: "
"command={}, retcode={}, output='{}'").format(
e.cmd, e.returncode, e.output))
try:
yield
finally:
try:
subprocess.check_output(["/bin/umount", local_dir],
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
raise OSError(("umount operation failed: "
"command={}, retcode={}, output='{}'").format(
e.cmd, e.returncode, e.output))
def _iptables(*args, **kwargs):
# NOTE(dtantsur): -w flag makes it wait for xtables lock
cmd = BASE_COMMAND + args
ignore = kwargs.pop('ignore', False)
LOG.debug('Running iptables %s', args)
kwargs['stderr'] = subprocess.STDOUT
try:
subprocess.check_output(cmd, **kwargs)
except subprocess.CalledProcessError as exc:
output = exc.output.replace('\n', '. ')
if ignore:
LOG.debug('Ignoring failed iptables %(args)s: %(output)s', {
'args': args,
'output': output
})
else:
LOG.error('iptables %(iptables)s failed: %(exc)s', {
'iptables': args,
'exc': output
})
raise
def n3000_img_accessible():
cmd = 'docker image list "%s" --format "{{.Repository}}:{{.Tag}}"' % \
constants.OPAE_IMG
items = subprocess.check_output(
shlex.split(cmd), # pylint: disable=not-callable
stderr=subprocess.STDOUT)
for line in items.splitlines():
if line == constants.OPAE_IMG:
LOG.info('%s image found' % constants.OPAE_IMG)
return True
LOG.info("%s image not found." % constants.OPAE_IMG)
return False
def process_generation(ifname,ofname):
global have_printed_header
gen_num = int(re.findall('gen(\d+)',ifname)[0])
gen_num2 = int(re.findall('gen(\d+)',ofname)[0])
assert(gen_num==gen_num2)
args = [MonKeyTestPath,'-1',ifname,'-2',ofname]
mkout = subprocess.check_output(args)
mklines = mkout.splitlines()
if not have_printed_header:
sys.stdout.write("generation\t"+mklines[0]+'\n')
have_printed_header = True
for line in mklines[1:]:
if line=='': next
sys.stdout.write("%i\t%s\n"%(gen_num,line))
def get_cgts_vg_free_space():
"""Determine free space in cgts-vg"""
try:
# Determine space in cgts-vg in GiB
vg_free_str = subprocess.check_output(
['vgdisplay', '-C', '--noheadings', '--nosuffix',
'-o', 'vg_free', '--units', 'g', 'cgts-vg'],
close_fds=True).rstrip()
cgts_vg_free = int(float(vg_free_str))
except subprocess.CalledProcessError:
LOG.error("Command vgdisplay failed")
raise Exception("Command vgdisplay failed")
return cgts_vg_free
def _get_host_join_command(self, host):
config = {}
if not utils.is_initial_config_complete():
return config
# The token expires after 24 hours and is needed for a reinstall.
# The puppet manifest handles the case where the node already exists.
try:
join_cmd_additions = ''
if host.personality == constants.CONTROLLER:
# Upload the certificates used during kubeadm join
# The cert key will be printed in the last line of the output
cmd = [
'kubeadm', 'init', 'phase', 'upload-certs',
'--upload-certs', '--config',
'/etc/kubernetes/kubeadm.yaml'
]
cmd_output = subprocess.check_output(cmd)
cert_key = cmd_output.strip().split('\n')[-1]
join_cmd_additions = " --control-plane --certificate-key %s" % cert_key
cmd = [
'kubeadm', 'token', 'create', '--print-join-command',
'--description',
'Bootstrap token for %s' % host.hostname
]
join_cmd = subprocess.check_output(cmd)
join_cmd_additions += " --cri-socket /var/run/containerd/containerd.sock"
join_cmd = join_cmd.strip() + join_cmd_additions
except subprocess.CalledProcessError:
raise exception.SysinvException(
'Failed to generate bootstrap token')
config.update({'platform::kubernetes::params::join_cmd': join_cmd})
return config
def _get_keyring_password(self, service, user, pw_format=None):
password = keyring.get_password(service, user)
if not password:
if pw_format == common.PASSWORD_FORMAT_CEPH:
try:
cmd = ['ceph-authtool', '--gen-print-key']
password = subprocess.check_output(cmd).strip()
except subprocess.CalledProcessError:
raise exception.SysinvException(
'Failed to generate ceph key')
else:
password = self._generate_random_password()
keyring.set_password(service, user, password)
# get_password() returns in unicode format, which leads to YAML
# that Armada doesn't like. Converting to UTF-8 is safe because
# we generated the password originally.
return password.encode('utf8', 'strict')
def get_ip_addresses():
lines = subprocess.check_output('ip addr'.split(' '))
if not isinstance(lines, str):
lines = lines.decode('utf8')
for line in lines.split('\n'):
if line.startswith(' inet6 '):
line = line.replace(' inet6 ', '').split('/')[0]
if line == '::1':
continue
elif line.startswith(' inet '):
line = line.replace(' inet ', '').split('/')[0]
if line == '127.0.0.1':
continue
if line.startswith('169.254.'):
continue
else:
continue
yield line
def fetch_and_store_one((index, row)):
try:
sha256 = subprocess.check_output([
'/bin/sh', '-c',
'source /etc/profile; nix-prefetch-url {}'.format(row['url']),
], stderr=open('/dev/null')).strip()
meta = dep_meta.SDist(
pypi_name=row['pypi_name'],
version=row['version'],
url=row['url'],
)
with open(os.path.join(row['store_root'], sha256), 'w') as f:
f.write(meta.SerializeToString())
except subprocess.CalledProcessError as e:
STATS['errors'] += 1
print "error fetching", e, row
def get_n3000_devices():
# First get the PCI addresses of each supported FPGA device
cmd = ["lspci", "-Dm", "-d " + constants.N3000_VENDOR + ":" +
constants.N3000_DEVICE]
try:
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as exc:
msg = ("Failed to get pci devices with vendor %s and device %s, "
"return code is %d, command output: %s." %
(constants.N3000_VENDOR, constants.N3000_DEVICE, exc.returncode, exc.output))
LOG.warn(msg)
raise exception.SysinvException(msg)
# Parse the output of the lspci command and grab the PCI address
fpga_addrs = []
for line in output.splitlines():
line = shlex.split(line.strip())
fpga_addrs.append(line[0])
return fpga_addrs
def assure_agent():
if sshver() <= 7.6:
return False
global agent_pid
if agent_pid is None:
sai = subprocess.check_output(['ssh-agent'])
for line in sai.split(b'\n'):
if b';' not in line:
continue
line, _ = line.split(b';', 1)
if b'=' not in line:
continue
k, v = line.split(b'=', 1)
if not isinstance(k, str):
k = k.decode('utf8')
v = v.decode('utf8')
if k == 'SSH_AGENT_PID':
agent_pid = v
os.environ[k] = v
return True
def thinpools_in_vg(self, vg, cinder_device=None):
"""Return number of thinpools in the specified vg. """
try:
command = ['vgs', '--noheadings', '-o', 'lv_name', vg]
if cinder_device:
if vg == constants.LVG_CINDER_VOLUMES:
global_filer = 'devices/global_filter=["a|' + \
cinder_device + '|","r|.*|"]'
command = command + ['--config', global_filer]
output = subprocess.check_output(command)
except Exception as e:
self.handle_exception("Could not retrieve vgdisplay "
"information: %s" % e)
output = ""
thinpools = 0
for line in output.splitlines():
# This makes some assumptions, the suffix is defined in nova.
if constants.LVM_POOL_SUFFIX in line:
thinpools += 1
return thinpools
def execute_command(name, command, cwd=None, shell=False):
Logger().write(LogVerbosity.Info, "Starting " + name)
start = current_time()
try:
if shell and sys.platform == "linux" or sys.platform == "linux2":
command = " ".join(command)
result = subprocess.check_output(command,
universal_newlines=True,
cwd=cwd,
shell=shell)
Logger().write(LogVerbosity.Debug, name + " output: " + result)
Logger().write(
LogVerbosity.Info,
name + " successful in " + str(current_time() - start) + "ms")
return True
except subprocess.CalledProcessError as e:
Logger().write(
LogVerbosity.Info, name + " failed in " +
str(current_time() - start) + "ms: " + e.output)
return False
def sync_list_to_node(sl, node, suffixes):
targdir = tempfile.mkdtemp('.syncto{}'.format(node))
output = ''
try:
for ent in sl.replacemap:
stage_ent(sl.replacemap, ent, targdir)
if 'append' in suffixes:
while suffixes['append'] and suffixes['append'][0] == '/':
suffixes['append'] = suffixes['append'][1:]
for ent in sl.appendmap:
stage_ent(sl.appendmap, ent,
os.path.join(targdir, suffixes['append']))
if 'merge' in suffixes:
while suffixes['merge'] and suffixes['merge'][0] == '/':
suffixes['merge'] = suffixes['merge'][1:]
for ent in sl.mergemap:
stage_ent(sl.mergemap, ent,
os.path.join(targdir, suffixes['merge']))
sshutil.prep_ssh_key('/etc/confluent/ssh/automation')
output = subprocess.check_output(
['rsync', '-rvL', targdir + '/', 'root@{}:/'.format(node)])
finally:
shutil.rmtree(targdir)
return output
def fix_crushmap(dbapi=None):
""" Set Ceph's CRUSH Map based on storage model """
def _create_crushmap_flag_file():
try:
open(crushmap_flag_file, "w").close()
except IOError as e:
LOG.warn(('Failed to create flag file: {}. '
'Reason: {}').format(crushmap_flag_file, e))
if not dbapi:
dbapi = pecan.request.dbapi
crushmap_flag_file = os.path.join(constants.SYSINV_CONFIG_PATH,
constants.CEPH_CRUSH_MAP_APPLIED)
if not os.path.isfile(crushmap_flag_file):
_operator = CephApiOperator()
if not cutils.is_aio_system(dbapi):
# At least two monitors have to be running on a standard deployment,
# otherwise don't even try to load the crushmap.
active_mons, required_mons, __ = _operator.get_monitors_status(
dbapi)
if required_mons > active_mons:
LOG.info("Not enough monitors yet available to fix crushmap.")
return False
# For AIO system, crushmap should be already loaded through puppet.
# If it was loaded, set the crushmap flag to avoid loading it twice.
default_ceph_tier_name = constants.SB_TIER_DEFAULT_NAMES[
constants.SB_TIER_TYPE_CEPH] + constants.CEPH_CRUSH_TIER_SUFFIX
rule_is_present, __, __ = _operator._crush_rule_status(
default_ceph_tier_name)
if rule_is_present:
_create_crushmap_flag_file()
return False
try:
# For AIO system, crushmap should alreadby be loaded through
# puppet. If for any reason it is not, as a precaution we set
# the crushmap here.
# Check if a backup crushmap exists. If it does, that means
# it is during restore. We need to restore the backup crushmap
# instead of generating it. For non-AIO system, it is stored in
# /opt/platform/sysinv which is a drbd fs. For AIO systems because
# when unlocking controller-0 for the first time, the crushmap is
# set thru ceph puppet when /opt/platform is not mounted yet, we
# store the crushmap in /etc/sysinv.
if cutils.is_aio_system(dbapi):
backup = os.path.join(
constants.CEPH_CRUSH_MAP_BACKUP_DIR_FOR_AIO,
constants.CEPH_CRUSH_MAP_BACKUP)
else:
backup = os.path.join(constants.SYSINV_CONFIG_PATH,
constants.CEPH_CRUSH_MAP_BACKUP)
crushmap_bin = "/etc/sysinv/crushmap.bin"
if os.path.exists(backup):
shutil.copyfile(backup, crushmap_bin)
else:
stor_model = get_ceph_storage_model(dbapi)
if stor_model == constants.CEPH_AIO_SX_MODEL:
crushmap_txt = "/etc/sysinv/crushmap-aio-sx.txt"
elif stor_model == constants.CEPH_CONTROLLER_MODEL:
crushmap_txt = "/etc/sysinv/crushmap-controller-model.txt"
elif stor_model == constants.CEPH_STORAGE_MODEL:
crushmap_txt = "/etc/sysinv/crushmap-storage-model.txt"
else:
reason = "Error: Undefined ceph storage model %s" % stor_model
raise exception.CephCrushMapNotApplied(reason=reason)
LOG.info("Updating crushmap with: %s" % crushmap_txt)
# Compile crushmap
subprocess.check_output("crushtool -c %s "
"-o %s" % (crushmap_txt, crushmap_bin),
stderr=subprocess.STDOUT,
shell=True)
# Set crushmap
subprocess.check_output("ceph osd setcrushmap -i %s" %
crushmap_bin,
stderr=subprocess.STDOUT,
shell=True)
if os.path.exists(backup):
os.remove(backup)
except (IOError, subprocess.CalledProcessError) as e:
# May not be critical, depends on where this is called.
reason = "Error: %s Output: %s" % (str(e), e.output)
raise exception.CephCrushMapNotApplied(reason=reason)
_create_crushmap_flag_file()
return True
return False
def is_drbd_fs_syncing():
output = subprocess.check_output("drbd-overview", stderr=subprocess.STDOUT) # pylint: disable=not-callable
LOG.info("is_drbd_fs_syncing returned '%s'" % output)
if "sync\'ed" in output:
return True
return False
def handle_request(env, start_response):
global currtz
global keymap
global currlocale
global currtzvintage
configmanager.check_quorum()
nodename = env.get('HTTP_CONFLUENT_NODENAME', None)
apikey = env.get('HTTP_CONFLUENT_APIKEY', None)
if not (nodename and apikey):
start_response('401 Unauthorized', [])
yield 'Unauthorized'
return
cfg = configmanager.ConfigManager(None)
eak = cfg.get_node_attributes(nodename, 'crypted.selfapikey').get(
nodename, {}).get('crypted.selfapikey', {}).get('hashvalue', None)
if not eak:
start_response('401 Unauthorized', [])
yield 'Unauthorized'
return
salt = '$'.join(eak.split('$', 3)[:-1]) + '$'
if crypt.crypt(apikey, salt) != eak:
start_response('401 Unauthorized', [])
yield 'Unauthorized'
return
retype = env.get('HTTP_ACCEPT', 'application/yaml')
isgeneric = False
if retype == '*/*':
isgeneric = True
retype = 'application/yaml'
if retype == 'application/yaml':
dumper = yamldump
elif retype == 'application/json':
dumper = json.dumps
else:
start_response('406 Not supported', [])
yield 'Unsupported content type in ACCEPT: ' + retype
return
if env['REQUEST_METHOD'] not in (
'HEAD', 'GET') and 'CONTENT_LENGTH' in env and int(
env['CONTENT_LENGTH']) > 0:
reqbody = env['wsgi.input'].read(int(env['CONTENT_LENGTH']))
if env['PATH_INFO'] == '/self/deploycfg':
if 'HTTP_CONFLUENT_MGTIFACE' in env:
ncfg = netutil.get_nic_config(cfg,
nodename,
ifidx=env['HTTP_CONFLUENT_MGTIFACE'])
else:
myip = env.get('HTTP_X_FORWARDED_HOST', None)
if ']' in myip:
myip = myip.split(']', 1)[0]
else:
myip = myip.split(':', 1)[0]
myip = myip.replace('[', '').replace(']', '')
ncfg = netutil.get_nic_config(cfg, nodename, serverip=myip)
if ncfg['prefix']:
ncfg['ipv4_netmask'] = netutil.cidr_to_mask(ncfg['prefix'])
deployinfo = cfg.get_node_attributes(
nodename, ('deployment.*', 'console.method', 'crypted.*', 'dns.*'))
deployinfo = deployinfo.get(nodename, {})
profile = deployinfo.get('deployment.pendingprofile',
{}).get('value', '')
ncfg['encryptboot'] = deployinfo.get('deployment.encryptboot',
{}).get('value', None)
if ncfg['encryptboot'] in ('', 'none'):
ncfg['encryptboot'] = None
ncfg['profile'] = profile
protocol = deployinfo.get('deployment.useinsecureprotocols',
{}).get('value', 'never')
ncfg['textconsole'] = bool(
deployinfo.get('console.method', {}).get('value', None))
if protocol == 'always':
ncfg['protocol'] = 'http'
else:
ncfg['protocol'] = 'https'
ncfg['rootpassword'] = deployinfo.get('crypted.rootpassword',
{}).get('hashvalue', None)
ncfg['grubpassword'] = deployinfo.get('crypted.grubpassword',
{}).get('grubhashvalue', None)
if currtzvintage and currtzvintage > (time.time() - 30.0):
ncfg['timezone'] = currtz
else:
langinfo = subprocess.check_output(['localectl',
'status']).split(b'\n')
for line in langinfo:
line = line.strip()
if line.startswith(b'System Locale:'):
ccurrlocale = line.split(b'=')[-1]
if not ccurrlocale:
continue
if not isinstance(ccurrlocale, str):
ccurrlocale = ccurrlocale.decode('utf8')
if ccurrlocale == 'n/a':
continue
currlocale = ccurrlocale
elif line.startswith(b'VC Keymap:'):
ckeymap = line.split(b':')[-1]
ckeymap = ckeymap.strip()
if not ckeymap:
continue
if not isinstance(ckeymap, str):
ckeymap = ckeymap.decode('utf8')
if ckeymap == 'n/a':
continue
keymap = ckeymap
tdc = subprocess.check_output(['timedatectl']).split(b'\n')
for ent in tdc:
ent = ent.strip()
if ent.startswith(b'Time zone:'):
currtz = ent.split(b': ', 1)[1].split(b'(', 1)[0].strip()
if not isinstance(currtz, str):
currtz = currtz.decode('utf8')
currtzvintage = time.time()
ncfg['timezone'] = currtz
break
ncfg['locale'] = currlocale
ncfg['keymap'] = keymap
ncfg['nameservers'] = []
for dns in deployinfo.get('dns.servers', {}).get('value',
'').split(','):
ncfg['nameservers'].append(dns)
dnsdomain = deployinfo.get('dns.domain', {}).get('value', None)
ncfg['dnsdomain'] = dnsdomain
start_response('200 OK', (('Content-Type', retype), ))
yield dumper(ncfg)
elif env['PATH_INFO'] == '/self/sshcert':
if not sshutil.ca_exists():
start_response('500 Unconfigured', ())
yield 'CA is not configured on this system (run ...)'
return
dnsinfo = cfg.get_node_attributes(nodename, ('dns.*'))
dnsinfo = dnsinfo.get(nodename, {}).get('dns.domain',
{}).get('value', None)
if dnsinfo in nodename:
dnsinfo = ''
cert = sshutil.sign_host_key(reqbody, nodename, [dnsinfo])
start_response('200 OK', (('Content-Type', 'text/plain'), ))
yield cert
elif env['PATH_INFO'] == '/self/nodelist':
nodes = set(cfg.list_nodes())
domaininfo = cfg.get_node_attributes(nodes, 'dns.domain')
for node in list(util.natural_sort(nodes)):
domain = domaininfo.get(node, {}).get('dns.domain',
{}).get('value', None)
if domain and domain not in node:
nodes.add('{0}.{1}'.format(node, domain))
for mgr in configmanager.list_collective():
nodes.add(mgr)
if domain and domain not in mgr:
nodes.add('{0}.{1}'.format(mgr, domain))
myname = collective.get_myname()
nodes.add(myname)
if domain and domain not in myname:
nodes.add('{0}.{1}'.format(myname, domain))
if isgeneric:
start_response('200 OK', (('Content-Type', 'text/plain'), ))
for node in util.natural_sort(nodes):
yield node + '\n'
else:
start_response('200 OK', (('Content-Type', retype), ))
yield dumper(sorted(nodes))
elif env['PATH_INFO'] == '/self/updatestatus':
update = yaml.safe_load(reqbody)
if update['status'] == 'staged':
targattr = 'deployment.stagedprofile'
elif update['status'] == 'complete':
targattr = 'deployment.profile'
else:
raise Exception('Unknown update status request')
currattr = cfg.get_node_attributes(nodename,
'deployment.*').get(nodename, {})
pending = None
if targattr == 'deployment.profile':
pending = currattr.get('deployment.stagedprofile',
{}).get('value', '')
if not pending:
pending = currattr.get('deployment.pendingprofile',
{}).get('value', '')
updates = {}
if pending:
updates['deployment.pendingprofile'] = {'value': ''}
if targattr == 'deployment.profile':
updates['deployment.stagedprofile'] = {'value': ''}
currprof = currattr.get(targattr, {}).get('value', '')
if currprof != pending:
updates[targattr] = {'value': pending}
cfg.set_node_attributes({nodename: updates})
start_response('200 OK', (('Content-Type', 'text/plain'), ))
yield 'OK'
else:
start_response('500 Error', (('Content-Type', 'text/plain'), ))
yield 'No pending profile detected, unable to accept status update'
else:
start_response('404 Not Found', ())
yield 'Not found'
def lldp_has_neighbour(self, name):
p = subprocess.check_output([
"lldpcli", "-f", "keyvalue", "show", "neighbors", "summary",
"ports", name
])
return len(p) > 0
def get_lspci_output_by_addr(self, pciaddr):
with open(os.devnull, "w") as fnull:
output = subprocess.check_output( # pylint: disable=not-callable
['lspci', '-vmmks', pciaddr],
stderr=fnull)
return output
def handle_request(env, start_response):
global currtz
global keymap
global currlocale
global currtzvintage
configmanager.check_quorum()
nodename = env.get('HTTP_CONFLUENT_NODENAME', None)
apikey = env.get('HTTP_CONFLUENT_APIKEY', None)
if not (nodename and apikey):
start_response('401 Unauthorized', [])
yield 'Unauthorized'
return
cfg = configmanager.ConfigManager(None)
ea = cfg.get_node_attributes(nodename,
['crypted.selfapikey', 'deployment.apiarmed'])
eak = ea.get(nodename, {}).get('crypted.selfapikey',
{}).get('hashvalue', None)
if not eak:
start_response('401 Unauthorized', [])
yield 'Unauthorized'
return
salt = '$'.join(eak.split('$', 3)[:-1]) + '$'
if crypt.crypt(apikey, salt) != eak:
start_response('401 Unauthorized', [])
yield 'Unauthorized'
return
if ea.get(nodename, {}).get('deployment.apiarmed', {}).get('value',
None) == 'once':
cfg.set_node_attributes({nodename: {'deployment.apiarmed': ''}})
retype = env.get('HTTP_ACCEPT', 'application/yaml')
isgeneric = False
if retype == '*/*':
isgeneric = True
retype = 'application/yaml'
if retype == 'application/yaml':
dumper = yamldump
elif retype == 'application/json':
dumper = json.dumps
else:
start_response('406 Not supported', [])
yield 'Unsupported content type in ACCEPT: ' + retype
return
operation = env['REQUEST_METHOD']
if operation not in ('HEAD', 'GET') and 'CONTENT_LENGTH' in env and int(
env['CONTENT_LENGTH']) > 0:
reqbody = env['wsgi.input'].read(int(env['CONTENT_LENGTH']))
if env['PATH_INFO'] == '/self/bmcconfig':
hmattr = cfg.get_node_attributes(nodename, 'hardwaremanagement.*')
hmattr = hmattr.get(nodename, {})
res = {}
port = hmattr.get('hardwaremanagement.port', {}).get('value', None)
if port is not None:
res['bmcport'] = port
vlan = hmattr.get('hardwaremanagement.vlan', {}).get('value', None)
if vlan is not None:
res['bmcvlan'] = vlan
bmcaddr = hmattr.get('hardwaremanagement.manager',
{}).get('value', None)
bmcaddr = socket.getaddrinfo(bmcaddr, 0)[0]
bmcaddr = bmcaddr[-1][0]
if '.' in bmcaddr: # ipv4 is allowed
netconfig = netutil.get_nic_config(cfg, nodename, ip=bmcaddr)
res['bmcipv4'] = bmcaddr
res['prefixv4'] = netconfig['prefix']
res['bmcgw'] = netconfig.get('ipv4_gateway', None)
# credential security results in user/password having to be deferred
start_response('200 OK', (('Content-Type', retype), ))
yield dumper(res)
elif env['PATH_INFO'] == '/self/deploycfg':
if 'HTTP_CONFLUENT_MGTIFACE' in env:
ncfg = netutil.get_nic_config(cfg,
nodename,
ifidx=env['HTTP_CONFLUENT_MGTIFACE'])
else:
myip = env.get('HTTP_X_FORWARDED_HOST', None)
if ']' in myip:
myip = myip.split(']', 1)[0]
else:
myip = myip.split(':', 1)[0]
myip = myip.replace('[', '').replace(']', '')
ncfg = netutil.get_nic_config(cfg, nodename, serverip=myip)
if ncfg['prefix']:
ncfg['ipv4_netmask'] = netutil.cidr_to_mask(ncfg['prefix'])
if ncfg['ipv4_method'] == 'firmwaredhcp':
ncfg['ipv4_method'] = 'static'
deployinfo = cfg.get_node_attributes(
nodename,
('deployment.*', 'console.method', 'crypted.*', 'dns.*', 'ntp.*'))
deployinfo = deployinfo.get(nodename, {})
profile = deployinfo.get('deployment.pendingprofile',
{}).get('value', '')
ncfg['encryptboot'] = deployinfo.get('deployment.encryptboot',
{}).get('value', None)
if ncfg['encryptboot'] in ('', 'none'):
ncfg['encryptboot'] = None
ncfg['profile'] = profile
protocol = deployinfo.get('deployment.useinsecureprotocols',
{}).get('value', 'never')
ncfg['textconsole'] = bool(
deployinfo.get('console.method', {}).get('value', None))
if protocol == 'always':
ncfg['protocol'] = 'http'
else:
ncfg['protocol'] = 'https'
ncfg['rootpassword'] = deployinfo.get('crypted.rootpassword',
{}).get('hashvalue', None)
ncfg['grubpassword'] = deployinfo.get('crypted.grubpassword',
{}).get('grubhashvalue', None)
if currtzvintage and currtzvintage > (time.time() - 30.0):
ncfg['timezone'] = currtz
else:
langinfo = subprocess.check_output(['localectl',
'status']).split(b'\n')
for line in langinfo:
line = line.strip()
if line.startswith(b'System Locale:'):
ccurrlocale = line.split(b'=')[-1]
if not ccurrlocale:
continue
if not isinstance(ccurrlocale, str):
ccurrlocale = ccurrlocale.decode('utf8')
if ccurrlocale == 'n/a':
continue
currlocale = ccurrlocale
elif line.startswith(b'VC Keymap:'):
ckeymap = line.split(b':')[-1]
ckeymap = ckeymap.strip()
if not ckeymap:
continue
if not isinstance(ckeymap, str):
ckeymap = ckeymap.decode('utf8')
if ckeymap == 'n/a':
continue
keymap = ckeymap
tdc = subprocess.check_output(['timedatectl']).split(b'\n')
for ent in tdc:
ent = ent.strip()
if ent.startswith(b'Time zone:'):
currtz = ent.split(b': ', 1)[1].split(b'(', 1)[0].strip()
if not isinstance(currtz, str):
currtz = currtz.decode('utf8')
currtzvintage = time.time()
ncfg['timezone'] = currtz
break
ncfg['locale'] = currlocale
ncfg['keymap'] = keymap
ncfg['nameservers'] = []
for dns in deployinfo.get('dns.servers', {}).get('value',
'').split(','):
ncfg['nameservers'].append(dns)
ntpsrvs = deployinfo.get('ntp.servers', {}).get('value', '')
if ntpsrvs:
ntpsrvs = ntpsrvs.split(',')
if ntpsrvs:
ncfg['ntpservers'] = []
for ntpsrv in ntpsrvs:
ncfg['ntpservers'].append(ntpsrv)
dnsdomain = deployinfo.get('dns.domain', {}).get('value', None)
ncfg['dnsdomain'] = dnsdomain
start_response('200 OK', (('Content-Type', retype), ))
yield dumper(ncfg)
elif env['PATH_INFO'] == '/self/sshcert' and reqbody:
if not sshutil.ca_exists():
start_response('500 Unconfigured', ())
yield 'CA is not configured on this system (run ...)'
return
pals = get_extra_names(nodename, cfg)
cert = sshutil.sign_host_key(reqbody, nodename, pals)
start_response('200 OK', (('Content-Type', 'text/plain'), ))
yield cert
elif env['PATH_INFO'] == '/self/nodelist':
nodes, _ = get_cluster_list(nodename, cfg)
if isgeneric:
start_response('200 OK', (('Content-Type', 'text/plain'), ))
for node in util.natural_sort(nodes):
yield node + '\n'
else:
start_response('200 OK', (('Content-Type', retype), ))
yield dumper(sorted(nodes))
elif env['PATH_INFO'] == '/self/remoteconfigbmc' and reqbody:
try:
reqbody = yaml.safe_load(reqbody)
except Exception:
reqbody = None
cfgmod = reqbody.get('configmod', 'unspecified')
if cfgmod == 'xcc':
xcc.remote_nodecfg(nodename, cfg)
elif cfgmod == 'tsm':
tsm.remote_nodecfg(nodename, cfg)
else:
start_response('500 unsupported configmod', ())
yield 'Unsupported configmod "{}"'.format(cfgmod)
start_response('200 Ok', ())
yield 'complete'
elif env['PATH_INFO'] == '/self/updatestatus' and reqbody:
update = yaml.safe_load(reqbody)
if update['status'] == 'staged':
targattr = 'deployment.stagedprofile'
elif update['status'] == 'complete':
targattr = 'deployment.profile'
else:
raise Exception('Unknown update status request')
currattr = cfg.get_node_attributes(nodename,
'deployment.*').get(nodename, {})
pending = None
if targattr == 'deployment.profile':
pending = currattr.get('deployment.stagedprofile',
{}).get('value', '')
if not pending:
pending = currattr.get('deployment.pendingprofile',
{}).get('value', '')
updates = {}
if pending:
updates['deployment.pendingprofile'] = {'value': ''}
if targattr == 'deployment.profile':
updates['deployment.stagedprofile'] = {'value': ''}
currprof = currattr.get(targattr, {}).get('value', '')
if currprof != pending:
updates[targattr] = {'value': pending}
cfg.set_node_attributes({nodename: updates})
start_response('200 OK', (('Content-Type', 'text/plain'), ))
yield 'OK'
else:
start_response('500 Error', (('Content-Type', 'text/plain'), ))
yield 'No pending profile detected, unable to accept status update'
elif env['PATH_INFO'] == '/self/saveapikey' and reqbody:
if not isinstance(reqbody, str):
reqbody = reqbody.decode('utf8')
cfg.set_node_attributes(
{nodename: {
'deployment.sealedapikey': {
'value': reqbody
}
}})
start_response('200 OK', ())
yield ''
elif env['PATH_INFO'].startswith(
'/self/remoteconfig/') and 'POST' == operation:
scriptcat = env['PATH_INFO'].replace('/self/remoteconfig/', '')
slist, profile = get_scriptlist(
scriptcat, cfg, nodename,
'/var/lib/confluent/public/os/{0}/ansible/{1}')
playlist = []
dirname = '/var/lib/confluent/public/os/{0}/ansible/{1}/'.format(
profile, scriptcat)
if not os.path.isdir(dirname):
dirname = '/var/lib/confluent/public/os/{0}/ansible/{1}.d/'.format(
profile, scriptcat)
for filename in slist:
if filename.endswith('.yaml') or filename.endswith('.yml'):
playlist.append(os.path.join(dirname, filename))
if playlist:
runansible.run_playbooks(playlist, [nodename])
start_response('202 Queued', ())
yield ''
else:
start_response('200 OK', ())
yield ''
return
elif env['PATH_INFO'].startswith('/self/remotesyncfiles'):
if 'POST' == operation:
result = syncfiles.start_syncfiles(nodename, cfg,
json.loads(reqbody))
start_response(result, ())
yield ''
return
if 'GET' == operation:
status, output = syncfiles.get_syncresult(nodename)
start_response(status, ())
yield output
return
elif env['PATH_INFO'].startswith('/self/remoteconfig/status'):
rst = runansible.running_status.get(nodename, None)
if not rst:
start_response('204 Not Running', (('Content-Length', '0'), ))
yield ''
return
start_response('200 OK', ())
if rst.complete:
del runansible.running_status[nodename]
yield rst.dump_text()
return
elif env['PATH_INFO'].startswith('/self/scriptlist/'):
scriptcat = env['PATH_INFO'].replace('/self/scriptlist/', '')
slist, _ = get_scriptlist(
scriptcat, cfg, nodename,
'/var/lib/confluent/public/os/{0}/scripts/{1}')
if slist:
start_response('200 OK', (('Content-Type', 'application/yaml'), ))
yield yaml.safe_dump(util.natural_sort(slist),
default_flow_style=False)
else:
start_response('200 OK', ())
yield ''
else:
start_response('404 Not Found', ())
yield 'Not found'
def merge_overrides(self, file_overrides=None, set_overrides=None):
""" Merge helm overrides together.
:param values: A dict of different types of user override values,
'files' (which generally specify many overrides) and
'set' (which generally specify one override).
"""
if file_overrides is None:
file_overrides = []
if set_overrides is None:
set_overrides = []
# At this point we have potentially two separate types of overrides
# specified by system or user, values from files and values passed in
# via --set . We need to ensure that we call helm using the same
# mechanisms to ensure the same behaviour.
cmd = ['helm', 'install', '--dry-run', '--debug']
# Process the newly-passed-in override values
tmpfiles = []
for value_file in file_overrides:
# For values passed in from files, write them back out to
# temporary files.
tmpfile = tempfile.NamedTemporaryFile(delete=False)
tmpfile.write(value_file)
tmpfile.close()
tmpfiles.append(tmpfile.name)
cmd.extend(['--values', tmpfile.name])
for value_set in set_overrides:
keypair = list(value_set.split("="))
# request user to input with "--set key=value" or
# "--set key=", for the second case, the value is assume ""
# skip setting like "--set =value", "--set xxxx"
if len(keypair) == 2 and keypair[0]:
if keypair[1] and keypair[1].isdigit():
cmd.extend(['--set-string', value_set])
else:
cmd.extend(['--set', value_set])
env = os.environ.copy()
env['KUBECONFIG'] = '/etc/kubernetes/admin.conf'
# Make a temporary directory with a fake chart in it
try:
tmpdir = tempfile.mkdtemp()
chartfile = tmpdir + '/Chart.yaml'
with open(chartfile, 'w') as tmpchart:
tmpchart.write('name: mychart\napiVersion: v1\n'
'version: 0.1.0\n')
cmd.append(tmpdir)
# Apply changes by calling out to helm to do values merge
# using a dummy chart.
output = subprocess.check_output(cmd, env=env)
# Check output for failure
# Extract the info we want.
values = output.split('USER-SUPPLIED VALUES:\n')[1].split(
'\nCOMPUTED VALUES:')[0]
except Exception:
raise
finally:
os.remove(chartfile)
os.rmdir(tmpdir)
for tmpfile in tmpfiles:
os.remove(tmpfile)
return values
