def timed_audit_smart_config(self):
rc = True
if (time.time() - self.last_config_audit) > 1800:
# It's been 30 minutes since the last completed audit
LOG.info("Kicking timed audit")
rc = self.audit_smart_config()
return rc
def upload_dir(self, **kwargs):
files = []
for key, path in kwargs.items():
LOG.info("upload-dir: Retrieving patches from %s" % path)
for f in glob.glob(path + '/*.patch'):
if os.path.isfile(f):
files.append(f)
if len(files) == 0:
return dict(error="No patches found")
try:
result = pc.patch_import_api(sorted(files))
except PatchError as e:
return dict(error=e.message)
pc.patch_sync()
return result
def handle(self, sock, addr):
LOG.info("Handling host install request, force=%s" % self.force)
global pa
resp = PatchMessageAgentInstallResp()
if not os.path.exists(node_is_locked_file):
if self.force:
LOG.info("Installing on unlocked node, with force option")
else:
LOG.info("Rejecting install request on unlocked node")
pa.state = constants.PATCH_AGENT_STATE_INSTALL_REJECTED
pa.rejection_timestamp = time.time()
resp.status = False
resp.reject_reason = 'Node must be locked.'
resp.send(sock, addr)
return
resp.status = pa.handle_install()
resp.send(sock, addr)
def run(self):
self.setup_socket()
while self.sock_out is None:
# Check every thirty seconds?
# Once we've got a conf file, tied into packstack,
# we'll get restarted when the file is updated,
# and this should be unnecessary.
time.sleep(30)
self.setup_socket()
self.setup_tcp_socket()
# Ok, now we've got our socket.
# Let's let the controllers know we're here
hello_ack = PatchMessageHelloAgentAck()
hello_ack.send(self.sock_out)
first_hello = True
connections = []
timeout = time.time() + 30.0
remaining = 30
while True:
inputs = [self.sock_in, self.listener] + connections
outputs = []
rlist, wlist, xlist = select.select(inputs, outputs, inputs,
remaining)
remaining = int(timeout - time.time())
if remaining <= 0 or remaining > 30:
timeout = time.time() + 30.0
remaining = 30
if (len(rlist) == 0 and len(wlist) == 0 and len(xlist) == 0):
# Timeout hit
self.audit_socket()
continue
for s in rlist:
if s == self.listener:
conn, addr = s.accept()
connections.append(conn)
continue
data = ''
addr = None
msg = None
if s == self.sock_in:
# Receive from UDP
data, addr = s.recvfrom(1024)
else:
# Receive from TCP
while True:
try:
packet = s.recv(1024)
except socket.error:
LOG.exception("Socket error on recv")
data = ''
break
if packet:
data += packet
if data == '':
break
try:
datachk = json.loads(data)
break
except ValueError:
# Message is incomplete
continue
else:
# End of TCP message received
break
if data == '':
# Connection dropped
connections.remove(s)
s.close()
continue
msgdata = json.loads(data)
# For now, discard any messages that are not msgversion==1
if 'msgversion' in msgdata and msgdata['msgversion'] != 1:
continue
if 'msgtype' in msgdata:
if msgdata['msgtype'] == messages.PATCHMSG_HELLO_AGENT:
if first_hello:
self.query()
first_hello = False
msg = PatchMessageHelloAgent()
elif msgdata[
'msgtype'] == messages.PATCHMSG_QUERY_DETAILED:
msg = PatchMessageQueryDetailed()
elif msgdata[
'msgtype'] == messages.PATCHMSG_AGENT_INSTALL_REQ:
msg = PatchMessageAgentInstallReq()
if msg is None:
msg = messages.PatchMessage()
msg.decode(msgdata)
if s == self.sock_in:
msg.handle(self.sock_out, addr)
else:
msg.handle(s, addr)
for s in xlist:
if s in connections:
connections.remove(s)
s.close()
# Check for in-service patch restart flag
if os.path.exists(insvc_patch_restart_agent):
# Make sure it's safe to restart, ie. no reqs queued
rlist, wlist, xlist = select.select(inputs, outputs, inputs, 0)
if (len(rlist) == 0 and len(wlist) == 0 and len(xlist) == 0):
# Restart
LOG.info(
"In-service patch restart flag detected. Exiting.")
os.remove(insvc_patch_restart_agent)
exit(0)
def handle_install(self,
verbose_to_stdout=False,
disallow_insvc_patch=False):
#
# The disallow_insvc_patch parameter is set when we're installing
# the patch during init. At that time, we don't want to deal with
# in-service patch scripts, so instead we'll treat any patch as
# a reboot-required when this parameter is set. Rather than running
# any scripts, the RR flag will be set, which will result in the node
# being rebooted immediately upon completion of the installation.
#
LOG.info("Handling install")
# Check the INSTALL_UUID first. If it doesn't match the active
# controller, we don't want to install patches.
if not check_install_uuid():
LOG.error("Failed install_uuid check. Skipping install")
self.patch_failed = True
setflag(patch_failed_file)
self.state = constants.PATCH_AGENT_STATE_INSTALL_FAILED
# Send a hello to provide a state update
if self.sock_out is not None:
hello_ack = PatchMessageHelloAgentAck()
hello_ack.send(self.sock_out)
return False
self.state = constants.PATCH_AGENT_STATE_INSTALLING
setflag(patch_installing_file)
try:
# Create insvc patch directories
if os.path.exists(insvc_patch_scripts):
shutil.rmtree(insvc_patch_scripts, ignore_errors=True)
if os.path.exists(insvc_patch_flags):
shutil.rmtree(insvc_patch_flags, ignore_errors=True)
os.mkdir(insvc_patch_scripts, 0o700)
os.mkdir(insvc_patch_flags, 0o700)
except Exception:
LOG.exception("Failed to create in-service patch directories")
# Send a hello to provide a state update
if self.sock_out is not None:
hello_ack = PatchMessageHelloAgentAck()
hello_ack.send(self.sock_out)
# Build up the install set
if verbose_to_stdout:
print("Checking for software updates...")
self.query()
install_set = []
for pkg, version in self.to_install.items():
install_set.append("%s-%s" % (pkg, version))
install_set += self.missing_pkgs
changed = False
rc = True
if len(install_set) > 0:
try:
if verbose_to_stdout:
print("Installing software updates...")
LOG.info("Installing: %s" % ", ".join(install_set))
output = subprocess.check_output(smart_install_cmd +
install_set,
stderr=subprocess.STDOUT)
changed = True
for line in output.split('\n'):
LOG.info("INSTALL: %s" % line)
if verbose_to_stdout:
print("Software updated.")
except subprocess.CalledProcessError as e:
LOG.exception("Failed to install RPMs")
LOG.error("Command output: %s" % e.output)
rc = False
if verbose_to_stdout:
print("WARNING: Software update failed.")
else:
if verbose_to_stdout:
print("Nothing to install.")
LOG.info("Nothing to install")
if rc:
self.query()
remove_set = self.to_remove
if len(remove_set) > 0:
try:
if verbose_to_stdout:
print("Handling patch removal...")
LOG.info("Removing: %s" % ", ".join(remove_set))
output = subprocess.check_output(smart_remove_cmd +
remove_set,
stderr=subprocess.STDOUT)
changed = True
for line in output.split('\n'):
LOG.info("REMOVE: %s" % line)
if verbose_to_stdout:
print("Patch removal complete.")
except subprocess.CalledProcessError as e:
LOG.exception("Failed to remove RPMs")
LOG.error("Command output: %s" % e.output)
rc = False
if verbose_to_stdout:
print("WARNING: Patch removal failed.")
else:
if verbose_to_stdout:
print("Nothing to remove.")
LOG.info("Nothing to remove")
if changed:
# Update the node_is_patched flag
setflag(node_is_patched_file)
self.node_is_patched = True
if verbose_to_stdout:
print("This node has been patched.")
if os.path.exists(node_is_patched_rr_file):
LOG.info("Reboot is required. Skipping patch-scripts")
elif disallow_insvc_patch:
LOG.info(
"Disallowing patch-scripts. Treating as reboot-required")
setflag(node_is_patched_rr_file)
else:
LOG.info("Running in-service patch-scripts")
try:
subprocess.check_output(run_insvc_patch_scripts_cmd,
stderr=subprocess.STDOUT)
# Clear the node_is_patched flag, since we've handled it in-service
clearflag(node_is_patched_file)
self.node_is_patched = False
except subprocess.CalledProcessError as e:
LOG.exception("In-Service patch scripts failed")
LOG.error("Command output: %s" % e.output)
# Fail the patching operation
rc = False
# Clear the in-service patch dirs
if os.path.exists(insvc_patch_scripts):
shutil.rmtree(insvc_patch_scripts, ignore_errors=True)
if os.path.exists(insvc_patch_flags):
shutil.rmtree(insvc_patch_flags, ignore_errors=True)
if rc:
self.patch_failed = False
clearflag(patch_failed_file)
self.state = constants.PATCH_AGENT_STATE_IDLE
else:
# Update the patch_failed flag
self.patch_failed = True
setflag(patch_failed_file)
self.state = constants.PATCH_AGENT_STATE_INSTALL_FAILED
clearflag(patch_installing_file)
self.query()
# Send a hello to provide a state update
if self.sock_out is not None:
hello_ack = PatchMessageHelloAgentAck()
hello_ack.send(self.sock_out)
return rc
def query(self):
""" Check current patch state """
if not check_install_uuid():
LOG.info("Failed install_uuid check. Skipping query")
return False
if not self.audit_smart_config():
# Set a state to "unknown"?
return False
try:
subprocess.check_output(smart_update, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
LOG.error("Failed to update smartpm")
LOG.error("Command output: %s" % e.output)
# Set a state to "unknown"?
return False
# Generate a unique query id
self.query_id = random.random()
self.changes = False
self.installed = {}
self.to_install = {}
self.to_remove = []
self.missing_pkgs = []
# Get the repo data
pkgs_installed = {}
pkgs_base = {}
pkgs_updates = {}
try:
output = subprocess.check_output(smart_query_installed)
pkgs_installed = self.parse_smart_pkglist(output)
except subprocess.CalledProcessError as e:
LOG.error("Failed to query installed pkgs: %s" % e.output)
# Set a state to "unknown"?
return False
try:
output = subprocess.check_output(smart_query_base)
pkgs_base = self.parse_smart_pkglist(output)
except subprocess.CalledProcessError as e:
LOG.error("Failed to query base pkgs: %s" % e.output)
# Set a state to "unknown"?
return False
try:
output = subprocess.check_output(smart_query_updates)
pkgs_updates = self.parse_smart_pkglist(output)
except subprocess.CalledProcessError as e:
LOG.error("Failed to query patched pkgs: %s" % e.output)
# Set a state to "unknown"?
return False
# There are four possible actions:
# 1. If installed pkg is not in base or updates, remove it.
# 2. If installed pkg version is higher than highest in base
# or updates, downgrade it.
# 3. If installed pkg version is lower than highest in updates,
# upgrade it.
# 4. If pkg in grouplist is not in installed, install it.
for pkg in pkgs_installed:
for arch in pkgs_installed[pkg]:
installed_version = pkgs_installed[pkg][arch]
updates_version = self.get_pkg_version(pkgs_updates, pkg, arch)
base_version = self.get_pkg_version(pkgs_base, pkg, arch)
if updates_version is None and base_version is None:
# Remove it
self.to_remove.append(pkg)
self.changes = True
continue
compare_version = updates_version
if compare_version is None:
compare_version = base_version
# Compare the installed version to what's in the repo
rc = rpm.labelCompare(stringToVersion(installed_version),
stringToVersion(compare_version))
if rc == 0:
# Versions match, nothing to do.
continue
else:
# Install the version from the repo
self.to_install[pkg] = "@".join([compare_version, arch])
self.installed[pkg] = "@".join([installed_version, arch])
self.changes = True
# Look for new packages
self.check_groups()
LOG.info("Patch state query returns %s" % self.changes)
LOG.info("Installed: %s" % self.installed)
LOG.info("To install: %s" % self.to_install)
LOG.info("To remove: %s" % self.to_remove)
LOG.info("Missing: %s" % self.missing_pkgs)
return True
def audit_smart_config(self):
LOG.info("Auditing smart configuration")
# Get the current channel config
try:
output = subprocess.check_output(smart_cmd + ["channel", "--yaml"],
stderr=subprocess.STDOUT)
config = yaml.load(output)
except subprocess.CalledProcessError as e:
LOG.exception("Failed to query channels")
LOG.error("Command output: %s" % e.output)
return False
except Exception:
LOG.exception("Failed to query channels")
return False
expected = [{
'channel': 'rpmdb',
'type': 'rpm-sys',
'name': 'RPM Database',
'baseurl': None
}, {
'channel':
'base',
'type':
'rpm-md',
'name':
'Base',
'baseurl':
"http://controller:%s/feed/rel-%s" % (http_port, SW_VERSION)
}, {
'channel':
'updates',
'type':
'rpm-md',
'name':
'Patches',
'baseurl':
"http://controller:%s/updates/rel-%s" % (http_port, SW_VERSION)
}]
updated = False
for item in expected:
channel = item['channel']
ch_type = item['type']
ch_name = item['name']
ch_baseurl = item['baseurl']
add_channel = False
if channel in config:
# Verify existing channel config
if (config[channel].get('type') != ch_type
or config[channel].get('name') != ch_name
or config[channel].get('baseurl') != ch_baseurl):
# Config is invalid
add_channel = True
LOG.warning("Invalid smart config found for %s" % channel)
try:
output = subprocess.check_output(
smart_cmd +
["channel", "--yes", "--remove", channel],
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
LOG.exception("Failed to configure %s channel" %
channel)
LOG.error("Command output: %s" % e.output)
return False
else:
# Channel is missing
add_channel = True
LOG.warning("Channel %s is missing from config" % channel)
if add_channel:
LOG.info("Adding channel %s" % channel)
cmd_args = [
"channel", "--yes", "--add", channel,
"type=%s" % ch_type,
"name=%s" % ch_name
]
if ch_baseurl is not None:
cmd_args += ["baseurl=%s" % ch_baseurl]
try:
output = subprocess.check_output(smart_cmd + cmd_args,
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
LOG.exception("Failed to configure %s channel" % channel)
LOG.error("Command output: %s" % e.output)
return False
updated = True
# Validate the smart config
try:
output = subprocess.check_output(smart_cmd + ["config", "--yaml"],
stderr=subprocess.STDOUT)
config = yaml.load(output)
except subprocess.CalledProcessError as e:
LOG.exception("Failed to query smart config")
LOG.error("Command output: %s" % e.output)
return False
except Exception:
LOG.exception("Failed to query smart config")
return False
# Check for the rpm-nolinktos flag
nolinktos = 'rpm-nolinktos'
if config.get(nolinktos) is not True:
# Set the flag
LOG.warning("Setting %s option" % nolinktos)
try:
output = subprocess.check_output(
smart_cmd + ["config", "--set",
"%s=true" % nolinktos],
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
LOG.exception("Failed to configure %s option" % nolinktos)
LOG.error("Command output: %s" % e.output)
return False
updated = True
# Check for the rpm-check-signatures flag
nosignature = 'rpm-check-signatures'
if config.get(nosignature) is not False:
# Set the flag
LOG.warning("Setting %s option" % nosignature)
try:
output = subprocess.check_output(
smart_cmd + ["config", "--set",
"%s=false" % nosignature],
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
LOG.exception("Failed to configure %s option" % nosignature)
LOG.error("Command output: %s" % e.output)
return False
updated = True
if updated:
try:
subprocess.check_output(smart_update, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
LOG.exception("Failed to update smartpm")
LOG.error("Command output: %s" % e.output)
return False
# Reset the patch op counter to force a detailed query
self.patch_op_counter = 0
self.last_config_audit = time.time()
return True
def handle(self, sock, addr):
# Send response
LOG.info("Handling detailed query")
resp = PatchMessageQueryDetailedResp()
resp.send(sock)
def handle(self, sock, addr):
LOG.info("Unhandled message type: %s" % self.msgtype)