def cberror(ignored):
out.warn('{} to {} failed'.format(request.method, request.url))
if self.max_retries is None or self.retries < self.max_retries:
reactor.callLater(self.retryDelay, self.send, report)
self.retries += 1
self.increaseDelay()
nexus.core.jwt_valid = False
def reloadAll(update):
"""
Reload pdconf configuration files.
"""
# Note: reloading all config files at once seems safer than individual
# files because of cross-dependencies.
statusString = client.reloadAll()
# Check the status to make sure all configuration sections
# related to this chute were successfully loaded.
#
# We should only abort if there were problems related to this chute.
# Example: a WiFi card was removed, so we fail to bring up old WiFi
# interfaces; however, installing a new chute that does not depend on WiFi
# should still succeed.
status = json.loads(statusString)
for section in status:
# Checking age > 0 filters out errors that occurred in the past.
if section['success'] or section['age'] > 0:
continue
message = "Error installing configuration section {} {} for chute {}".format(
section['type'], section['name'], section['comment'])
if section['comment'] == update.new.name:
raise Exception(message)
else:
out.warn(message)
def start_container(update, service):
"""
Start running a service in a new container.
"""
client = docker.DockerClient(base_url="unix://var/run/docker.sock", version='auto')
container_name = service.get_container_name()
image_name = service.get_image_name()
out.info("Attempting to start container {} from image {}\n".format(
container_name, image_name))
host_config = build_host_config(update, service)
# TODO
# Set environment variables for the new container.
# PARADROP_ROUTER_ID can be used to change application behavior based on
# what router it is running on.
environment = prepare_environment(update, service)
try:
container = client.containers.run(detach=True, image=image_name,
name=container_name, environment=environment, **host_config)
out.info("Successfully started chute with Id: %s\n" % (str(container.id)))
except Exception as e:
raise e
try:
network = client.networks.get(update.new.name)
network.connect(container_name, aliases=[service.name])
except docker.errors.NotFound:
out.warn("Bridge network {} not found; connectivity between containers is limited.".format(update.new.name))
def reloadAll(update):
"""
Reload pdconf configuration files.
"""
# Note: reloading all config files at once seems safer than individual
# files because of cross-dependencies.
statusString = client.reloadAll()
# Check the status to make sure all configuration sections
# related to this chute were successfully loaded.
#
# We should only abort if there were problems related to this chute.
# Example: a WiFi card was removed, so we fail to bring up old WiFi
# interfaces; however, installing a new chute that does not depend on WiFi
# should still succeed.
status = json.loads(statusString)
for section in status:
# Checking age > 0 filters out errors that occurred in the past.
if section['success'] or section['age'] > 0:
continue
message = "Error installing configuration section {} {} for chute {}".format(
section['type'], section['name'], section['comment'])
if section['comment'] == update.new.name:
raise Exception(message)
else:
out.warn(message)
def _setResourceAllocation(allocation):
client = docker.DockerClient(base_url="unix://var/run/docker.sock",
version='auto')
for container_name, resources in six.iteritems(allocation):
out.info("Update chute {} set cpu_shares={}\n".format(
container_name, resources['cpu_shares']))
container = client.containers.get(container_name)
container.update(cpu_shares=resources['cpu_shares'])
# Using class id 1:1 for prioritized, 1:3 for best effort.
# Prioritization is implemented in confd/qos.py. Class-ID is
# represented in hexadecimal.
# Reference: https://www.kernel.org/doc/Documentation/cgroup-v1/net_cls.txt
if resources.get('prioritize_traffic', False):
classid = "0x10001"
else:
classid = "0x10003"
container = ChuteContainer(container_name)
try:
container_id = container.getID()
fname = "/sys/fs/cgroup/net_cls/docker/{}/net_cls.classid".format(
container_id)
with open(fname, "w") as output:
output.write(classid)
except Exception as error:
out.warn("Error setting traffic class: {}\n".format(error))
def onStart(self):
super(Nexus, self).onStart()
# onStart is called when the reactor starts, not when the connection is made.
# Check for provisioning keys and attempt to connect
if not self.provisioned() and not provisioning.can_provision():
out.warn("The node is not provisioned and is not configured to self-provision.")
return
while not self.provisioned():
yield provisioning.provision_self(self.update_manager)
try:
# Set up communication with pdserver.
# 1. Create a report of the current system state and send that.
# 2. Send the node public key.
# 3. Poll for a list of updates that should be applied.
# 4. Open WAMP session.
yield sendStateReport()
yield sendNodeIdentity()
yield self.update_fetcher.start_polling()
yield self.connect(WampSession)
except Exception:
out.warn('The router ID or password is invalid!')
else:
out.info('WAMP session is ready!')
def call_retry(cmd, env, delay=3, tries=3):
# Make sure each component of the command is a string. Otherwisew we will
# get errors.
clean_cmd = [str(v) for v in cmd]
while tries >= 0:
tries -= 1
out.info("Calling: {}\n".format(" ".join(clean_cmd)))
try:
proc = subprocess.Popen(clean_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
for line in proc.stdout:
out.info("{}: {}\n".format(clean_cmd[0], line.strip()))
for line in proc.stderr:
out.warn("{}: {}\n".format(clean_cmd[0], line.strip()))
return proc.returncode
except OSError as e:
out.warn('Command "{}" failed\n'.format(" ".join(clean_cmd)))
if tries <= 0:
out.exception(e, True)
raise e
time.sleep(delay)
def onStart(self):
super(Nexus, self).onStart()
# onStart is called when the reactor starts, not when the connection is made.
# Check for provisioning keys and attempt to connect
if not self.provisioned() and not provisioning.can_provision():
out.warn(
"The node is not provisioned and is not configured to self-provision."
)
return
while not self.provisioned():
yield provisioning.provision_self(self.update_manager)
try:
# Set up communication with pdserver.
# 1. Create a report of the current system state and send that.
# 2. Send the node public key.
# 3. Poll for a list of updates that should be applied.
# 4. Open WAMP session.
yield sendStateReport()
yield sendNodeIdentity()
yield self.update_fetcher.start_polling()
yield self.connect(WampSession)
except Exception:
out.warn('The router ID or password is invalid!')
else:
out.info('WAMP session is ready!')
def sshKeys(self, request, user):
"""
Manage list of authorized keys for SSH access.
"""
cors.config_cors(request)
request.setHeader('Content-Type', 'application/json')
if request.method == "GET":
try:
if GovernorClient.isAvailable():
client = GovernorClient()
keys = client.listAuthorizedKeys(user)
else:
keys = ssh_keys.getAuthorizedKeys(user)
return json.dumps(keys)
except Exception as e:
out.warn(str(e))
request.setResponseCode(404)
return json.dumps({'message': str(e)})
else:
body = str2json(request.content.read())
key = body['key'].strip()
try:
if GovernorClient.isAvailable():
client = GovernorClient()
client.addAuthorizedKey(key, user)
else:
ssh_keys.addAuthorizedKey(key, user)
return json.dumps(body)
except Exception as e:
out.warn(str(e))
request.setResponseCode(404)
return json.dumps({'message': str(e)})
def _setResourceAllocation(allocation):
client = docker.DockerClient(base_url="unix://var/run/docker.sock", version='auto')
for container_name, resources in six.iteritems(allocation):
out.info("Update chute {} set cpu_shares={}\n".format(
container_name, resources['cpu_shares']))
container = client.containers.get(container_name)
container.update(cpu_shares=resources['cpu_shares'])
# Using class id 1:1 for prioritized, 1:3 for best effort.
# Prioritization is implemented in confd/qos.py. Class-ID is
# represented in hexadecimal.
# Reference: https://www.kernel.org/doc/Documentation/cgroup-v1/net_cls.txt
if resources.get('prioritize_traffic', False):
classid = "0x10001"
else:
classid = "0x10003"
container = ChuteContainer(container_name)
try:
container_id = container.getID()
fname = "/sys/fs/cgroup/net_cls/docker/{}/net_cls.classid".format(container_id)
with open(fname, "w") as output:
output.write(classid)
except Exception as error:
out.warn("Error setting traffic class: {}\n".format(error))
def call_in_netns(service, env, command, onerror="raise", pid=None):
"""
Call command within a service's namespace.
command: should be a list of strings.
onerror: should be "raise" or "ignore"
"""
container_name = service.get_container_name()
if pid is None:
# We need the chute's PID in order to work with Linux namespaces.
container = ChuteContainer(container_name)
pid = container.getPID()
# Try first with `nsenter`. This is preferred because it works using
# commands in the host. We cannot be sure the `docker exec` version will
# work with all chute images.
cmd = ['nsenter', '--target', str(pid), '--net', '--no-fork'] + command
try:
code = call_retry(cmd, env, tries=1)
except:
code = -1
# We fall back to `docker exec` which relies on the container image having
# an `ip` command available to configure interfaces from within.
if code != 0:
out.warn("nsenter command failed, resorting to docker exec\n")
try:
client = docker.DockerClient(base_url="unix://var/run/docker.sock", version='auto')
container = client.containers.get(container_name)
container.exec_run(command, user='******')
except Exception:
if onerror == "raise":
raise
def call_in_netns(chute, env, command, onerror="raise", pid=None):
"""
Call command within a chute's namespace.
command: should be a list of strings.
onerror: should be "raise" or "ignore"
"""
if pid is None:
# We need the chute's PID in order to work with Linux namespaces.
container = ChuteContainer(chute.name)
pid = container.getPID()
# Try first with `nsenter`. This is preferred because it works using
# commands in the host. We cannot be sure the `docker exec` version will
# work with all chute images.
cmd = ['nsenter', '--target', str(pid), '--net', '--no-fork'] + command
try:
code = call_retry(cmd, env, tries=1)
except:
code = -1
# We fall back to `docker exec` which relies on the container image having
# an `ip` command available to configure interfaces from within.
if code != 0:
out.warn("nsenter command failed, resorting to docker exec\n")
try:
client = docker.DockerClient(base_url="unix://var/run/docker.sock",
version='auto')
container = client.containers.get(chute.name)
container.exec_run(command, user='******')
except Exception as error:
if onerror == "raise":
raise
def cberror(ignored):
out.warn('{} to {} failed'.format(request.method, request.url))
if self.max_retries is None or self.retries < self.max_retries:
reactor.callLater(self.retryDelay, self.send, report)
self.retries += 1
self.increaseDelay()
nexus.core.jwt_valid = False
def decorated(self, request, *args, **kwargs):
if not check_auth(request, self.password_manager, self.token_manager):
out.warn('Failed to authenticate')
request.setResponseCode(401)
request.setHeader("WWW-Authenticate",
"Basic realm=\"Login Required\"")
return
return func(self, request, *args, **kwargs)
def executePlans(update):
"""
Primary function that actually executes all the functions that were added to plans by all
the exc modules. This function can heavily modify the OS/files/etc.. so the return value is
very important.
Returns:
True in error : abortPlans function should be called
False otherwise : everything is OK
"""
out.header('Executing plans %r\n' % (update))
# Finding the functions to call is actually done by a 'iterator' like function in the plangraph module
while (True):
# This function either returns None or a tuple just like generate added to it
p = update.plans.getNextTodo()
# No more to do?
if (not p):
break
# Explode tuple otherwise
func, args = p
# We are in a try-except block so if func isn't callable that will catch it
try:
out.verbose('Calling %s\n' % (func))
update.progress("Calling {}".format(func.__name__))
#
# Call the function from the execution plan
#
# args may be empty, but we don't want to pass in a tuple if we don't need to.
# This below explodes the args so if @args is (), then what is passed is @update
skipme = func(*((update, ) + args))
except Exception as e:
out.exception(e, True)
# plans = str(update.plans)) # Removed because breaks new out.exception call
out.warn("Failed to execute plan %s%s" % (func.__name__, args))
update.responses.append({
'exception': str(e),
'traceback': traceback.format_exc()
})
update.failure = str(e)
return True
# The functions we call here can return other functions, if they do these are functions that should
# be skipped later on (for instance a set* function discovering it didn't change anything, later on
# we shouldn't call the corresponding reload function)
if (skipme):
# These functions can return individual functions to skip, or a list of multiple functions
if (not isinstance(skipme, list)):
skipme = [skipme]
for skip in skipme:
out.warn('Identified a skipped function: %r\n' % (skip))
update.plans.registerSkip(skip)
# Now we are done
return False
def cbresponse(response):
if not response.success:
out.warn('{} to {} returned code {}'.format(
request.method, request.url, response.code))
if self.max_retries is None or self.retries < self.max_retries:
reactor.callLater(self.retryDelay, self.send, report)
self.retries += 1
self.increaseDelay()
nexus.core.jwt_valid = False
else:
nexus.core.jwt_valid = True
def getPid(self):
if self.fromFile:
try:
with open(self.pid, "r") as inputFile:
return int(inputFile.read().strip())
except:
# No pid file --- maybe it was not running?
out.warn("File not found: {}\n".format(self.pid))
return None
else:
return self.pid
def cbresponse(response):
if not response.success:
out.warn('{} to {} returned code {}'.format(request.method,
request.url, response.code))
if self.max_retries is None or self.retries < self.max_retries:
reactor.callLater(self.retryDelay, self.send, report)
self.retries += 1
self.increaseDelay()
nexus.core.jwt_valid = False
else:
nexus.core.jwt_valid = True
def getPid(self):
if self.fromFile:
try:
with open(self.pid, "r") as inputFile:
return int(inputFile.read().strip())
except:
# No pid file --- maybe it was not running?
out.warn("File not found: {}\n".format(self.pid))
return None
else:
return self.pid
def sendTelemetryReport():
# Do not try to send telemetry report if not provisioned.
if not nexus.core.provisioned():
if not getattr(sendTelemetryReport, 'provisionWarningShown', False):
out.warn("Unable to send telemetry report: not provisioned")
sendTelemetryReport.provisionWarningShown = True
return None
builder = TelemetryReportBuilder()
report = builder.prepare()
sender = ReportSender(model="telemetry", max_retries=0)
return sender.send(report)
def sendTelemetryReport():
# Do not try to send telemetry report if not provisioned.
if not nexus.core.provisioned():
if not getattr(sendTelemetryReport, 'provisionWarningShown', False):
out.warn("Unable to send telemetry report: not provisioned")
sendTelemetryReport.provisionWarningShown = True
return None
builder = TelemetryReportBuilder()
report = builder.prepare()
sender = ReportSender(model="telemetry", max_retries=0)
return sender.send(report)
def _removeImage(chute):
"""
Remove the image for a chute.
"""
image = getImageName(chute)
out.info("Removing image {}\n".format(image))
try:
client = docker.DockerClient(base_url="unix://var/run/docker.sock",
version='auto')
client.images.remove(image=image)
except Exception as error:
out.warn("Error removing image: {}".format(error))
def update_chute(self, request, chute):
cors.config_cors(request)
if not chute_access_allowed(request, chute):
return permission_denied(request)
update = dict(updateClass='CHUTE',
updateType='update',
tok=pdutils.timeint(),
user=request.user,
name=chute)
ctype = request.requestHeaders.getRawHeaders('Content-Type',
default=[None])[0]
if ctype == "application/x-tar":
workdir, paradrop_yaml = extract_tarred_chute(request.content)
config = paradrop_yaml.get("config", {})
# Try to read chute name from top level (preferred) or from config
# object (deprecated).
if 'name' in paradrop_yaml:
update['name'] = paradrop_yaml['name']
elif 'name' in config:
out.warn("Deprecated: move chute name to top level of config file.")
update['name'] = config['name']
else:
raise Exception("Chute name not found in configuration file.")
update['workdir'] = workdir
update.update(config)
else:
body = json.loads(request.content.read())
config = body['config']
update.update(config)
# Set a time-based version number for side-loaded chutes because we do
# not expect the to receive it from the config file.
update['version'] = "x{}".format(update['tok'])
# We will return the change ID to the caller for tracking and log
# retrieval.
update['change_id'] = self.update_manager.assign_change_id()
self.update_manager.add_update(**update)
result = {
'change_id': update['change_id']
}
request.setHeader('Content-Type', 'application/json')
return json.dumps(result)
def execute(self):
"""
The function that actually walks through the main process required to create the chute.
It follows the executeplan module through the paces of:
1) Generate the plans for each plan module
2) Prioritize the plans
3) Execute the plans
If at any point we fail then this function will directly take care of completing
the update process with an error state and will close the API connection.
"""
if not self.execute_called:
# Save a timestamp from when we started execution.
self.startTime = time.time()
# Generate the plans we need to setup the chute
if (executionplan.generatePlans(self)):
out.warn('Failed to generate plans\n')
self.complete(success=False, message=self.failure)
return
# Aggregate those plans
executionplan.aggregatePlans(self)
self.execute_called = True
# Execute on those plans
exec_result = executionplan.executePlans(self)
if isinstance(exec_result, defer.Deferred):
return exec_result
elif exec_result is True:
# Getting here means we need to abort what we did
res = executionplan.abortPlans(self)
# Did aborting also fail? This is bad!
if (res):
###################################################################################
# Getting here means the abort system thinks it wasn't able to get the system
# back into the state it was in prior to this update.
###################################################################################
out.err('TODO: What do we do when we fail during abort?\n')
pass
# Report the failure back to the user
self.complete(success=False, message=self.failure)
return
# Respond to the API server to let them know the result
self.complete(success=True,
message='Chute {} {} success'.format(
self.name, self.updateType))
def appendCache(self, key, val):
"""
Finds the key they requested and appends the val into it, this function assumes the cache object
is of list type, if the key hasn't been defined yet then it will set it to an empty list.
"""
r = self.getCache(key)
if (not r):
r = []
elif (not isinstance(r, list)):
out.warn('Unable to append to cache, not list type\n')
return
r.append(val)
self.setCache(key, r)
return True
def remove_image(update, service):
"""
Remove a Docker image.
"""
client = docker.APIClient(base_url="unix://var/run/docker.sock", version='auto')
image_name = service.get_image_name()
out.info("Removing image {}\n".format(image_name))
try:
client = docker.DockerClient(base_url="unix://var/run/docker.sock",
version='auto')
client.images.remove(image=image_name)
except Exception as error:
out.warn("Error removing image: {}".format(error))
def _perform_updates(self, checkDocker=True):
"""This is the main working function of the PDConfigurer class.
It should be executed as a separate thread, it does the following:
checks for any updates to perform
does them
responds to the server
removes the update
checks for more updates
if more exist it calls itself again more quickly
else it puts itself to sleep for a little while
"""
if checkDocker:
ready = dockerMonitor.ensureReady()
if not ready:
out.warn("Docker does not appear to be running. "
"Most functionality with containers will be broken.")
ready = containerdMonitor.ensureReady()
if not ready:
out.warn("Docker containerd does not appear to be running. "
"Most functionality with containers will be broken.")
# add any chutes that should already be running to the front of the
# update queue before processing any updates
startQueue = reloadChutes()
self.updateLock.acquire()
self.updateQueue.append(self._make_router_update("prehostconfig"))
self.updateQueue.append(self._make_router_update("inithostconfig"))
self.updateQueue.extend(update_object.parse(u) for u in startQueue)
self.updateLock.release()
# Always perform this work
while self.reactor.running:
# Check for new updates
self.active_change = self._get_next_update()
if self.active_change is None:
time.sleep(1)
continue
self._perform_update(self.active_change)
self.active_change = None
# Apply a batch of updates and when the queue is empty, send a
# state report. We're not reacquiring the mutex here because the
# worst case is we send out an extra state update.
if len(self.updateQueue) == 0 and nexus.core.provisioned():
threads.blockingCallFromThread(self.reactor,
reporting.sendStateReport)
def remove_image(update, service):
"""
Remove a Docker image.
"""
client = docker.APIClient(base_url="unix://var/run/docker.sock",
version='auto')
image_name = service.get_image_name()
out.info("Removing image {}\n".format(image_name))
try:
client = docker.DockerClient(base_url="unix://var/run/docker.sock",
version='auto')
client.images.remove(image=image_name)
except Exception as error:
out.warn("Error removing image: {}".format(error))
def handleMissingWiFi(hostConfig):
"""
Take appropriate action in response to missing WiFi devices.
Depending on the host configuration, we may either emit a warning or reboot
the system.
"""
# Missing WiFi devices - check what we should do.
action = datastruct.getValue(hostConfig, "system.onMissingWiFi")
if action == "reboot":
out.warn("Missing WiFi devices, system will be rebooted.")
cmd = ["shutdown", "-r", "now"]
subprocess.call(cmd)
elif action == "warn":
out.warn("Missing WiFi devices.")
def handleMissingWiFi(hostConfig):
"""
Take appropriate action in response to missing WiFi devices.
Depending on the host configuration, we may either emit a warning or reboot
the system.
"""
# Missing WiFi devices - check what we should do.
action = datastruct.getValue(hostConfig, "system.onMissingWiFi")
if action == "reboot":
out.warn("Missing WiFi devices, system will be rebooted.")
cmd = ["shutdown", "-r", "now"]
subprocess.call(cmd)
elif action == "warn":
out.warn("Missing WiFi devices.")
def _perform_updates(self):
"""This is the main working function of the PDConfigurer class.
It should be executed as a separate thread, it does the following:
checks for any updates to perform
does them
responds to the server
removes the update
checks for more updates
if more exist it calls itself again more quickly
else it puts itself to sleep for a little while
"""
if settings.CHECK_DOCKER:
ready = dockerMonitor.ensureReady()
if not ready:
out.warn("Docker does not appear to be running. "
"Most functionality with containers will be broken.")
ready = containerdMonitor.ensureReady()
if not ready:
out.warn("Docker containerd does not appear to be running. "
"Most functionality with containers will be broken.")
# add any chutes that should already be running to the front of the
# update queue before processing any updates
startQueue = reloadChutes()
self.updateLock.acquire()
self.updateQueue.append(self._make_router_update("prehostconfig"))
self.updateQueue.append(self._make_router_update("inithostconfig"))
self.updateQueue.extend(startQueue)
self.updateLock.release()
# Always perform this work
while self.reactor.running:
# Check for new updates
change = self._get_next_update()
if change is None:
time.sleep(1)
continue
self._perform_update(change)
# Apply a batch of updates and when the queue is empty, send a
# state report. We're not reacquiring the mutex here because the
# worst case is we send out an extra state update.
if len(self.active_changes) == 0 and nexus.core.provisioned():
threads.blockingCallFromThread(self.reactor,
reporting.sendStateReport)
def getWifiKeySettings(cfg, iface):
"""
Read encryption settings from cfg and transfer them to iface.
"""
# If 'key' is present, but 'encryption' is not, then default to
# psk2. If 'key' is not present, and 'encryption' is psk or psk2,
# then we have an error.
iface['encryption'] = "none"
if 'key' in cfg:
iface['key'] = cfg['key']
iface['encryption'] = 'psk2' # default to psk2
if 'encryption' in cfg:
iface['encryption'] = cfg['encryption']
if cfg['encryption'].startswith('psk') and 'key' not in cfg:
out.warn("Key field must be defined "
"when encryption is enabled.")
raise Exception("No key field defined for WiFi encryption")
def getWifiKeySettings(cfg, iface):
"""
Read encryption settings from cfg and transfer them to iface.
"""
# If 'key' is present, but 'encryption' is not, then default to
# psk2. If 'key' is not present, and 'encryption' is psk or psk2,
# then we have an error.
iface['encryption'] = "none"
if 'key' in cfg:
iface['key'] = cfg['key']
iface['encryption'] = 'psk2' # default to psk2
if 'encryption' in cfg:
iface['encryption'] = cfg['encryption']
if cfg['encryption'].startswith('psk') and 'key' not in cfg:
out.warn("Key field must be defined "
"when encryption is enabled.")
raise Exception("No key field defined for WiFi encryption")
def getBridgeGateway():
"""
Look up the gateway IP address for the docker bridge network.
This is the docker0 IP address; it is the IP address of the host from the
chute's perspective.
"""
client = docker.DockerClient(base_url="unix://var/run/docker.sock", version='auto')
network = client.networks.get("bridge")
for config in network.attrs['IPAM']['Config']:
if 'Gateway' in config:
return config['Gateway']
# Fall back to a default if we could not find it. This address will work
# in most places unless Docker changes to use a different address.
out.warn('Could not find bridge gateway, using default')
return '172.17.0.1'
def progress(self, message):
if self.pkg is not None:
self.pkg.request.write(message + '\n')
# TODO Look into this.
# This might happen during router initialization. If nexus.core is
# None, we do not know the router's identity, so we cannot publish any
# messages.
if nexus.core is None:
return
data = {'time': time.time(), 'message': message}
def handleError(error):
print("Error sending message: {}".format(error.getErrorMessage()))
# The external field is set for updates from pdserver but not for
# locally-initiated (sideloaded) updates.
update_id = None
if hasattr(self, 'external'):
update_id = self.external['update_id']
request = PDServerRequest(
'/api/routers/{}/updates/{}/messages'.format(
nexus.core.info.pdid, update_id))
d = request.post(**data)
d.addErrback(handleError)
session = getattr(nexus.core, 'session', None)
if session is not None:
data['update_id'] = update_id
# Catch the occasional Exception due to connectivity failure. We
# don't want to fail a chute installation just because we had problems
# sending the log messages.
try:
session.publish(session.uriPrefix + 'updateProgress', data)
except Exception as error:
out.warn("Publish failed: {} {}".format(
error.__class__, error))
# Send messages to internal consumers (e.g. open websocket connections)
self.messages.append(data)
for observer in self.message_observers:
observer.on_message(data)
def remove_container(update, service):
"""
Remove a service's container.
"""
container_name = service.get_container_name()
out.info("Removing container {}\n".format(container_name))
try:
client = docker.DockerClient(base_url="unix://var/run/docker.sock",
version='auto')
# Grab the last 40 log messages to help with debugging.
container = client.containers.get(container_name)
logs = container.logs(stream=False, tail=40, timestamps=False)
update.progress("{}: {}".format(container_name, logs.rstrip()))
container.remove(force=True)
except Exception as error:
out.warn("Error removing container: {}".format(error))
def getSystemDevices(update):
"""
Detect devices on the system.
Store device information in cache key "networkDevices" as well as
"networkDevicesByName".
"""
devices = detectSystemDevices()
devicesByName = dict()
for dtype, dlist in six.iteritems(devices):
for dev in dlist:
name = dev['name']
if name in devicesByName:
out.warn("Multiple network devices named {}".format(name))
devicesByName[name] = dev
update.cache_set('networkDevices', devices)
update.cache_set('networkDevicesByName', devicesByName)
def restore(self, backupToken, saveBackup=True):
"""
Replaces real file (at /etc/config/*) with backup copy from /tmp/*-@backupToken location.
Arguments:
backupToken: The backup token appended at the end of the backup path
saveBackup : A flag to keep a backup copy or delete it (default is keep backup)
"""
# Make sure it exists!
backupPath = "{}/{}-{}".format(settings.UCI_BACKUP_DIR, self.myname,
backupToken)
if(pdos.exists(backupPath)):
if(saveBackup):
pdos.copy(backupPath, self.filepath)
else:
pdos.move(backupPath, self.filepath)
else:
# This might be ok if they didn't actually make any changes
out.warn('Cannot restore, %s missing backup (might be OK if no changes made)\n' % (self.myname))
def getBridgeGateway():
"""
Look up the gateway IP address for the docker bridge network.
This is the docker0 IP address; it is the IP address of the host from the
chute's perspective.
"""
client = docker.DockerClient(base_url="unix://var/run/docker.sock",
version='auto')
network = client.networks.get("bridge")
for config in network.attrs['IPAM']['Config']:
if 'Gateway' in config:
return config['Gateway']
# Fall back to a default if we could not find it. This address will work
# in most places unless Docker changes to use a different address.
out.warn('Could not find bridge gateway, using default')
return '172.17.0.1'
def remove_container(update, service):
"""
Remove a service's container.
"""
container_name = service.get_container_name()
out.info("Removing container {}\n".format(container_name))
try:
client = docker.DockerClient(base_url="unix://var/run/docker.sock",
version='auto')
# Grab the last 40 log messages to help with debugging.
container = client.containers.get(container_name)
logs = container.logs(stream=False, tail=40, timestamps=False)
update.progress("{}: {}".format(container_name, logs.rstrip()))
container.remove(force=True)
except Exception as error:
out.warn("Error removing container: {}".format(error))
def getSystemDevices(update):
"""
Detect devices on the system.
Store device information in cache key "networkDevices" as well as
"networkDevicesByName".
"""
devices = detectSystemDevices()
devicesByName = dict()
for dtype, dlist in devices.iteritems():
for dev in dlist:
name = dev['name']
if name in devicesByName:
out.warn("Multiple network devices named {}".format(name))
devicesByName[name] = dev
update.cache_set('networkDevices', devices)
update.cache_set('networkDevicesByName', devicesByName)
def checkSystemDevices(update):
"""
Check whether expected devices are present.
This may reboot the machine if devices are missing and the host config is
set to do that.
"""
devices = update.new.getCache('networkDevices')
hostConfig = update.new.getCache('hostConfig')
if len(devices['wifi']) == 0:
# No WiFi devices - check what we should do.
action = datastruct.getValue(hostConfig, "system.onMissingWiFi")
if action == "reboot":
out.warn("No WiFi devices were detected, system will be rebooted.")
cmd = ["shutdown", "-r", "now"]
subprocess.call(cmd)
elif action == "warn":
out.warn("No WiFi devices were detected.")
def _maybeWait(self, result):
"""
Wait for completed result if configured to wait on async calls.
"""
if result['type'] == 'sync' or not self.wait_async:
return result['result']
if result['type'] == 'error':
result = result['result']
if self.logging:
out.warn("snapd error: {}".format(result['message']))
return result
while True:
change = self.get_change(result['change'])
if change['ready']:
if self.logging:
out.info("{}: {}".format(change['summary'], change['status']))
return change
time.sleep(1)
def removeNewContainer(update):
"""
Remove the newly started container during abort sequence.
"""
name = update.new.name
out.info("Removing container {}\n".format(name))
cleanup_net_interfaces(update.new)
try:
client = docker.DockerClient(base_url="unix://var/run/docker.sock",
version='auto')
# Grab the last 40 log messages to help with debugging.
container = client.containers.get(name)
logs = container.logs(stream=False, tail=40, timestamps=False)
update.progress("{}: {}".format(name, logs.rstrip()))
container.remove(force=True)
except Exception as error:
out.warn("Error removing container: {}".format(error))
def onStart(self):
super(Nexus, self).onStart()
# onStart is called when the reactor starts, not when the connection is made.
# Check for provisioning keys and attempt to connect
if not self.provisioned():
out.warn(
'Router has no keys or identity. Waiting to connect to to server.'
)
else:
try:
# Set up communication with pdserver.
# 1. Create a report of the current system state and send that.
# 2. Poll for a list of updates that should be applied.
# 3. Open WAMP session.
yield sendStateReport()
yield self.update_fetcher.start_polling()
yield self.connect(WampSession)
except Exception:
out.warn('The router ID or password is invalid!')
else:
out.info('WAMP session is ready!')
def _maybeWait(self, result):
"""
Wait for completed result if configured to wait on async calls.
"""
if result['type'] == 'sync' or not self.wait_async:
return result['result']
if result['type'] == 'error':
result = result['result']
if self.logging:
out.warn("snapd error: {}".format(result['message']))
return result
while True:
change = self.get_change(result['change'])
if change['ready']:
if self.logging:
out.info("{}: {}".format(change['summary'],
change['status']))
return change
time.sleep(1)
def getNetworkConfigWifi(update, name, cfg, iface):
dtype, mode = split_interface_type(cfg['type'])
# Later code still depends on the mode field.
iface['mode'] = mode
# Generate a name for the new interface in the host.
iface['externalIntf'] = chooseExternalIntf(update, iface)
if len(iface['externalIntf']) > constants.MAX_INTERFACE_NAME_LEN:
out.warn("Interface name ({}) is too long\n".format(
iface['externalIntf']))
raise Exception("Interface name is too long")
wireless = cfg.get("wireless", {})
iface['wireless'] = wireless
if iface['type'] in ["wifi-ap", "wifi-sta"]:
# Check for required fields.
res = pdutils.check(wireless, dict, ['ssid'])
if res:
out.warn('WiFi network interface definition {}\n'.format(res))
raise Exception("Interface definition missing field(s)")
iface['ssid'] = wireless['ssid']
# Optional encryption settings
getWifiKeySettings(wireless, iface)
# Give a warning if the dhcp block is missing, since it is likely
# that developers will want a DHCP server to go with their AP.
if 'dhcp' not in cfg:
out.warn("No dhcp block found for interface {}; "
"will not run a DHCP server".format(name))
def getNetworkConfigWifi(update, name, cfg, iface):
dtype, mode = split_interface_type(cfg['type'])
# Later code still depends on the mode field.
iface['mode'] = mode
# Generate a name for the new interface in the host.
iface['externalIntf'] = chooseExternalIntf(update, iface)
if len(iface['externalIntf']) > MAX_INTERFACE_NAME_LEN:
out.warn("Interface name ({}) is too long\n".format(
iface['externalIntf']))
raise Exception("Interface name is too long")
wireless = cfg.get("wireless", {})
iface['wireless'] = wireless
if iface['type'] in ["wifi-ap", "wifi-sta"]:
# Check for required fields.
res = pdutils.check(wireless, dict, ['ssid'])
if res:
out.warn('WiFi network interface definition {}\n'.format(res))
raise Exception("Interface definition missing field(s)")
iface['ssid'] = wireless['ssid']
# Optional encryption settings
getWifiKeySettings(wireless, iface)
# Give a warning if the dhcp block is missing, since it is likely
# that developers will want a DHCP server to go with their AP.
if 'dhcp' not in cfg:
out.warn("No dhcp block found for interface {}; "
"will not run a DHCP server".format(name))
def sshKeys(self, request, user):
"""
Manage list of authorized keys for SSH access.
"""
cors.config_cors(request)
request.setHeader('Content-Type', 'application/json')
if request.method == "GET":
try:
if GovernorClient.isAvailable():
client = GovernorClient()
keys = client.listAuthorizedKeys(user)
# Move keys to the root because the API consumer is
# expecting a list, not an object.
if 'keys' in keys:
keys = keys['keys']
else:
keys = ssh_keys.getAuthorizedKeys(user)
return json.dumps(keys)
except Exception as e:
out.warn(str(e))
request.setResponseCode(404)
return json.dumps({'message': str(e)})
else:
body = json.loads(request.content.read().decode('utf-8'))
key = body['key'].strip()
try:
if GovernorClient.isAvailable():
client = GovernorClient()
client.addAuthorizedKey(key, user)
else:
ssh_keys.addAuthorizedKey(key, user)
return json.dumps(body)
except Exception as e:
out.warn(str(e))
request.setResponseCode(404)
return json.dumps({'message': str(e)})
def call_retry(cmd, env, delay=3, tries=3):
# Make sure each component of the command is a string. Otherwisew we will
# get errors.
clean_cmd = [str(v) for v in cmd]
while tries >= 0:
tries -= 1
out.info("Calling: {}\n".format(" ".join(clean_cmd)))
try:
proc = subprocess.Popen(clean_cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, env=env)
for line in proc.stdout:
out.info("{}: {}\n".format(clean_cmd[0], line.strip()))
for line in proc.stderr:
out.warn("{}: {}\n".format(clean_cmd[0], line.strip()))
return proc.returncode
except OSError as e:
out.warn('Command "{}" failed\n'.format(" ".join(clean_cmd)))
if tries <= 0:
out.exception(e, True)
raise e
time.sleep(delay)
def writeDockerConfig():
"""
Write options to Docker configuration.
Mainly, we want to tell Docker not to start containers automatically on
system boot.
"""
# First we have to find the configuration file.
# On ubuntu 16.04 with docker snap, it should be in
# "/var/snap/docker/{version}/etc/docker/", but version could change.
path = "/var/snap/docker/current/etc/docker/docker.conf"
written = False
if os.path.exists(path):
try:
with open(path, "w") as output:
output.write(DOCKER_CONF)
written = True
except Exception as e:
out.warn('Error writing to {}: {}'.format(path, str(e)))
if not written:
out.warn('Could not write docker configuration.')
return written
def reloadChutes():
"""
Get update objects to chutes that should be running at startup.
This function is called to restart any chutes that were running
prior to the system being restarted. It waits for pdconfd to
come up and report whether or not it failed to bring up any of the
interfaces that existed before the power cycle. If pdconfd indicates
something failed we then force a stop update in order to bring down
all interfaces associated with that chute and mark it with a warning.
If the stop fails we mark the chute with a warning manually and change
its state to stopped and save to storage this isn't great as it could
mean our system still has interfaces up associated with that chute.
If pdconfd doesn't report failure we attempt to start the chute and
if this fails we trust the abort process to restore the system to a
consistent state and we manually mark the chute as stopped and add
a warning to it.
:returns: (list) A list of UpdateChute objects that should be run
before accepting new updates.
"""
if not settings.PDCONFD_ENABLED:
return []
chuteStore = ChuteStorage()
chutes = [ ch for ch in chuteStore.getChuteList() if ch.isRunning() ]
# Part of restoring the chute to its previously running state is reclaiming
# IP addresses, interface names, etc. that it had previously.
for chute in chutes:
reclaimNetworkResources(chute)
#We need to make sure confd is up and all interfaces have been brought up properly
confdup = False
while not confdup:
confdInfo = waitSystemUp()
if confdInfo == None:
time.sleep(1)
continue
confdup = True
confdInfo = json.loads(confdInfo)
#Remove any chutes from the restart queue if confd failed to bring up the
#proper interfaces
#
# At this point, we only care about the chute names, not the full objects.
# We are using sets of chute names for their O(1) membership test and
# element uniqueness.
okChutes = set([ch.name for ch in chutes])
failedChutes = set()
for iface in confdInfo:
if iface.get('success') is False:
failedChuteName = iface.get('comment')
if failedChuteName == constants.RESERVED_CHUTE_NAME:
out.warn('Failed to load a system config section')
elif failedChuteName in okChutes:
# This was a chute that we are supposed to restart, but one of
# its config sections failed to load.
okChutes.remove(failedChuteName)
failedChutes.add(failedChuteName)
elif failedChuteName not in failedChutes:
# In this case, the name in the comment was not something that
# we recognize from the chute storage. Maybe the chute storage
# file was deleted but not the config files, or someone
# manually modified the config files. Anyway, we cannot
# attempt to stop this chute because the object does not exist,
# but we can give a warning message.
out.warn('Failed to load config section for '
'unrecognized chute: {}'.format(failedChuteName))
updates = []
# There was code here that looped over the failedChutes set and explicitly
# stopped those chutes. However, if the cause of the failure was
# transient, those chutes would remain stopped until manually restarted.
# It seems better to leave them alone so that we try again next time the
# system reboots.
#
# TODO: We should still record the failure somewhere.
# Only try to restart the chutes that had no problems during pdconf
# initialization.
for ch in okChutes:
update_spec = dict(updateClass='CHUTE', updateType='restart', name=ch,
tok=timeint(), func=updateStatus,
user=User.get_internal_user())
update = UpdateChute(update_spec, reuse_existing=True)
updates.append(update)
return updates
def readConfig(self, files):
"""
Load configuration files and return configuration objects.
This method only loads the configuration files without making any
changes to the system and returns configuration objects as a generator.
"""
# Keep track of headers (section type and name) that have been
# processed so far. The dictionary maps them to filename, so that we
# can print a useful warning message about duplicates.
usedHeaders = dict()
for fn in files:
out.info("Reading file {}\n".format(fn))
# Extract just the filename (e.g. wireless, network, qos).
basename = os.path.basename(fn)
uci = UCIConfig(fn)
config = uci.readConfig()
for section, options in config:
# Sections differ in where they put the name, if they have one.
if "name" in section:
name = section['name']
elif "name" in options:
name = options['name']
else:
name = None
# Get section comment string (optional, but Paradop uses it).
comment = section.get('comment', None)
# Try looking up by extended type first. It turns out, both
# "network" and "qos" configuration files should support
# sections with typename "interface". We can disambiguate
# them by calling the latter one "qos:interface".
ext_type = "{}:{}".format(basename, section['type'])
if ext_type in configTypeMap:
cls = configTypeMap[ext_type]
elif section['type'] in configTypeMap:
cls = configTypeMap[section['type']]
else:
out.warn("Unsupported section type {} in {}\n".format(
section['type'], fn))
continue
try:
obj = cls.build(self, fn, name, options, comment)
except Exception as e:
out.warn("Error building object from section {}:{} in "
"{}\n".format(section['type'], name, fn))
out.warn(e)
continue
key = obj.getTypeAndName()
if key in usedHeaders:
out.warn("Section {}:{} from {} overrides section in "
"{}\n".format(section['type'], name, fn,
usedHeaders[key]))
usedHeaders[key] = fn
yield obj
def getVirtDHCPSettings(update):
"""
Looks at the runtime rules the developer defined to see if they want a dhcp
server. If so it generates the data and stores it into the chute cache
key:virtDHCPSettings.
"""
interfaces = update.cache_get('networkInterfaces')
if interfaces is None:
return
dhcpSettings = list()
for iface in interfaces:
# Only look at interfaces with DHCP server requested.
if 'dhcp' not in iface:
continue
dhcp = iface['dhcp']
# Construct a path for the lease file that will be visible inside the
# chute.
leasefile = os.path.join(
update.cache_get('externalSystemDir'),
"dnsmasq-{}.leases".format(iface['name'])
)
# NOTE: Having one dnsmasq section for each interface deviates from how
# OpenWRT does things, where they assume a single instance of dnsmasq
# will be handling all DHCP and DNS needs.
config = {'type': 'dnsmasq'}
options = {
'leasefile': leasefile,
'interface': [iface['externalIntf']]
}
# Optional: developer can pass in a list of DNS nameservers to use
# instead of the system default.
#
# This path -> clients query our dnsmasq server; dnsmasq uses the
# specified nameservers and caches the results.
if DNSMASQ_CACHE_ENABLED and 'dns' in dhcp:
options['noresolv'] = '1'
options['server'] = dhcp['dns']
# Disable authoritative mode if serving as a relay.
relay = dhcp.get('relay', None)
if relay is not None:
options['authoritative'] = False
dhcpSettings.append((config, options))
# Check for fields that are required if not in relay mode.
res = pdutils.check(dhcp, dict, ['lease', 'start', 'limit'])
if relay is None and res:
out.warn('DHCP server definition {}\n'.format(res))
raise Exception("DHCP server definition missing field(s)")
config = {'type': 'dhcp', 'name': iface['externalIntf']}
options = {
'interface': iface['externalIntf'],
'start': dhcp.get('start', None),
'limit': dhcp.get('limit', None),
'leasetime': dhcp.get('lease', None),
'dhcp_option': []
}
# This option tells clients that the router is the interface inside the
# chute not the one in the host.
options['dhcp_option'].append("option:router,{}".format(iface['internalIpaddr']))
# Optional: developer can pass in a list of DNS nameservers to use
# instead of the system default.
#
# This path -> clients receive the list of DNS servers and query them
# directly.
if not DNSMASQ_CACHE_ENABLED and 'dns' in dhcp:
options['dhcp_option'].append(",".join(["option:dns-server"] + dhcp['dns']))
# Interpret the optional DHCP relay configuration.
# - string: interpret as the IP address of the relay server.
# - list: interpret as list of strings to pass to dnsmasq (--dhcp-relay options).
if isinstance(relay, six.string_types):
# TODO: Set the optional third argument (interface) to the name of
# the network interface on which we expect DHCP response. This
# could be the WAN interface or it could be VPN interface.
options['relay'] = ["{},{}".format(iface['externalIpaddr'], relay)]
elif isinstance(relay, list):
options['relay'] = relay
dhcpSettings.append((config, options))
update.cache_set('virtDHCPSettings', dhcpSettings)
def getNetworkConfig(update):
"""
For the Chute provided, return the dict object of a 100% filled out
configuration set of network configuration. This would include determining
what the IP addresses, interfaces names, etc...
Store configuration in networkInterfaces cache entry.
"""
# Notes:
#
# Fill in the gaps of knowledge between what the dev provided us in their
# config and what would be required to add all the config changes to get
# the chute working By the end of this function there should be a
# cache:networkInterfaces key containing a list of dictionary objects for
# each interface we will need to create, including netmasks IP addresses,
# etc.. this is important to allow all other modules to know what the IP
# addr is for this chute, etc..
#
# old code under lib.internal.chs.chutelxc same function name
interfaces = list()
# Put the list in the cache now (shared reference), so that if we fail out
# of this function after partial completion, the abort function can take
# care of what made it into the list.
update.cache_set('networkInterfaces', interfaces)
# Deprecated: we set this value in the chute cache here because there is
# still code that depends on reading this from the chute storage. This can
# be removed when the corresponding call to getCache is removed.
update.new.setCache('networkInterfaces', interfaces)
# Make sure we only assign interfaces to running chutes.
if not update.new.isRunning():
return None
devices = update.cache_get('networkDevices')
for service in update.new.get_services():
for name, cfg in six.iteritems(service.interfaces):
# Check for required fields.
res = pdutils.check(cfg, dict, ['type'])
if res:
out.warn('Network interface definition {}\n'.format(res))
raise Exception("Interface definition missing field(s)")
iface = {
'name': name, # Name (not used?)
'service': service.name, # Service name
'type': cfg['type'], # Type (wan, lan, wifi)
'internalIntf': cfg['intfName'], # Interface name in chute
'l3bridge': cfg.get('l3bridge', None) # Optional
}
getInterfaceAddress(update, name, cfg, iface)
if cfg['type'].startswith("wifi"):
# Try to find a physical device of the requested type.
#
# Note: we try this first because it can fail, and then we will not try
# to allocate any resources for it.
device = fulfillDeviceRequest(update, cfg, devices)
iface['device'] = device['name']
iface['phy'] = device['phy']
getNetworkConfigWifi(update, name, cfg, iface)
elif cfg['type'] == "vlan":
getNetworkConfigVlan(update, name, cfg, iface)
elif cfg['type'] == "lan":
device = fulfillDeviceRequest(update, cfg, devices)
iface['device'] = device['name']
getNetworkConfigLan(update, name, cfg, iface)
else:
raise Exception("Unsupported network type, {}".format(
cfg['type']))
# Pass on DHCP configuration if it exists.
if 'dhcp' in cfg:
iface['dhcp'] = cfg['dhcp']
# TODO: Refactor! The problem here is that `cfg` contains a mixture of
# fields, some that we will interpret and some that we will pass on to
# pdconf. The result is a lot of logic that tests and copies without
# actually accomplishing much.
iface['options'] = getExtraOptions(cfg)
interfaces.append(iface)
update.cache_set('networkInterfaces', interfaces)
# Deprecated: we set this value in the chute cache here because there is
# still code that depends on reading this from the chute storage. This can
# be removed when the corresponding call to getCache is removed.
update.new.setCache('networkInterfaces', interfaces)
def getDeveloperFirewallRules(update):
"""
Generate other firewall rules requested by the developer such as redirects.
The object returned is a list of tuples (config, options).
"""
interfaces = update.cache_get('networkInterfaces')
if interfaces is None:
return None
rules = list()
if hasattr(update.new, "firewall"):
for rule in update.new.firewall:
if rule['type'] == 'redirect':
config = {'type': 'redirect'}
options = {
'name': rule['name'],
'proto': 'tcpudp'
}
from_parts = rule['from'].strip().split(':')
to_parts = rule['to'].strip().split(':')
# Do not allow rules that do not pertain to the chute.
if "@host.lan" in from_parts[0] and "@host.lan" in to_parts[0]:
raise Exception("Unable to add firewall rule - "
"dst and src are both outside of chute")
# From @host.lan means this is a DNAT rule (redirect to the chute).
if from_parts[0] == "@host.lan":
options['target'] = "DNAT"
options['src'] = "wan"
if len(from_parts) > 1:
options['src_dport'] = from_parts[1]
# Find the interface specified in the rule, so we can get its
# IP address.
iface = findMatchingInterface(to_parts[0], interfaces)
if iface is None:
out.warn("No interface found with name {}\n".format(to_parts[0]))
raise Exception("Interface not found")
options['dest_ip'] = iface['externalIpaddr']
if len(to_parts) > 1:
options['dest_port'] = to_parts[1]
# This is an SNAT rule (redirect from the chute to host network).
elif to_parts[0] == "@host.lan":
options['target'] = "SNAT"
# TODO: Implement
out.warn("SNAT rules not supported yet")
raise Exception("SNAT rules not implemented")
# Could be forwarding between chute interfaces?
else:
out.warn("Other rules not supported yet")
raise Exception("Other rules not implemented")
rules.append((config, options))
update.cache_set('developerFirewallRules', rules)
def executePlans(update):
"""
Primary function that actually executes all the functions that were added to plans by all
the exc modules. This function can heavily modify the OS/files/etc.. so the return value is
very important.
Returns:
True in error : abortPlans function should be called
False otherwise : everything is OK
"""
out.header('Executing plans %r\n' % (update))
# Finding the functions to call is actually done by a 'iterator' like function in the plangraph module
while(True):
# This function either returns None or a tuple just like generate added to it
p = update.plans.getNextTodo()
# No more to do?
if(not p):
break
# Explode tuple otherwise
func, args = p
# We are in a try-except block so if func isn't callable that will catch it
try:
out.verbose('Calling %s\n' % (func))
update.progress("Calling {}".format(func.__name__))
#
# Call the function from the execution plan
#
# args may be empty, but we don't want to pass in a tuple if we don't need to.
# This below explodes the args so if @args is (), then what is passed is @update
skipme = func(*((update, ) + args))
except Exception as e:
out.exception(e, True)
# plans = str(update.plans)) # Removed because breaks new out.exception call
out.warn("Failed to execute plan %s%s" % (func.__name__, args))
update.responses.append({'exception': str(e), 'traceback': traceback.format_exc()})
update.failure = str(e)
return True
# The functions we call here can return other functions, if they do
# these are functions that should be skipped later on (for instance a
# set* function discovering it didn't change anything, later on we
# shouldn't call the corresponding reload function)
if skipme is not None:
# If the function returned a Deferred, we will drop out of the
# execution pipeline and resume later.
if isinstance(skipme, Deferred):
out.verbose('Function {} returned a Deferred'.format(func))
return skipme
# These functions can return individual functions to skip, or a
# list of multiple functions
elif callable(skipme):
skipme = [skipme]
for skip in skipme:
out.warn('Identified a skipped function: %r\n' % (skip))
update.plans.registerSkip(skip)
# Now we are done
return False
