def __init__(self, autoSave=True, router=False):
'''
Autosave saves baseConfig on change. (not implemented, see methods below)
The router optional flag should be set to configure runtime store allocation specific
for VM routers
'''
self.autosave = autoSave
# Check and see if we are running locally-- if so place the logs in a subdirectory
# and respecify the paths
if router:
configureLocalPaths()
# Create needed directories
for path in [STORE_PATH, KEY_PATH, LOG_PATH, MISC_PATH]:
if not os.path.exists(path):
os.makedirs(path)
# No config file found. Create a default one
if not os.path.isfile(INFO_PATH):
createDefaultInfo(INFO_PATH)
self.baseConfig = loadYaml(INFO_PATH)
# Sanity check contents of info and throw it out if bad
if not validateInfo(self.baseConfig):
out.warn('Saved configuration data invalid, destroying it.')
os.remove(INFO_PATH)
createDefaultInfo(INFO_PATH)
self.baseConfig = loadYaml(INFO_PATH)
def getVirtPreamble(update):
out.warn('TODO implement me\n')
if update.updateType == 'create':
if not hasattr(update, 'dockerfile'):
return
if update.dockerfile is None:
return
else:
out.info('Using prexisting dockerfile.\n')
update.dockerfile = BytesIO(update.dockerfile.encode('utf-8'))
def getPid(self):
if self.fromFile:
try:
with open(self.pid, "r") as inputFile:
return int(inputFile.read().strip())
except:
# No pid file --- maybe it was not running?
out.warn("File not found: {}\n".format(self.pid))
return None
else:
return self.pid
def failprocess(self, ip, request, logFailure, errorStmt, logUsage, errType):
"""
If logFailure is not None, Update the failureDict when the request does something wrong
If logUsage is not None, log the usage track info.
Arguments:
ip : IP of client
request : the request we received
logFailure : If none, we do not log this failure to failure dict. Otherwise it is a tuple of
key : the key to use in the failureDict
failureDict : the dict record the failure attempt history
errorStmt : A string to return to the user, looks like 'Malformed Body: %s'
so that we can add things like "Number of attempts remaining: 2" to the response
logUsage : A tuple of (tictoc and devid) used for usage tracker
errorResponse: The error code to set response code
Returns:
String to respond to the client
"""
time = timeflt()
# Set the response error code
if(errType == pdapi.ERR_BADPARAM):
request.setResponseCode(*pdapi.getResponse(errType, ""))
else:
request.setResponseCode(*pdapi.getResponse(errType))
headers = request.received_headers
if(logUsage is not None):
tictoc, devid = logUsage
if(devid is None):
devid = "Null"
duration = 0 # self.perf.toc(tictoc)
# Log the info of this call
# TODO self.usageTracker.addTrackInfo(ip, devid , request.path, self.usageTracker.FAIL_AUTH, duration, request.content.getvalue())
if(logFailure is not None):
key, failureDict = logFailure
# update the accessInfo
if(key in failureDict):
failureDict[key].update(ip, headers, time)
else:
failureDict[key] = AccessInfo(ip, headers, time)
attempts = failureDict[key].attempts
out.warn('Failure access recorded: %s Attempts: %d\n' % (key, attempts))
remaining = str(max(0, settings.DBAPI_FAILURE_THRESH - attempts))
if(errorStmt):
return errorStmt % ("%s attempts remaining" % remaining)
if(errorStmt):
return errorStmt % ("Null")
def executePlans(update):
"""
Primary function that actually executes all the functions that were added to plans by all
the exc modules. This function can heavily modify the OS/files/etc.. so the return value is
very important.
Returns:
True in error : abortPlans function should be called
False otherwise : everything is OK
"""
out.header('Executing plans %r\n' % (update))
# Finding the functions to call is actually done by a 'iterator' like function in the plangraph module
while(True):
# This function either returns None or a tuple just like generate added to it
p = update.plans.getNextTodo()
# No more to do?
if(not p):
break
# Explode tuple otherwise
func, args = p
# We are in a try-except block so if func isn't callable that will catch it
try:
out.verbose('Calling %s\n' % (func))
#
# Call the function from the execution plan
#
# args may be empty, but we don't want to pass in a tuple if we don't need to
# so this below explodes the args so if @args is (), then what is passed is @update
skipme = func(*((update, ) + args))
except Exception as e:
out.exception(e, True) #, plans=str(update.plans)) # Removed because breaks new out.exception call
update.responses.append({'exception': str(e), 'traceback': traceback.format_exc()})
update.failure = str(e)
return True
# The functions we call here can return other functions, if they do these are functions that should
# be skipped later on (for instance a set* function discovering it didn't change anything, later on
# we shouldn't call the corresponding reload function)
if(skipme):
# These functions can return individual functions to skip, or a list of multiple functions
if (not isinstance(skipme, list)):
skipme = [skipme]
for skip in skipme:
out.warn('Identified a skipped function: %r\n' % (skip))
update.plans.registerSkip(skip)
# Now we are done
return False
def appendCache(self, key, val):
"""
Finds the key they requested and appends the val into it, this function assumes the cache object
is of list type, if the key hasn't been defined yet then it will set it to an empty list.
"""
r = self.getCache(key)
if(not r):
r = []
elif(not isinstance(r, list)):
out.warn('Unable to append to cache, not list type\n' )
return
r.append(val)
self.setCache(key, r)
return True
def execute(self):
"""
The function that actually walks through the main process required to create the chute.
It follows the executeplan module through the paces of:
1) Generate the plans for each exc module
2) Prioritize the plans
3) Execute the plans
If at any point we fail then this function will directly take care of completing
the update process with an error state and will close the API connection.
"""
# Save a timestamp from when we started execution.
self.startTime = time.time()
# Generate the plans we need to setup the chute
if(exc.executionplan.generatePlans(self)):
out.warn('Failed to generate plans\n')
self.complete(success=False, message=self.failure)
return
# Aggregate those plans
exc.executionplan.aggregatePlans(self)
# Execute on those plans
if(exc.executionplan.executePlans(self)):
# Getting here means we need to abort what we did
res = exc.executionplan.abortPlans(self)
# Did aborting also fail? This is bad!
if(res):
###################################################################################
# Getting here means the abort system thinks it wasn't able to get the system
# back into the state it was in prior to this update.
###################################################################################
out.err('TODO: What do we do when we fail during abort?\n')
pass
# Report the failure back to the user
self.complete(success=False, message=self.failure)
return
# Now save the new state if we are all ok
self.saveState()
# Respond to the API server to let them know the result
self.complete(success=True, message='Chute {} {} success'.format(
self.name, self.updateType))
def getWifiKeySettings(cfg, iface):
"""
Read encryption settings from cfg and transfer them to iface.
"""
# If 'key' is present, but 'encryption' is not, then default to
# psk2. If 'key' is not present, and 'encryption' is not none,
# then we have an error.
iface['encryption'] = "none"
if 'key' in cfg:
iface['key'] = cfg['key']
iface['encryption'] = 'psk2' # default to psk2
if 'encryption' in cfg:
iface['encryption'] = cfg['encryption']
if cfg['encryption'] != "none" and 'key' not in cfg:
out.warn("Key field must be defined "
"when encryption is enabled.")
raise Exception("No key field defined for WiFi encryption")
def restore(self, backupToken, saveBackup=True):
"""
Replaces real file (at /etc/config/*) with backup copy from /tmp/*-@backupToken location.
Arguments:
backupToken: The backup token appended at the end of the backup path
saveBackup : A flag to keep a backup copy or delete it (default is keep backup)
"""
# Make sure it exists!
backupPath = '/tmp/%s-%s' % (self.myname, backupToken)
if(pdos.exists(backupPath)):
if(saveBackup):
pdos.copy(backupPath, self.filepath)
else:
pdos.move(backupPath, self.filepath)
else:
# This might be ok if they didn't actually make any changes
out.warn('Cannot restore, %s missing backup (might be OK if no changes made)\n' % (self.myname))
def makeHostapdConf(self, wifiDevice, interface):
outputPath = "{}/hostapd-{}.conf".format(
self.manager.writeDir, self.internalName)
with open(outputPath, "w") as outputFile:
# Write our informative header block.
outputFile.write("#" * 80 + "\n")
outputFile.write("# hostapd configuration file generated by "
"pdconfd\n")
outputFile.write("# Source: {}\n".format(self.source))
outputFile.write("# Section: {}\n".format(str(self)))
outputFile.write("# Device: {}\n".format(str(wifiDevice)))
outputFile.write("# Interface: {}\n".format(str(interface)))
outputFile.write("#" * 80 + "\n")
# 802.11ac not working, need to find out why
outputFile.write("hw_mode=g\n")
#outputFile.write("ieee80211ac=1\n")
# Write essential options.
outputFile.write("interface={}\n".format(self._ifname))
outputFile.write("ssid={}\n".format(self.ssid))
outputFile.write("channel={}\n".format(wifiDevice.channel))
if interface.type == "bridge":
outputFile.write("bridge={}\n".format(interface.config_ifname))
# Optional encryption options.
if self.encryption is None or self.encryption == "none":
pass
elif self.encryption == "psk2":
outputFile.write("wpa=1\n")
# If key is a 64 character hex string, then treat it as the PSK
# directly, else treat it as a passphrase.
if len(self.key) == 64 and isHexString(self.key):
outputFile.write("wpa_psk={}\n".format(self.key))
else:
outputFile.write("wpa_passphrase={}\n".format(self.key))
else:
out.warn("Encryption type {} not supported (supported: "
"none|psk2)".format(self.encryption))
raise Exception("Encryption type not supported")
return outputPath
def writeDockerConfig():
"""
Write options to Docker configuration.
Mainly, we want to tell Docker not to start containers automatically on
system boot.
"""
# First we have to find the configuration file. On Snappy, it should be in
# "/var/lib/apps/docker/{version}/etc/docker.conf", but version could
# change.
path = "/var/lib/apps/docker"
if not os.path.exists(path):
out.warn('No directory "{}" found'.format(path))
return False
written = False
for d in os.listdir(path):
finalPath = os.path.join(path, d, "etc/docker.conf")
if not os.path.exists(finalPath):
continue
try:
with open(finalPath, "w") as output:
output.write(DOCKER_CONF)
written = True
except Exception as e:
out.warn('Error writing to {}: {}'.format(finalPath, str(e)))
if not written:
out.warn('Could not write docker configuration.')
return written
def setup_net_interfaces(update):
"""
Link interfaces in the host to the internal interface in the docker container using pipework.
:param update: The update object containing information about the chute.
:type update: obj
:returns: None
"""
interfaces = update.new.getCache('networkInterfaces')
for iface in interfaces:
if iface.get('netType') == 'wifi':
IP = iface.get('ipaddrWithPrefix')
internalIntf = iface.get('internalIntf')
externalIntf = iface.get('externalIntf')
else: # pragma: no cover
continue
# Construct environment for pipework call. It only seems to require
# the PATH variable to include the directory containing the docker
# client. On Snappy this was not happening by default, which is why
# this code is here.
env = {"PATH": os.environ.get("PATH", "")}
if settings.DOCKER_BIN_DIR not in env['PATH']:
env['PATH'] += ":" + settings.DOCKER_BIN_DIR
cmd = ['/apps/paradrop/current/bin/pipework', externalIntf, '-i',
internalIntf, update.name, IP]
out.info("Calling: {}\n".format(" ".join(cmd)))
try:
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, env=env)
for line in proc.stdout:
out.info("pipework: {}\n".format(line.strip()))
for line in proc.stderr:
out.warn("pipework: {}\n".format(line.strip()))
except OSError as e:
out.warn('Command "{}" failed\n'.format(" ".join(cmd)))
out.exception(e, True)
raise e
def getNetworkConfigWifi(update, name, cfg, iface):
# Claim a subnet for this interface from the pool.
subnet = networkPool.next()
hosts = subnet.hosts()
# Generate internal (in the chute) and external (in the host)
# addresses.
#
# Example:
# subnet: 192.168.30.0/24
# netmask: 255.255.255.0
# external: 192.168.30.1
# internal: 192.168.30.2
iface['subnet'] = subnet
iface['netmask'] = str(subnet.netmask)
iface['externalIpaddr'] = str(hosts.next())
iface['internalIpaddr'] = str(hosts.next())
# Generate the internal IP address with prefix length (x.x.x.x/y) for
# convenience of other code that expect that format (e.g. pipework).
iface['ipaddrWithPrefix'] = "{}/{}".format(
iface['internalIpaddr'], subnet.prefixlen)
# Generate initial portion (prefix) of interface name.
#
# NOTE: We add a "v" in front of the interface name to avoid triggering
# the udev persistent net naming rules, which are hard-coded to certain
# typical strings such as "eth*" and "wlan*" but not "veth*" or
# "vwlan*". We do NOT want udev renaming our virtual interfaces.
iface['extIntfPrefix'] = "v" + iface['device'] + "."
iface['extIntfNumber'] = interfaceNumberPool.next()
# Generate a name for the new interface in the host.
iface['externalIntf'] = "{}{:04x}".format(
iface['extIntfPrefix'], iface['extIntfNumber'])
if len(iface['externalIntf']) > MAX_INTERFACE_NAME_LEN:
out.warn("Interface name ({}) is too long\n".
format(iface['externalIntf']))
raise Exception("Interface name is too long")
# Add extra fields for WiFi devices.
if cfg['type'] == "wifi":
# Check for required fields.
res = pdutils.check(cfg, dict, ['ssid'])
if res:
out.warn('WiFi network interface definition {}\n'.format(res))
raise Exception("Interface definition missing field(s)")
iface['ssid'] = cfg['ssid']
# Optional encryption settings
getWifiKeySettings(cfg, iface)
# Give a warning if the dhcp block is missing, since it is likely
# that developers will want a DHCP server to go with their AP.
if 'dhcp' not in cfg:
out.warn("No dhcp block found for interface {}; "
"will not run a DHCP server".format(name))
def readConfig(self, files):
"""
Load configuration files and return configuration objects.
This method only loads the configuration files without making any
changes to the system and returns configuration objects as a generator.
"""
# Keep track of headers (section type and name) that have been
# processed so far. The dictionary maps them to filename, so that we
# can print a useful warning message about duplicates.
usedHeaders = dict()
for fn in files:
out.info("Reading file {}\n".format(fn))
uci = UCIConfig(fn)
config = uci.readConfig()
for section, options in config:
# Sections differ in where they put the name, if they have one.
if "name" in section:
name = section['name']
elif "name" in options:
name = options['name']
else:
name = None
# Get section comment string (optional, but Paradop uses it).
comment = section.get('comment', None)
try:
cls = configTypeMap[section['type']]
except:
out.warn("Unsupported section type {} in {}\n".format(
section['type'], fn))
continue
try:
obj = cls.build(self, fn, name, options, comment)
except:
out.warn("Error building object from section {}:{} in "
"{}\n".format(section['type'], name, fn))
continue
key = obj.getTypeAndName()
if key in usedHeaders:
out.warn("Section {}:{} from {} overrides section in "
"{}\n".format(section['type'], name, fn,
usedHeaders[key]))
usedHeaders[key] = fn
yield obj
def getVirtDHCPSettings(update):
"""
Looks at the runtime rules the developer defined to see if they want a dhcp
server. If so it generates the data and stores it into the chute cache
key:virtDHCPSettings.
"""
interfaces = update.new.getCache('networkInterfaces')
if interfaces is None:
return
dhcpSettings = list()
for iface in interfaces:
# Only look at interfaces with DHCP server requested.
if 'dhcp' not in iface:
continue
dhcp = iface['dhcp']
# Check for required fields.
res = pdutils.check(dhcp, dict, ['lease', 'start', 'limit'])
if(res):
out.warn('DHCP server definition {}\n'.format(res))
raise Exception("DHCP server definition missing field(s)")
# NOTE: Having one dnsmasq section for each interface deviates from how
# OpenWRT does things, where they assume a single instance of dnsmasq
# will be handling all DHCP and DNS needs.
config = {'type': 'dnsmasq'}
options = {}
uciutils.setList(options, 'interface', [iface['externalIntf']])
# Optional: developer can pass in a list of DNS nameservers to use
# instead of the system default.
#
# This path -> clients query our dnsmasq server; dnsmasq uses the
# specified nameservers and caches the results.
if DNSMASQ_CACHE_ENABLED and 'dns' in dhcp:
options['noresolv'] = '1'
uciutils.setList(options, 'server', dhcp['dns'])
dhcpSettings.append((config, options))
config = {'type': 'dhcp', 'name': iface['externalIntf']}
options = {
'interface': iface['externalIntf'],
'start': dhcp['start'],
'limit': dhcp['limit'],
'leasetime': dhcp['lease'],
}
# This option tells clients that the router is the interface inside the
# chute not the one in the host.
uciutils.setList(options, 'dhcp_option', ["option:router,{}".format(
iface['internalIpaddr'])])
# Optional: developer can pass in a list of DNS nameservers to use
# instead of the system default.
#
# This path -> clients receive the list of DNS servers and query them
# directly.
if not DNSMASQ_CACHE_ENABLED and 'dns' in dhcp:
uciutils.appendListItem(options, 'dhcp_option',
",".join(["option:dns-server"] + dhcp['dns']))
dhcpSettings.append((config, options))
update.new.setCache('virtDHCPSettings', dhcpSettings)
def printError(self, error):
out.warn('Default riffle error trap: ' + str(error))
def getVirtNetworkConfig(update):
out.warn('TODO implement me\n')
def getNetworkConfig(update):
"""
For the Chute provided, return the dict object of a 100% filled out
configuration set of network configuration. This would include determining
what the IP addresses, interfaces names, etc...
"""
# Notes:
#
# Fill in the gaps of knowledge between what the dev provided us in their
# config and what would be required to add all the config changes to get
# the chute working By the end of this function there should be a
# cache:networkInterfaces key containing a list of dictionary objects for
# each interface we will need to create, including netmasks IP addresses,
# etc.. this is important to allow all other modules to know what the IP
# addr is for this chute, etc..
#
# old code under lib.internal.chs.chutelxc same function name
interfaces = list()
# Put the list in the cache now (shared reference), so that if we fail out
# of this function after partial completion, the abort function can take
# care of what made it into the list.
update.new.setCache('networkInterfaces', interfaces)
if not hasattr(update.new, 'net'):
return None
# Make a dictionary of old interfaces. Any new interfaces that are
# identical to an old one do not need to be changed.
oldInterfaces = dict()
if update.old is not None:
cachedInterfaces = update.old.getCache('networkInterfaces')
oldInterfaces = {iface['name']: iface for iface in cachedInterfaces}
devices = update.new.getCache('networkDevices')
devIters = {t: itertools.cycle(devices[t]) for t in devices.keys()}
for name, cfg in update.new.net.iteritems():
# Check for required fields.
res = pdutils.check(cfg, dict, ['intfName', 'type'])
if res:
out.warn('Network interface definition {}\n'.format(res))
raise Exception("Interface definition missing field(s)")
iface = {
'name': name, # Name (not used?)
'netType': cfg['type'], # Type (wan, lan, wifi)
'internalIntf': cfg['intfName'] # Interface name in chute
}
if iface['name'] in oldInterfaces:
oldIface = oldInterfaces[iface['name']]
if interfaceDefsEqual(iface, oldIface):
# If old interface is the same, then keep it and move on.
interfaces.append(oldIface)
continue
# Try to find a physical device of the requested type.
#
# Note: we try this first because it can fail, and then we will not try
# to allocate any resources for it.
try:
device = devIters[cfg['type']].next()
iface['device'] = device['name']
except (KeyError, StopIteration):
out.warn("Request for {} device cannot be fulfilled".
format(cfg['type']))
raise Exception("Missing device(s) requested by chute")
if cfg['type'] == "wifi":
getNetworkConfigWifi(update, name, cfg, iface)
# Pass on DHCP configuration if it exists.
if 'dhcp' in cfg:
iface['dhcp'] = cfg['dhcp']
interfaces.append(iface)
update.new.setCache('networkInterfaces', interfaces)
def timeoutConnection(self):
out.warn('Connection has timed out!')
self.d.errback(RiffleError("The remote connection is unavailable."))
def reloadChutes():
"""
This function is called to restart any chutes that were running prior to the system being restarted.
It waits for pdconfd to come up and report whether or not it failed to bring up any of the interfaces
that existed before the power cycle. If pdconfd indicates something failed we then force a stop update
in order to bring down all interfaces associated with that chute and mark it with a warning.
If the stop fails we mark the chute with a warning manually and change its state to stopped and save to
storage this isn't great as it could mean our system still has interfaces up associated with that chute.
If pdconfd doesn't report failure we attempt to start the chute and if this fails we trust the abort process
to restore the system to a consistent state and we manually mark the chute as stopped and add a warning to it.
:param None
:returns: (list) A list of update dicts to be used to create updateObjects that should be run before accepting new updates.
"""
if not settings.PDCONFD_ENABLED:
return []
chuteStore = chutestorage.ChuteStorage()
chutes = [ ch for ch in chuteStore.getChuteList() if ch.state == 'running']
# Part of restoring the chute to its previously running state is reclaiming
# IP addresses, interface names, etc. that it had previously.
for chute in chutes:
reclaimNetworkResources(chute)
#We need to make sure confd is up and all interfaces have been brought up properly
confdup = False
while not confdup:
confdInfo = waitSystemUp()
if confdInfo == None:
time.sleep(1)
continue
confdup = True
confdInfo = str2json(confdInfo)
#Remove any chutes from the restart queue if confd failed to bring up the
#proper interfaces
#
# At this point, we only care about the chute names, not the full objects.
# We are using sets of chute names for their O(1) membership test and
# element uniqueness.
okChutes = set([ch.name for ch in chutes])
failedChutes = set()
for iface in confdInfo:
if iface.get('success') is False:
failedChuteName = iface.get('comment')
if failedChuteName == settings.RESERVED_CHUTE:
out.warn('Failed to load a system config section')
elif failedChuteName in okChutes:
# This was a chute that we are supposed to restart, but one of
# its config sections failed to load.
okChutes.remove(failedChuteName)
failedChutes.add(failedChuteName)
elif failedChuteName not in failedChutes:
# In this case, the name in the comment was not something that
# we recognize from the chute storage. Maybe the chute storage
# file was deleted but not the config files, or someone
# manually modified the config files. Anyway, we cannot
# attempt to stop this chute because the object does not exist,
# but we can give a warning message.
out.warn('Failed to load config section for '
'unrecognized chute: {}'.format(failedChuteName))
#First stop all chutes that failed to bring up interfaces according to
#pdconfd then start successful ones #We do this because pdfcd needs to
#handle cleaning up uci files and then tell pdconfd
updates = []
for ch in failedChutes:
updates.append(dict(updateClass='CHUTE', updateType='stop', name=ch,
tok=timeint(), func=updateStatus,
warning=FAILURE_WARNING))
for ch in okChutes:
updates.append(dict(updateClass='CHUTE', updateType='restart', name=ch,
tok=timeint(), func=updateStatus))
return updates
def getDeveloperFirewallRules(update):
"""
Generate other firewall rules requested by the developer such as redirects.
The object returned is a list of tuples (config, options).
"""
interfaces = update.new.getCache('networkInterfaces')
if interfaces is None:
return None
rules = list()
if hasattr(update.new, "firewall"):
for rule in update.new.firewall:
if rule['type'] == 'redirect':
config = {'type': 'redirect'}
options = {
'name': rule['name'],
'proto': 'tcpudp'
}
from_parts = rule['from'].strip().split(':')
to_parts = rule['to'].strip().split(':')
# Do not allow rules that do not pertain to the chute.
if "@host.lan" in from_parts[0] and "@host.lan" in to_parts[0]:
raise Exception("Unable to add firewall rule - "
"dst and src are both outside of chute")
# From @host.lan means this is a DNAT rule (redirect to the chute).
if from_parts[0] == "@host.lan":
options['target'] = "DNAT"
options['src'] = "wan"
if len(from_parts) > 1:
options['src_dport'] = from_parts[1]
# Find the interface specified in the rule, so we can get its
# IP address.
iface = findMatchingInterface(to_parts[0], interfaces)
if iface is None:
out.warn("No interface found with name {}\n".format(to_parts[0]))
raise Exception("Interface not found")
options['dest_ip'] = iface['externalIpaddr']
if len(to_parts) > 1:
options['dest_port'] = to_parts[1]
# This is an SNAT rule (redirect from the chute to host network).
elif to_parts[0] == "@host.lan":
options['target'] = "SNAT"
# TODO: Implement
out.warn("SNAT rules not supported yet")
raise Exception("SNAT rules not implemented")
# Could be forwarding between chute interfaces?
else:
out.warn("Other rules not supported yet")
raise Exception("Other rules not implemented")
rules.append((config, options))
update.new.setCache('developerFirewallRules', rules)
