def setup(args=None):
"""
This is the main setup function to establish the TCP listening logic for
the API server. This code also takes into account development or unit test mode.
"""
# Setup API server
api = ParadropAPIServer(reactor)
api.putChild('internal', Base(apiinternal, allowNone=True))
site = Site(api, timeout=None)
# Development mode
if(args and args.development):
thePort = settings.PDFCD_PORT + 10000
out.info('Using DEVELOPMENT variables')
# Disable sending the error traceback to the client
site.displayTracebacks = True
elif(args and args.unittest):
thePort = settings.PDFCD_PORT + 20000
out.info('Running under unittest mode')
site.displayTracebacks = True
else:
thePort = settings.PDFCD_PORT
site.displayTracebacks = False
initializeSystem()
# Setup the port we listen on
reactor.listenTCP(thePort, site)
# Never return from here
reactor.run()
def release(self, item):
out.info("Trying to release {} from pool {}\n".format(str(item), self.__class__.__name__))
if item in self.used:
self.used.remove(item)
self.recentlyReleased.append(item)
else:
raise Exception("Trying to release unreserved item")
def failAndCleanUpDocker(validImages, validContainers):
"""
Clean up any intermediate containers that may have resulted from a failure and throw an Exception so that
the abort process is called.
:param validImages: A list of dicts containing the Id's of all the images that should exist on the system.
:type validImages: list
:param validContainers: A list of the Id's of all the containers that should exist on the system.
:type validContainers: list
:returns: None
"""
c = docker.Client(base_url="unix://var/run/docker.sock", version='auto')
#Clean up containers from failed build/start
currContainers = c.containers(quiet=True, all=True)
for cntr in currContainers:
if not cntr in validContainers:
out.info('Removing Invalid container with id: %s' % str(cntr.get('Id')))
c.remove_container(container=cntr.get('Id'))
#Clean up images from failed build
currImages = c.images(quiet=True, all=False)
for img in currImages:
if not img in validImages:
out.info('Removing Invalid image with id: %s' % str(img))
c.remove_image(image=img)
#Throw exception so abort plan is called and user is notified
raise Exception('Building or starting of docker image failed check your Dockerfile for errors.')
def castSuccess(res):
out.info("Completed API call (TODO: add details)")
# screen out Objectids on mongo returns. The remote objects have no
# need for them, and they confuse xmlrpc
if isinstance(res, dict):
res.pop('_id', None)
return res
def GET_test(self, request):
"""
A Simple test method to ping if the API server is working properly.
"""
request.setHeader('Access-Control-Allow-Origin', settings.PDFCD_HEADER_VALUE)
ip = apiutils.getIP(request)
out.info('Test called (%s)\n' % (ip))
request.setResponseCode(*pdapi.getResponse(pdapi.OK))
return "SUCCESS\n"
def close(self):
'''
Close all connections in all realms. Stop all polling connections.
'''
out.info("Portal closing all connections")
for k, v in self.realms.iteritems():
for c in v.connections:
c.destroy()
def getVirtPreamble(update):
out.warn('TODO implement me\n')
if update.updateType == 'create':
if not hasattr(update, 'dockerfile'):
return
if update.dockerfile is None:
return
else:
out.info('Using prexisting dockerfile.\n')
update.dockerfile = BytesIO(update.dockerfile.encode('utf-8'))
def stopChute(update):
"""
Stop a docker container based on the passed in update.
:param update: The update object containing information about the chute.
:type update: obj
:returns: None
"""
out.info('Attempting to stop chute %s\n' % (update.name))
c = docker.Client(base_url='unix://var/run/docker.sock', version='auto')
c.stop(container=update.name)
def reserve(self, item, strict=True):
"""
Mark item as used.
If strict is True, raises an exception if the item is already used.
"""
out.info("Trying to reserve {} from pool {}\n".format(str(item), self.__class__.__name__))
if item in self.used:
if strict:
raise Exception("Trying to reserve a used item")
else:
self.used.add(item)
def attach(self, avatar, mind):
'''
Completes the riffle association by attaching the avatar to its remote, adding
it to the pool of connection stored here, and broadcasting the new connection.
To listen for this
'''
avatar.attached(mind)
self.connections.add(avatar)
out.info('Connected: ' + avatar.name)
smokesignal.emit('%sConnected' % self.avatar.__name__, avatar, self)
def startChute(update):
"""
Build and deploy a docker container based on the passed in update.
:param update: The update object containing information about the chute.
:type update: obj
:returns: None
"""
out.info('Attempting to start new Chute %s \n' % (update.name))
repo = update.name + ":latest"
dockerfile = update.dockerfile
name = update.name
host_config = build_host_config(update)
c = docker.Client(base_url="unix://var/run/docker.sock", version='auto')
#Get Id's of current images for comparison upon failure
validImages = c.images(quiet=True, all=False)
validContainers = c.containers(quiet=True, all=True)
buildFailed = False
for line in c.build(rm=True, tag=repo, fileobj=dockerfile):
#if we encountered an error make note of it
if 'errorDetail' in line:
buildFailed = True
for key, value in json.loads(line).iteritems():
if isinstance(value, dict):
continue
elif key == 'stream':
update.pkg.request.write(str(value))
else:
update.pkg.request.write(str(value) + '\n')
#If we failed to build skip creating and starting clean up and fail
if buildFailed:
failAndCleanUpDocker(validImages, validContainers)
try:
container = c.create_container(
image=repo, name=name, host_config=host_config
)
c.start(container.get('Id'))
out.info("Successfully started chute with Id: %s\n" % (str(container.get('Id'))))
except Exception as e:
failAndCleanUpDocker(validImages, validContainers)
setup_net_interfaces(update)
def readConfig(self, files):
"""
Load configuration files and return configuration objects.
This method only loads the configuration files without making any
changes to the system and returns configuration objects as a generator.
"""
# Keep track of headers (section type and name) that have been
# processed so far. The dictionary maps them to filename, so that we
# can print a useful warning message about duplicates.
usedHeaders = dict()
for fn in files:
out.info("Reading file {}\n".format(fn))
uci = UCIConfig(fn)
config = uci.readConfig()
for section, options in config:
# Sections differ in where they put the name, if they have one.
if "name" in section:
name = section['name']
elif "name" in options:
name = options['name']
else:
name = None
# Get section comment string (optional, but Paradop uses it).
comment = section.get('comment', None)
try:
cls = configTypeMap[section['type']]
except:
out.warn("Unsupported section type {} in {}\n".format(
section['type'], fn))
continue
try:
obj = cls.build(self, fn, name, options, comment)
except:
out.warn("Error building object from section {}:{} in "
"{}\n".format(section['type'], name, fn))
continue
key = obj.getTypeAndName()
if key in usedHeaders:
out.warn("Section {}:{} from {} overrides section in "
"{}\n".format(section['type'], name, fn,
usedHeaders[key]))
usedHeaders[key] = fn
yield obj
def restartChute(update):
"""
Start a docker container based on the passed in update.
:param update: The update object containing information about the chute.
:type update: obj
:returns: None
"""
out.info('Attempting to restart chute %s\n' % (update.name))
c = docker.Client(base_url='unix://var/run/docker.sock', version='auto')
c.start(container=update.name)
setup_net_interfaces(update)
def onJoin(self, details):
out.info(str(self.__class__.__name__) + ' crossbar session connected')
yield
# Inform whoever created us that the session has finished connecting.
# Useful in situations where you need to fire off a single call and not a
# full wamplet
try:
if self.dee is not None:
yield self.dee.callback(self)
except:
# print 'No onJoin deferred callback set.'
pass
def performUpdates(self):
"""This is the main working function of the PDConfigurer class.
It should be executed as a separate thread, it does the following:
checks for any updates to perform
does them
responds to the server
removes the update
checks for more updates
if more exist it calls itself again more quickly
else it puts itself to sleep for a little while
"""
#add any chutes that should already be running to the front of the update queue before processing any updates
startQueue = reloadChutes()
self.updateLock.acquire()
# insert the data into the front of our update queue so that all old chutes restart befor new ones are processed
for updateObj in startQueue:
self.updateQueue.insert(0, updateObj)
self.updateLock.release()
# Always perform this work
while(self.reactor.running):
# Check for new updates
updateObj = self.getNextUpdate()
if(updateObj is None):
time.sleep(1)
continue
try:
# Take the object and identify the update type
update = updateObject.parse(updateObj)
out.info('Performing update %s\n' % (update))
# TESTING start
if(settings.FC_BOUNCE_UPDATE): # pragma: no cover
out.testing('Bouncing update %s, result: %s\n' % (
update, settings.FC_BOUNCE_UPDATE))
update.complete(success=True, message=settings.FC_BOUNCE_UPDATE)
continue
# TESTING end
# Based on each update type execute could be different
update.execute()
except Exception as e:
out.exception(e, True)
def setConfig(chute, old, cacheKeys, filepath):
"""
Helper function used to modify config file of each various setting in /etc/config/
Returns:
True: if it modified a file
False: if it did NOT cause any modifications
Raises exception if an error occurs.
"""
# First pull out all the cache keys from the @new chute
newconfigs = []
for c in cacheKeys:
t = chute.getCache(c)
if(t):
newconfigs += t
if(len(newconfigs) == 0):
out.info('no settings to add %r\n' % (chute))
# We are no longer returning because we need to remove the old configs if necessary
# return False
# add comment to each config so we can differentiate between different chute specific configs
for e in newconfigs:
c, o = e
c['comment'] = chute.name
# Get the old configs from the file for this chuteid
# Find the config file
cfgFile = uci.UCIConfig(filepath)
# Get all the configs that existed in the old version
# Note we are getting the old configs from the etc/config/ file instead of the chute object
# This is to improve reliability - sometimes the file isn't changed it should be
# because we have reset the board, messed with chutes, etc. and the new/old chuteobjects are identical
oldconfigs = cfgFile.getChuteConfigs(chute.name)
if (uci.chuteConfigsMatch(oldconfigs, newconfigs)):
# configs match, skipping reloading
return False
else:
# We need to make changes so delete old configs, load new configs
# configs don't match, changing chutes and reloading
cfgFile.delConfigs(oldconfigs)
cfgFile.addConfigs(newconfigs)
cfgFile.save(backupToken="paradrop", internalid=chute.name)
return True
def saveToDisk(self):
"""Saves the data to disk."""
out.info("Saving to disk (%s)\n" % (self.filename))
# Make sure they want to save
if not self.attrSaveable():
return
# Get whatever the data is
pyld = self.exportAttr(self.getAttr())
# Write the file to disk, truncate if it exists
try:
pickle.dump(pyld, pdos.open(self.filename, "wb"))
pdos.syncFS()
except Exception as e:
out.err("Error writing to disk %s\n" % (str(e)))
def removeChute(update):
"""
Remove a docker container and the image it was built on based on the passed in update.
:param update: The update object containing information about the chute.
:type update: obj
:returns: None
"""
out.info('Attempting to remove chute %s\n' % (update.name))
c = docker.Client(base_url='unix://var/run/docker.sock', version='auto')
repo = update.name + ":latest"
name = update.name
try:
c.remove_container(container=name, force=True)
c.remove_image(image=repo)
except Exception as e:
#TODO: Might want to notify ourselves we could have removed container but failed to remove image for a number of reasons
update.complete(success=False, message=e.message)
raise e
def postprocess(self, request, key, failureDict, logUsage):
"""
If the client is successful in their request, we should:
* reset their failure attempts if failureDict is not none.
* set success response code
* If usage is not none, add usage track info of the api call
"""
request.setResponseCode(*pdapi.getResponse(pdapi.OK))
if(logUsage):
tictoc, ip, devid = logUsage
duration = 0 # self.perf.toc(tictoc)
# when devid is none, we log "Null" into the database
if(devid is None):
devid = "Null"
# Log the info of this call
# TODO self.usageTracker.addTrackInfo(ip, devid, request.path, self.usageTracker.SUCCESS, duration, request.content.getvalue())
if(failureDict is not None):
if(key in failureDict):
out.info('Clearing %s from failure list\n' % (key))
del failureDict[key]
def POST_stopChute(self, apiPkg):
"""
Description:
Arguments:
POST request:
Returns:
On success: SUCCESS object
On failure: FAILURE object
"""
out.info('Stopping chute...')
# For now fake out a create chute message
update = dict(updateClass='CHUTE', updateType='stop', name=apiPkg.inputArgs.get('name'),
tok=timeint(), pkg=apiPkg, func=self.rest.complete)
self.rest.configurer.updateList(**update)
# Tell our system we aren't done yet (the configurer will deal with
# closing the connection)
apiPkg.setNotDoneYet()
def execute(self):
try:
proc = subprocess.Popen(self.command, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
self.pid = proc.pid
for line in proc.stdout:
out.verbose("{}: {}".format(self.command[0], line))
for line in proc.stderr:
out.verbose("{}: {}".format(self.command[0], line))
self.result = proc.wait()
out.info('Command "{}" returned {}\n'.format(
" ".join(self.command), self.result))
except Exception as e:
out.info('Command "{}" raised exception {}\n'.format(
" ".join(self.command), e))
self.result = e
if self.parent is not None:
self.parent.executed.append(self)
return (self.result == 0)
def setup_net_interfaces(update):
"""
Link interfaces in the host to the internal interface in the docker container using pipework.
:param update: The update object containing information about the chute.
:type update: obj
:returns: None
"""
interfaces = update.new.getCache('networkInterfaces')
for iface in interfaces:
if iface.get('netType') == 'wifi':
IP = iface.get('ipaddrWithPrefix')
internalIntf = iface.get('internalIntf')
externalIntf = iface.get('externalIntf')
else: # pragma: no cover
continue
# Construct environment for pipework call. It only seems to require
# the PATH variable to include the directory containing the docker
# client. On Snappy this was not happening by default, which is why
# this code is here.
env = {"PATH": os.environ.get("PATH", "")}
if settings.DOCKER_BIN_DIR not in env['PATH']:
env['PATH'] += ":" + settings.DOCKER_BIN_DIR
cmd = ['/apps/paradrop/current/bin/pipework', externalIntf, '-i',
internalIntf, update.name, IP]
out.info("Calling: {}\n".format(" ".join(cmd)))
try:
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, env=env)
for line in proc.stdout:
out.info("pipework: {}\n".format(line.strip()))
for line in proc.stderr:
out.warn("pipework: {}\n".format(line.strip()))
except OSError as e:
out.warn('Command "{}" failed\n'.format(" ".join(cmd)))
out.exception(e, True)
raise e
def loadFromDisk(self):
"""Attempts to load the data from disk.
Returns True if success, False otherwise."""
if pdos.exists(self.filename):
deleteFile = False
out.info("Loading from disk\n")
data = ""
try:
pyld = pickle.load(pdos.open(self.filename, "rb"))
self.setAttr(self.importAttr(pyld))
return True
except Exception as e:
out.err("Error loading from disk: %s\n" % (str(e)))
deleteFile = True
# Delete the file
if deleteFile:
try:
pdos.unlink(self.filename)
except Exception as e:
out.err("Error unlinking %s\n" % (self.filename))
return False
def next(self):
if len(self.used) >= self.numValues:
raise Exception("No items left in pool")
while len(self.recentlyReleased) > 0:
item = self.recentlyReleased.pop(0)
if item not in self.used:
out.info("Claiming recently released {} from pool {}\n".format(str(item), self.__class__.__name__))
self.used.add(item)
return item
# The for loop puts a limit on the number of iterations that we spend
# looking for a free subnet. Passing the check above should imply that
# there is at least one available item, but we want to be extra
# careful to avoid a busy loop.
for i in range(self.numValues):
item = self.cycle.next()
if item not in self.used:
out.info("Claiming new item {} from pool {}\n".format(str(item), self.__class__.__name__))
self.used.add(item)
return item
# There is a bug if we hit this line.
raise Exception("No items left in pool (BUG)")
def stockSubscribe(self, handler, topic=None, options=None):
out.info('cxbr: (%s) subscribe (%s)' % (self.pdid, topic,))
return ApplicationSession.subscribe(self, handler, topic=topic, options=options)
def stockCall(self, pdid, procedure, *args, **kwargs):
out.info('cxbr: (%s) calling (%s)' % (self.pdid, procedure,))
return ApplicationSession.call(self, procedure, *args, **kwargs)
def save(self, backupToken=None, internalid=None):
"""
Saves out the file in the proper format.
Arguments:
[backupPath] : Save a backup copy of the UCI file to the path provided.
Should be a token name like 'backup', it gets appended with a hyphen.
"""
# Save original copy
if(backupToken):
self.backup(backupToken)
output = ""
# Now generate what the file would look like
for c, o in self.config:
#print("c: %s\n" % c.keys())
line = "config %s" % c['type']
# Check for optional name
if('name' in c.keys()):
line += " %s" % c['name']
if('comment' in c.keys()):
line += " #%s" % c['comment']
output += "%s\n" % line
# Get options
# check for lists first, if they exist remove them first
if('list' in o.keys()):
theLists = o['list']
else:
theLists = None
# Now process everything else quick
for k,v in o.iteritems():
# Make sure we skip the lists key
if(k != 'list'):
line = "\toption %s '%s'\n" % (k,v)
output += line
# Now process the list
if(theLists):
# theLists is a dict where the key is each list name
# and the value is a list of the options we need to include
for k,v in theLists.iteritems():
# so @v here is a list
for vals in v:
# Now append a list set to the config
line = "\tlist %s '%s'\n" % (k, vals)
output += line
# Now add one extra newline before the next set
output += "\n"
# Now write to disk
try:
out.info('Saving %s to disk\n' % (self.filepath))
fd = pdos.open(self.filepath, 'w')
fd.write(output)
# Guarantee that its written to disk before we close
fd.flush()
os.fsync(fd.fileno())
fd.close()
except Exception as e:
out.err('Unable to save new config %s, %s\n' % (self.filepath, str(e)))
out.err('Config may be corrupted, backup exists at /tmp/%s\n' % (self.myname))
def stockRegister(self, endpoint, procedure=None, options=None):
out.info('cxbr: (%s) registering (%s)' % (self.pdid, procedure,))
return ApplicationSession.register(self, endpoint, procedure=procedure, options=options)
def castFailure(failure):
''' Converts an exception (or general failure) into an xmlrpc fault for transmission. '''
out.info("Failed API call (TODO: categorize errors)")
raise xmlrpc.Fault(123, failure.getErrorMessage())
def connectionClosed(self, avatar):
out.info('Disconnected: ' + str(avatar.name))
smokesignal.emit('%sDisconnected' % self.avatar.__name__, avatar, self)
self.connections.remove(avatar)
