def _perform_update(self, update):
"""
Perform a single update, to be called by perform_updates.
This is split from perform_updates for easier unit testing.
"""
try:
# Mark update as having been started.
update.started()
out.info('Performing update %s\n' % (update))
# TESTING start
# TODO: still need this?
if(settings.FC_BOUNCE_UPDATE): # pragma: no cover
out.testing('Bouncing update %s, result: %s\n' % (
update, settings.FC_BOUNCE_UPDATE))
update.complete(success=True, message=settings.FC_BOUNCE_UPDATE)
return
# TESTING end
# Based on each update type execute could be different
update.execute()
except Exception as e:
out.exception(e, True)
def call_retry(cmd, env, delay=3, tries=3):
# Make sure each component of the command is a string. Otherwisew we will
# get errors.
clean_cmd = [str(v) for v in cmd]
while tries >= 0:
tries -= 1
out.info("Calling: {}\n".format(" ".join(clean_cmd)))
try:
proc = subprocess.Popen(clean_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
for line in proc.stdout:
out.info("{}: {}\n".format(clean_cmd[0], line.strip()))
for line in proc.stderr:
out.warn("{}: {}\n".format(clean_cmd[0], line.strip()))
return proc.returncode
except OSError as e:
out.warn('Command "{}" failed\n'.format(" ".join(clean_cmd)))
if tries <= 0:
out.exception(e, True)
raise e
time.sleep(delay)
def executePlans(update):
"""
Primary function that actually executes all the functions that were added to plans by all
the exc modules. This function can heavily modify the OS/files/etc.. so the return value is
very important.
Returns:
True in error : abortPlans function should be called
False otherwise : everything is OK
"""
out.header('Executing plans %r\n' % (update))
# Finding the functions to call is actually done by a 'iterator' like function in the plangraph module
while (True):
# This function either returns None or a tuple just like generate added to it
p = update.plans.getNextTodo()
# No more to do?
if (not p):
break
# Explode tuple otherwise
func, args = p
# We are in a try-except block so if func isn't callable that will catch it
try:
out.verbose('Calling %s\n' % (func))
update.progress("Calling {}".format(func.__name__))
#
# Call the function from the execution plan
#
# args may be empty, but we don't want to pass in a tuple if we don't need to.
# This below explodes the args so if @args is (), then what is passed is @update
skipme = func(*((update, ) + args))
except Exception as e:
out.exception(e, True)
# plans = str(update.plans)) # Removed because breaks new out.exception call
out.warn("Failed to execute plan %s%s" % (func.__name__, args))
update.responses.append({
'exception': str(e),
'traceback': traceback.format_exc()
})
update.failure = str(e)
return True
# The functions we call here can return other functions, if they do these are functions that should
# be skipped later on (for instance a set* function discovering it didn't change anything, later on
# we shouldn't call the corresponding reload function)
if (skipme):
# These functions can return individual functions to skip, or a list of multiple functions
if (not isinstance(skipme, list)):
skipme = [skipme]
for skip in skipme:
out.warn('Identified a skipped function: %r\n' % (skip))
update.plans.registerSkip(skip)
# Now we are done
return False
def complete(self, **kwargs):
"""
Signal to the API server that any action we need to perform is
complete and the API server can finish its connection with the
client that initiated the API request.
"""
# Save a timestamp from when we finished execution.
self.endTime = time.time()
if (settings.DEBUG_MODE):
kwargs['responses'] = self.responses
# Set our results
self.result = kwargs
d = None
if hasattr(self, 'deferred'):
d = self.deferred
self.deferred = None
try:
message = "Completed {} operation on chute {}: {}".format(
self.updateType, self.new.name,
"success" if kwargs['success'] else "failure")
out.usage(message,
chute=self.new.name,
updateType=self.updateType,
createdTime=self.createdTime,
startTime=self.startTime,
endTime=self.endTime,
**kwargs)
except Exception as e:
out.exception(e, True)
if d:
reactor.callFromThread(d.errback, Failure(e))
# Last message to send to observers.
msg = {'time': self.endTime, 'message': message}
self.messages.append(msg)
# Mark the update as complete and notify any observers. Observers
# should call remove_message_observer in their on_complete handler.
self.completed = True
for observer in self.message_observers:
observer.on_message(msg)
observer.on_complete()
if 'message' in kwargs:
self.progress(kwargs['message'])
if d:
reactor.callFromThread(d.callback, self)
def _perform_update(self, update):
"""
Perform a single update, to be called by perform_updates.
This is split from perform_updates for easier unit testing.
"""
# Add to the active set when processing starts. It is a dictionary, so
# it does not matter if this is the first time we see this update or
# if we are resuming it.
self.active_changes[update.change_id] = update
try:
# Mark update as having been started.
update.started()
out.info('Performing update %s\n' % (update))
# TESTING start
# TODO: still need this?
if (settings.FC_BOUNCE_UPDATE): # pragma: no cover
out.testing('Bouncing update %s, result: %s\n' %
(update, settings.FC_BOUNCE_UPDATE))
update.complete(success=True,
message=settings.FC_BOUNCE_UPDATE)
return
# TESTING end
# Based on each update type execute could be different
result = update.execute()
if isinstance(result, defer.Deferred):
# Update is not done, but it is yielding. When the deferred
# fires, add it back to the work queue to resume it.
#
# TODO: We could handle errors separately and use that to break
# the update pipeline, but then we need to propagate that
# through the update.execute, executionplan chain. For now,
# we have an update stage right after prepare_image that checks
# if the build was successful or throws an exception. That
# should work but is not very general.
def resume(result):
self.updateQueue.append(update)
result.addBoth(resume)
elif update.change_id in self.active_changes:
# Update is done, so remove it from the active list.
del self.active_changes[update.change_id]
except Exception as e:
out.exception(e, True)
def _perform_update(self, update):
"""
Perform a single update, to be called by perform_updates.
This is split from perform_updates for easier unit testing.
"""
# Add to the active set when processing starts. It is a dictionary, so
# it does not matter if this is the first time we see this update or
# if we are resuming it.
self.active_changes[update.change_id] = update
try:
# Mark update as having been started.
update.started()
out.info('Performing update %s\n' % (update))
# TESTING start
# TODO: still need this?
if(settings.FC_BOUNCE_UPDATE): # pragma: no cover
out.testing('Bouncing update %s, result: %s\n' % (
update, settings.FC_BOUNCE_UPDATE))
update.complete(success=True, message=settings.FC_BOUNCE_UPDATE)
return
# TESTING end
# Based on each update type execute could be different
result = update.execute()
if isinstance(result, defer.Deferred):
# Update is not done, but it is yielding. When the deferred
# fires, add it back to the work queue to resume it.
#
# TODO: We could handle errors separately and use that to break
# the update pipeline, but then we need to propagate that
# through the update.execute, executionplan chain. For now,
# we have an update stage right after prepare_image that checks
# if the build was successful or throws an exception. That
# should work but is not very general.
def resume(result):
self.updateQueue.append(update)
result.addBoth(resume)
elif update.change_id in self.active_changes:
# Update is done, so remove it from the active list.
del self.active_changes[update.change_id]
except Exception as e:
out.exception(e, True)
def call_retry(cmd, env, delay=3, tries=3):
# Make sure each component of the command is a string. Otherwisew we will
# get errors.
clean_cmd = [str(v) for v in cmd]
while tries >= 0:
tries -= 1
out.info("Calling: {}\n".format(" ".join(clean_cmd)))
try:
proc = subprocess.Popen(clean_cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, env=env)
for line in proc.stdout:
out.info("{}: {}\n".format(clean_cmd[0], line.strip()))
for line in proc.stderr:
out.warn("{}: {}\n".format(clean_cmd[0], line.strip()))
return proc.returncode
except OSError as e:
out.warn('Command "{}" failed\n'.format(" ".join(clean_cmd)))
if tries <= 0:
out.exception(e, True)
raise e
time.sleep(delay)
def executePlans(update):
"""
Primary function that actually executes all the functions that were added to plans by all
the exc modules. This function can heavily modify the OS/files/etc.. so the return value is
very important.
Returns:
True in error : abortPlans function should be called
False otherwise : everything is OK
"""
out.header('Executing plans %r\n' % (update))
# Finding the functions to call is actually done by a 'iterator' like function in the plangraph module
while(True):
# This function either returns None or a tuple just like generate added to it
p = update.plans.getNextTodo()
# No more to do?
if(not p):
break
# Explode tuple otherwise
func, args = p
# We are in a try-except block so if func isn't callable that will catch it
try:
out.verbose('Calling %s\n' % (func))
update.progress("Calling {}".format(func.__name__))
#
# Call the function from the execution plan
#
# args may be empty, but we don't want to pass in a tuple if we don't need to.
# This below explodes the args so if @args is (), then what is passed is @update
skipme = func(*((update, ) + args))
except Exception as e:
out.exception(e, True)
# plans = str(update.plans)) # Removed because breaks new out.exception call
out.warn("Failed to execute plan %s%s" % (func.__name__, args))
update.responses.append({'exception': str(e), 'traceback': traceback.format_exc()})
update.failure = str(e)
return True
# The functions we call here can return other functions, if they do
# these are functions that should be skipped later on (for instance a
# set* function discovering it didn't change anything, later on we
# shouldn't call the corresponding reload function)
if skipme is not None:
# If the function returned a Deferred, we will drop out of the
# execution pipeline and resume later.
if isinstance(skipme, Deferred):
out.verbose('Function {} returned a Deferred'.format(func))
return skipme
# These functions can return individual functions to skip, or a
# list of multiple functions
elif callable(skipme):
skipme = [skipme]
for skip in skipme:
out.warn('Identified a skipped function: %r\n' % (skip))
update.plans.registerSkip(skip)
# Now we are done
return False