def _perform_update(self, update): """ Perform a single update, to be called by perform_updates. This is split from perform_updates for easier unit testing. """ try: # Mark update as having been started. update.started() out.info('Performing update %s\n' % (update)) # TESTING start # TODO: still need this? if(settings.FC_BOUNCE_UPDATE): # pragma: no cover out.testing('Bouncing update %s, result: %s\n' % ( update, settings.FC_BOUNCE_UPDATE)) update.complete(success=True, message=settings.FC_BOUNCE_UPDATE) return # TESTING end # Based on each update type execute could be different update.execute() except Exception as e: out.exception(e, True)
def call_retry(cmd, env, delay=3, tries=3): # Make sure each component of the command is a string. Otherwisew we will # get errors. clean_cmd = [str(v) for v in cmd] while tries >= 0: tries -= 1 out.info("Calling: {}\n".format(" ".join(clean_cmd))) try: proc = subprocess.Popen(clean_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) for line in proc.stdout: out.info("{}: {}\n".format(clean_cmd[0], line.strip())) for line in proc.stderr: out.warn("{}: {}\n".format(clean_cmd[0], line.strip())) return proc.returncode except OSError as e: out.warn('Command "{}" failed\n'.format(" ".join(clean_cmd))) if tries <= 0: out.exception(e, True) raise e time.sleep(delay)
def executePlans(update): """ Primary function that actually executes all the functions that were added to plans by all the exc modules. This function can heavily modify the OS/files/etc.. so the return value is very important. Returns: True in error : abortPlans function should be called False otherwise : everything is OK """ out.header('Executing plans %r\n' % (update)) # Finding the functions to call is actually done by a 'iterator' like function in the plangraph module while (True): # This function either returns None or a tuple just like generate added to it p = update.plans.getNextTodo() # No more to do? if (not p): break # Explode tuple otherwise func, args = p # We are in a try-except block so if func isn't callable that will catch it try: out.verbose('Calling %s\n' % (func)) update.progress("Calling {}".format(func.__name__)) # # Call the function from the execution plan # # args may be empty, but we don't want to pass in a tuple if we don't need to. # This below explodes the args so if @args is (), then what is passed is @update skipme = func(*((update, ) + args)) except Exception as e: out.exception(e, True) # plans = str(update.plans)) # Removed because breaks new out.exception call out.warn("Failed to execute plan %s%s" % (func.__name__, args)) update.responses.append({ 'exception': str(e), 'traceback': traceback.format_exc() }) update.failure = str(e) return True # The functions we call here can return other functions, if they do these are functions that should # be skipped later on (for instance a set* function discovering it didn't change anything, later on # we shouldn't call the corresponding reload function) if (skipme): # These functions can return individual functions to skip, or a list of multiple functions if (not isinstance(skipme, list)): skipme = [skipme] for skip in skipme: out.warn('Identified a skipped function: %r\n' % (skip)) update.plans.registerSkip(skip) # Now we are done return False
def complete(self, **kwargs): """ Signal to the API server that any action we need to perform is complete and the API server can finish its connection with the client that initiated the API request. """ # Save a timestamp from when we finished execution. self.endTime = time.time() if (settings.DEBUG_MODE): kwargs['responses'] = self.responses # Set our results self.result = kwargs d = None if hasattr(self, 'deferred'): d = self.deferred self.deferred = None try: message = "Completed {} operation on chute {}: {}".format( self.updateType, self.new.name, "success" if kwargs['success'] else "failure") out.usage(message, chute=self.new.name, updateType=self.updateType, createdTime=self.createdTime, startTime=self.startTime, endTime=self.endTime, **kwargs) except Exception as e: out.exception(e, True) if d: reactor.callFromThread(d.errback, Failure(e)) # Last message to send to observers. msg = {'time': self.endTime, 'message': message} self.messages.append(msg) # Mark the update as complete and notify any observers. Observers # should call remove_message_observer in their on_complete handler. self.completed = True for observer in self.message_observers: observer.on_message(msg) observer.on_complete() if 'message' in kwargs: self.progress(kwargs['message']) if d: reactor.callFromThread(d.callback, self)
def _perform_update(self, update): """ Perform a single update, to be called by perform_updates. This is split from perform_updates for easier unit testing. """ # Add to the active set when processing starts. It is a dictionary, so # it does not matter if this is the first time we see this update or # if we are resuming it. self.active_changes[update.change_id] = update try: # Mark update as having been started. update.started() out.info('Performing update %s\n' % (update)) # TESTING start # TODO: still need this? if (settings.FC_BOUNCE_UPDATE): # pragma: no cover out.testing('Bouncing update %s, result: %s\n' % (update, settings.FC_BOUNCE_UPDATE)) update.complete(success=True, message=settings.FC_BOUNCE_UPDATE) return # TESTING end # Based on each update type execute could be different result = update.execute() if isinstance(result, defer.Deferred): # Update is not done, but it is yielding. When the deferred # fires, add it back to the work queue to resume it. # # TODO: We could handle errors separately and use that to break # the update pipeline, but then we need to propagate that # through the update.execute, executionplan chain. For now, # we have an update stage right after prepare_image that checks # if the build was successful or throws an exception. That # should work but is not very general. def resume(result): self.updateQueue.append(update) result.addBoth(resume) elif update.change_id in self.active_changes: # Update is done, so remove it from the active list. del self.active_changes[update.change_id] except Exception as e: out.exception(e, True)
def _perform_update(self, update): """ Perform a single update, to be called by perform_updates. This is split from perform_updates for easier unit testing. """ # Add to the active set when processing starts. It is a dictionary, so # it does not matter if this is the first time we see this update or # if we are resuming it. self.active_changes[update.change_id] = update try: # Mark update as having been started. update.started() out.info('Performing update %s\n' % (update)) # TESTING start # TODO: still need this? if(settings.FC_BOUNCE_UPDATE): # pragma: no cover out.testing('Bouncing update %s, result: %s\n' % ( update, settings.FC_BOUNCE_UPDATE)) update.complete(success=True, message=settings.FC_BOUNCE_UPDATE) return # TESTING end # Based on each update type execute could be different result = update.execute() if isinstance(result, defer.Deferred): # Update is not done, but it is yielding. When the deferred # fires, add it back to the work queue to resume it. # # TODO: We could handle errors separately and use that to break # the update pipeline, but then we need to propagate that # through the update.execute, executionplan chain. For now, # we have an update stage right after prepare_image that checks # if the build was successful or throws an exception. That # should work but is not very general. def resume(result): self.updateQueue.append(update) result.addBoth(resume) elif update.change_id in self.active_changes: # Update is done, so remove it from the active list. del self.active_changes[update.change_id] except Exception as e: out.exception(e, True)
def call_retry(cmd, env, delay=3, tries=3): # Make sure each component of the command is a string. Otherwisew we will # get errors. clean_cmd = [str(v) for v in cmd] while tries >= 0: tries -= 1 out.info("Calling: {}\n".format(" ".join(clean_cmd))) try: proc = subprocess.Popen(clean_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) for line in proc.stdout: out.info("{}: {}\n".format(clean_cmd[0], line.strip())) for line in proc.stderr: out.warn("{}: {}\n".format(clean_cmd[0], line.strip())) return proc.returncode except OSError as e: out.warn('Command "{}" failed\n'.format(" ".join(clean_cmd))) if tries <= 0: out.exception(e, True) raise e time.sleep(delay)
def executePlans(update): """ Primary function that actually executes all the functions that were added to plans by all the exc modules. This function can heavily modify the OS/files/etc.. so the return value is very important. Returns: True in error : abortPlans function should be called False otherwise : everything is OK """ out.header('Executing plans %r\n' % (update)) # Finding the functions to call is actually done by a 'iterator' like function in the plangraph module while(True): # This function either returns None or a tuple just like generate added to it p = update.plans.getNextTodo() # No more to do? if(not p): break # Explode tuple otherwise func, args = p # We are in a try-except block so if func isn't callable that will catch it try: out.verbose('Calling %s\n' % (func)) update.progress("Calling {}".format(func.__name__)) # # Call the function from the execution plan # # args may be empty, but we don't want to pass in a tuple if we don't need to. # This below explodes the args so if @args is (), then what is passed is @update skipme = func(*((update, ) + args)) except Exception as e: out.exception(e, True) # plans = str(update.plans)) # Removed because breaks new out.exception call out.warn("Failed to execute plan %s%s" % (func.__name__, args)) update.responses.append({'exception': str(e), 'traceback': traceback.format_exc()}) update.failure = str(e) return True # The functions we call here can return other functions, if they do # these are functions that should be skipped later on (for instance a # set* function discovering it didn't change anything, later on we # shouldn't call the corresponding reload function) if skipme is not None: # If the function returned a Deferred, we will drop out of the # execution pipeline and resume later. if isinstance(skipme, Deferred): out.verbose('Function {} returned a Deferred'.format(func)) return skipme # These functions can return individual functions to skip, or a # list of multiple functions elif callable(skipme): skipme = [skipme] for skip in skipme: out.warn('Identified a skipped function: %r\n' % (skip)) update.plans.registerSkip(skip) # Now we are done return False