class Base(object): """ Define all helper necessary functions. This class must be inherited by all modules. If the class is to be a virtual machine, L{Vm} should be inherited instead. """ def __init__(self, vm = None, vms = [], factory = '', **kwargs): self._factory = factory if vm != None or vms != []: self.run_on_vm = True else: self.run_on_vm = False #: the vm on which the steps take place self.vm = vm #: multiple vm can be specified instead. #: if a list, then a list of factories will be returned by L{start} #: if a dict, then it's L{'vm1' : [ vm_obj, factory_obj], ...} self.is_vm = False self.can_snap = False self.commands = Dummy(self.name) self.paths = [] self.post_start_hook = [] self.pre_start_hook = [] self.fathers = [] self.from_inside = False self.parents = [] #: keep track of the whole path that led here self.ancestors = [] #: keep track of all the final leave from here self.descendants = set([]) self.vms = vms if 'force_new_builders' in kwargs: self.builderfactory = BuilderFactory(self, force_new_builders = True) else: self.builderfactory = BuilderFactory(self) #: just a counter for statistics self.nbr_of_steps = 0 try: self.name except AttributeError: self.name = 'Unknown' # fill ancestors and descendants self._get_parent() # and some stats self.max_descendant_depth = 0 for descendant in self.descendants: depth = len(descendant.ancestors) if self.max_descendant_depth < depth: self.max_descendant_depth = depth # debug inforamtion print ">>> DESCENDANTS OF %s (%s)" % (self ,self.name) for d in self.descendants: print " %s (%s)" % (str(d), d.name) print " with ancestors %s (%s)" % \ (str(d.paths), str(map(lambda x:zip(x), d.paths))) print "<<< DESCENDANTS OF %s (%s)" % (self ,self.name) def _get_parent(self): """ DSF of the ancestors giving back the complete path to the caller. """ parent = self ancestors = [] # fill the number of children that will have this element as an ancestor for cpt in self.vms: ancestors = [self] + ancestors # tricky! Make a copy of parent.vms instead of keeping a # pointer on it, to avoid modification when we pop it. children = [] + parent.vms # depth first traversal while children: parent = children.pop() children += parent.vms # get its ancestor direct_ancestor = ancestors.pop() # record the path to the root parent.ancestors = [direct_ancestor] + direct_ancestor.ancestors # fill the number of children that will have this element as an ancestor for cpt in parent.vms: ancestors = [parent] + ancestors if not parent.vms: # record the end of a path if parent in self.descendants: raise MyFactoryError( "We got a cycle in the tree of dependances! %s appears twice." % \ parent.name) self.descendants.add(parent) for cpt, descendant in enumerate(self.descendants): self.paths.append([descendant] + descendant.ancestors) if len(self.paths) == 0: self.paths = [[self]] print "MY PATH IS: (%s)" % self print " %s " % self.paths def set_factory(self, factory): self._factory = factory def get_factory(self): return self._factory factory = property(fget = get_factory, fset = set_factory) def get_current_builder(self): builders = [] for b in self.builderfactory.get_builders(): builders.append(b) return builders[-1] def start(self): """ This is the heart of the program. It creates the precise sequence of steps which enable: 1. to start the underlying vm(s) if necessary; 2. to install necessary paquages; 3. to take the snapshot at the end of the setup if the underlying vm permit it. The steps are added to the factory by L{addStep}. It B{HAS TO BE} started from the last element in the chain, not a sub-vm. It supports several root elements. When called, it returns the factories to the user, who can then add steps to each. The factories is a hash of root composed of a hash of names/factory elements. """ # TODO: should return a object to facilitate adding step: # check puppet/master.cfg print ">>> ROOTS: (%s)" % self for root in self.builderfactory.roots(): print " roots are %s (%s)" % (root, root.name) print "<<< ROOTS:" if self.builderfactory.has_started(): raise MyFactoryError("Start has to be called only one time.") if self not in self.builderfactory.roots(): raise MyFactoryError( "Must be called from the last element in the chain \"%s\"not %s" % \ ('or '.join( map(lambda x: str(x) + ' ', self.builderfactory.roots())), str(self))) # We are good to go. factories = {} nbr_of_chains = 0 nbr_of_elements = 0 nbr_of_steps = 0 for quick in [False, True]: for root in self.builderfactory.roots(): print 'ROOT: %s (%s)' % (root, str(quick)) root_name = root.name if quick: root_name += '_quick' # TODO: support multi-vms definitions print "STARTING FROM %s" % root for one_path in root.paths: b = ThisBuilder(root = root, path = one_path, quick = quick) self.builderfactory.add_builder(b) path = b.get_path() current_factory_name = b.get_factory_name() current_factory = b.get_factory() b.update_vm() b.update_factory() # now we can start, and the behaviour will be ok. snapper = None if path[0].can_snap: snapper = path[0] if not quick and snapper: snapper.addSetPropertyTF( command = snapper.commands.snap_exists(root.name), property = root.name) nbr_of_chains += 1 level = 1 last = len(path) for element in path: print " ELEMENT: %s" % element.name nbr_of_elements += 1 for command in element.pre_start_hook: command() if not quick: # TODO: should install heavy, non changing # stuff. All those commands should be # idempotent, ie, doing it over an # existing installation should work # (better still, it shld avoid to install # them again altogther with a embeded # test.) element.install_packages() if element.is_vm: element.install_vm() if element.can_snap and level == 0: element.install_snap() if element.is_vm: element.start_vm() # should install lightweight stuff and a # snap should be done before this one. # Then we can delete the last snap without # having to reinstall the heavy stuff # again. for command in element.post_start_hook: command() if level == last and snapper: # If I'm the caller (last step) and I've got a snapper, so # revert to it. print "DEBUG: REVERTING TO %s for %s" % (element.name, element) snapper.addRevertToSnap(element.name, assume_exists = True) if not quick and snapper: # Take the snap if necessary at the end of the setup snapper.addTakeSnap(element.name) # stats if snapper != element: nbr_of_steps += element.nbr_of_steps element.nbr_of_steps = 0 level += 1 if snapper: nbr_of_steps += snapper.nbr_of_steps snapper.nbr_of_steps = 0 self.builderfactory.save_steps(b) print "CREATED %d chains with %d elements composed of %d steps" % \ (nbr_of_chains, nbr_of_elements, nbr_of_steps) self.stats = {'nbr_of_chains' : nbr_of_chains, 'nbr_of_elements' : nbr_of_elements, 'nbr_of_steps' : nbr_of_steps } import pprint pp = pprint.PrettyPrinter() pp.pprint(factories) return self.builderfactory # #: This will be the vm (if available) that will take the snap at the end. # snapper = None # # if self.run_on_vm: # # recurse # snapper = self.vm._start(quick = quick, first = False) # # if len(self.fathers) > 0 and not self.from_inside: # # The user call a underlying vm directly. Forbid it. # father = self.fathers[-1].name # raise MyFactoryError("I must be start from the end of the chain: %s" % father) # # if self.is_vm and self.can_snap and not self.run_on_vm: # # I the last vm and I can snap, so I'm the snapper. # snapper = self # # if snapper: # # got a snapper # if first and not quick: # # I'm in the first level caller and I'm not in a hury. # snapper.addSetPropertyTF( # command = snapper.commands.snap_exists(self.name), # property = self.name) # # for command in self.pre_start_hook: # command() # # if not quick: # self.install_packages() # if self.is_vm: # self.install_vm() # # if self.can_snap: # self.install_snap() # # if self.is_vm: # self.start_vm() # # for command in self.post_start_hook: # command() # # if first and snapper: # # If I'm the caller (last step) and I've got a snapper, so # # revert to it. # snapper.addRevertToSnap(self.name, assume_exists = True) # # if not quick and snapper: # # Take the snap if necessary at the end of the setup # snapper.addTakeSnap(self.name) # # return snapper def addStep(self, step): """ Delegator to the buildbot factory module. All sub-class should use this X{addStep} and not X{self.factory.addStep} directly. """ self.nbr_of_steps += 1 print " STEP: %s (%s)" % (step, self.factory) self.get_current_builder().addStep(step) # keep a copy of the steps for debug self.factory.addStep(step) def addDownloadFile(self, src_file, dst_file, workdir = '/', as_root = False): """ Download a nfile from the master to the slave. It will properly handle the copy to all the underlying vm. """ self._addDownloadFile(src_file, dst_file, workdir, as_root) def _addDownloadFile(self, src_file, dst_file, workdir = '/', as_root = False, steps = [], nbr_steps = 0): """ This is where the real meat is. Recurse all the way back to the root vm and depile the stack of intermediary vm asking them to upload the file to its final destination. The actual Downloading from one vm to another is delegated to each vm class with the L{addDownloadFileFromSocle} method. """ this_step = [self] + steps if self.is_vm: nbr_steps += 1 if self.run_on_vm: self.vm._addDownloadFile(src_file = src_file, dst_file = dst_file, workdir = workdir, steps = this_step, nbr_steps = nbr_steps, as_root = as_root) else: # "root" vm last = len(this_step) dst_file_rel = dst_file workdir_rel = workdir if os.path.isabs(workdir): workdir_rel = os.path.relpath(workdir,'/') if os.path.isabs(dst_file): dst_file_rel = os.path.relpath(dst_file,'/') # workdir and all are for the final vm, here we assure # that the final file will be in 'build/<workdir>/<dst_file> dst_file_rel = os.path.join(workdir_rel, dst_file_rel) # when only one step the hardware node is the final destination. # if last == 1: # dst_file_rel = os.path.join('/', dst_file_rel) if not os.path.exists(src_file): self.addStep(StringDownload(s = src_file, slavedest = dst_file_rel)) else: self.addStep(FileDownload(mastersrc = src_file, slavedest = dst_file_rel)) # now I deleguate the "upload" to each vm. iter_file = False cmpt = 1 for step in this_step: if not step.is_vm: # only work with vm cmpt += 1 continue dest_dir_tmp = dst_file if os.path.isabs(dst_file): dst_file_tmp = os.path.relpath(dst_file,'/') final_dst = os.path.join('/tmp', dst_file_tmp) # final destination as_root_final = False if cmpt == nbr_steps: final_dst = dst_file as_root_final = as_root if not iter_file: # root vm iter_file = step.addDownloadFileFromSocle( src_file = dst_file_rel, dst_file = final_dst, workdir = workdir, as_root = as_root_final) else: # itermediary vm iter_file = step.addDownloadFileFromSocle( src_file = iter_file, dst_file = final_dst, workdir = workdir, as_root = as_root_final) cmpt += 1 def addDownloadGitDir(self, repo_url = '', dest_dir = '', mode='copy', use_new = False, as_root = False, **kwargs): """ Download a dir a Git(+patch) repository. Support any number of underlying VM """ self._addDownloadGitDir(repo_url, dest_dir, mode, use_new, as_root, **kwargs) def _addDownloadGitDir(self, repo_url = '', dest_dir = '', mode='copy', use_new = False, as_root = False, steps = [], nbr_steps = 0, **kwargs): this_step = [self] + steps if self.is_vm: nbr_steps += 1 if self.run_on_vm: self.vm._addDownloadGitDir(repo_url = repo_url, dest_dir = dest_dir, mode = mode, use_new = use_new, steps = this_step, nbr_steps = nbr_steps, as_root = as_root, **kwargs) else: # the Try module does not work with the new Git module 0.8.5 if use_new: self.addStep(Git(repourl=repo_url, mode='full', method='clean', **kwargs)) else: self.addStep(GitOld(repourl=repo_url, mode=mode, **kwargs)) iter_dir = False cmpt = 1 for step in this_step: if not step.is_vm: cmpt += 1 continue dest_dir_tmp = dest_dir if os.path.isabs(dest_dir): dest_dir_tmp = os.path.relpath(dest_dir,'/') final_dst = os.path.join('/tmp', dest_dir_tmp) if cmpt == nbr_steps: # != of last, it's the nbr_of_ VM steps final_dst = dest_dir if not iter_dir: iter_dir = step.addDownloadDirectory('../build', final_dst, as_root) else: iter_dir = step.addDownloadDirectory(iter_dir, final_dst, as_root) cmpt += 1 def addMakeDirectory(self, directory, as_root = False, **kwargs): self.addStep(ShellCommand( command = self.commands.simple('mkdir','-p', directory), description = 'Creating ' + directory, haltOnFailure = True, descriptionDone = directory, **kwargs)) def addCpDirectory(self, src_dir, dst_dir, as_root = False, **kwargs): """ Copy a directory, ensure it's pristine new each time. """ cmd = ['bash', '-c', ' '.join(['rm','-rf', dst_dir, '&&', 'mkdir', '-p', dst_dir, '&&', 'cp', '-af',src_dir + '/*', dst_dir])] if as_root: cmd = ['sudo'] + cmd self.addStep(ShellCommand( command = self.commands.simple(cmd), haltOnFailure = True, description = 'Copying Dir', descriptionDone = 'Dir Copied', **kwargs)) def addCpFile(self, src_file, dst_file, as_root = False, **kwargs): dst_dir = os.path.dirname(dst_file) cmd = ['bash', '-c', ' '.join( ['mkdir', '-p', dst_dir, '&&', 'cp', '-vf', src_file, dst_file])] if as_root: cmd = ['sudo'] + cmd self.addStep(ShellCommand( haltOnFailure = True, command = self.commands.simple(cmd), description = 'Copying file', descriptionDone = 'File copied', **kwargs)) def addSetProperty(self, command = [], description = 'Setting Property', **kwargs): self.addStep(shell.SetProperty( command=self.commands.simple(command), description = description, **kwargs)) def addSetPropertyTF(self, command = [], description = 'Setting Property', descriptionDone = False, property = '', **kwargs): """ Convenient method to set a property to TRUE or FALSE depending on the status of a shell command """ if not descriptionDone: descriptionDone = property cmd = self.commands.simple(['bash', '-c',' '.join(command + [' && echo TRUE'])]) cmd[-1] += ' || echo FALSE' self.addStep(shell.SetProperty( command = cmd, description = description, descriptionDone = descriptionDone, property = property, **kwargs)) def addShellCmd(self, command=[], workdir = None, **kwargs): """ Main method to add a shell command. It will properly handle the underlying vm and execute the code in the final one. It delegates the command creation (which handle the underlying vm) to the X{simple} methode of the X{<User>Command} module provided. """ cmd = self.commands.simple(cmd = command, workdir = workdir) self.addShellCmdBasic(command = cmd, **kwargs) def addShellCmdBasic(self, command=[], description = 'Running', descriptionDone='Done', timeout=1200, doStepIf=True, **kwargs): """ The same as L{addShellCmd} but use the X{basic} method which should not any decoration to the command, but the underlying vm handle code. """ self.addStep(ShellCommand( haltOnFailure = True, command = command, description = description, descriptionDone = descriptionDone, timeout = timeout, doStepIf = doStepIf, **kwargs )) def addCommandIf(self, command = [], **kwargs): self.addCommandIfRaw(command = self.commands.simple(command), **kwargs) def addCommandIfBasic(self, command = [], **kwargs): self.addCommandIfRaw(command = self.commands.basic(command), **kwargs) def addCommandIfRaw(self, command = [], true_or_false = 'FALSE', property_name = '', doStepIf = False, assume = 'FALSE', **kwargs): """ Add a step if a certain X{Property} is met. Designed to work with L{addSetPropertyTF}. It can be configured to assume TRUE or FALSE if the property is missing. It will test against TRUE or FALSE as required. By default, assumes FALSE and do the command only if FALSE. """ description = 'Running Maybe' descriptionDone = 'Done Maybe' if 'description' in kwargs: description = kwargs['description'] if 'descriptionDone' in kwargs: descriptionDone = kwargs['descriptionDone'] # this is the test doStepIf = lambda s, name=property_name,tf=true_or_false: \ s.getProperty(name, assume) == tf self.addStep(ShellCommand( haltOnFailure = True, command = command, doStepIf = doStepIf, **kwargs)) def add_to_post_start_hook(self, step): """ Hook to enable caller to stuck steps before the installation. """ self.post_start_hook.append(step) def add_to_pre_start_hook(self, function): """ Hook to enable caller to stuck steps after the installation. """ self.pre_start_hook.append(function)