示例#1
0
 def precheck(self):
     """
     Checks to see if the preconditions for this HappyJob have been met.
     If so, returns true, and the HappyJob is executed.
     
     It is expected that this method will be overidden to implement custom
     checks in a subclass (use lamdbas instead?)
     
     @return: STATUS_READY if the HappyJob's preconditions are met and the job can be run.
              STATUS_WAIT if the job is not ready to be run
              STATUS_SKIP if the job has already been run
              STATUS_ERROR if we should abort
     """
     if (not dfs.exists(self.outputpath)):
         logger.debug("precheck(%s): outputpath %s does not exist, ready to run." 
                      % (self, self.outputpath))
         return 'ready'
     inTSs = [dfs.modtime(file) for file in self.inputpaths]
     outTS = dfs.modtime(self.outputpath)
     newer = reduce(lambda x,y: x or y, [(inTS>outTS) for inTS in inTSs])
     logger.debug("Input timestamps: %s" % inTSs)
     logger.debug("Output timestamp: %s" % outTS)
     if newer:
         logger.debug("At least one input file is newer than outputfile, ready to run.")
         dfs.delete(self.outputpath)
         return 'ready'
     else:
         logger.debug("All input files are newer than outputfile, skipping.")
         return 'skip'
示例#2
0
 def linkNodes(self, workingDir=None):
     """
     Assures that every parent/child pair have a matching file in
     their inFile / outFile lists.  Creates files if necessary. 
     
     @param workingDir: the directory to create temp files in. 
     """
     if workingDir:
         logger.info("Linking nodes, using workingDir = %s" % (workingDir))    
         if dfs.exists(workingDir):
             fs = dfs.fileStatus(workingDir)
             if not fs.isDir():
                 raise FlowException, "%s is a file, not a directory." % (workingDir)
         else:
             logger.info("Creating working directory %s." % (workingDir))    
             # dfs.mkdir(workingDir)
     stack = self.sources()
     for source in stack:
         if ((not source.inputpaths) or len(source.inputpaths)<1):
             raise FlowException, "Source node %s has no inputpaths defined." % source
     while stack:
         node = stack.pop(0)
         if node.outputpath:
             logger.trace("linkNodes(): %s has an outputpath '%s'.  Using it." % (node, node.outputpath))
             filename = node.outputpath
         else:
             filename = "tmp.%s" % (node.name)
             if workingDir:
                 filename = "%s/%s" % (workingDir, filename)
             logger.trace("linkNodes(): Created temp outfile '%s' for %s." % (filename, node))
             node.outputpath = filename
         for child in node.children():
             if ((not child.inputpaths) or 
                (len(set(node.outputpath) & set(child.inputpaths)) == 0)):
                 logger.debug("linkNodes(): Linked %s and %s with file '%s'." % (node, child, filename))
                 child.inputpaths = castList(child.inputpaths) + [filename]
             stack.append(child)
         logger.debug("%s has inputs %s and outputs %s" % (node, node.inputpaths, node.outputpath))