def precheck(self):
"""
Checks to see if the preconditions for this HappyJob have been met.
If so, returns true, and the HappyJob is executed.
It is expected that this method will be overidden to implement custom
checks in a subclass (use lamdbas instead?)
@return: STATUS_READY if the HappyJob's preconditions are met and the job can be run.
STATUS_WAIT if the job is not ready to be run
STATUS_SKIP if the job has already been run
STATUS_ERROR if we should abort
"""
if (not dfs.exists(self.outputpath)):
logger.debug("precheck(%s): outputpath %s does not exist, ready to run."
% (self, self.outputpath))
return 'ready'
inTSs = [dfs.modtime(file) for file in self.inputpaths]
outTS = dfs.modtime(self.outputpath)
newer = reduce(lambda x,y: x or y, [(inTS>outTS) for inTS in inTSs])
logger.debug("Input timestamps: %s" % inTSs)
logger.debug("Output timestamp: %s" % outTS)
if newer:
logger.debug("At least one input file is newer than outputfile, ready to run.")
dfs.delete(self.outputpath)
return 'ready'
else:
logger.debug("All input files are newer than outputfile, skipping.")
return 'skip'
def linkNodes(self, workingDir=None):
"""
Assures that every parent/child pair have a matching file in
their inFile / outFile lists. Creates files if necessary.
@param workingDir: the directory to create temp files in.
"""
if workingDir:
logger.info("Linking nodes, using workingDir = %s" % (workingDir))
if dfs.exists(workingDir):
fs = dfs.fileStatus(workingDir)
if not fs.isDir():
raise FlowException, "%s is a file, not a directory." % (workingDir)
else:
logger.info("Creating working directory %s." % (workingDir))
# dfs.mkdir(workingDir)
stack = self.sources()
for source in stack:
if ((not source.inputpaths) or len(source.inputpaths)<1):
raise FlowException, "Source node %s has no inputpaths defined." % source
while stack:
node = stack.pop(0)
if node.outputpath:
logger.trace("linkNodes(): %s has an outputpath '%s'. Using it." % (node, node.outputpath))
filename = node.outputpath
else:
filename = "tmp.%s" % (node.name)
if workingDir:
filename = "%s/%s" % (workingDir, filename)
logger.trace("linkNodes(): Created temp outfile '%s' for %s." % (filename, node))
node.outputpath = filename
for child in node.children():
if ((not child.inputpaths) or
(len(set(node.outputpath) & set(child.inputpaths)) == 0)):
logger.debug("linkNodes(): Linked %s and %s with file '%s'." % (node, child, filename))
child.inputpaths = castList(child.inputpaths) + [filename]
stack.append(child)
logger.debug("%s has inputs %s and outputs %s" % (node, node.inputpaths, node.outputpath))