def __callDone(self, fil):
"call done function, if exception, set firstEx if it is None"
try:
fil.done()
except Exception as ex:
ex = ExRunException("Exception on file: " + str(fil), cause=ex)
self.verb.pr(Verb.error, str(ex) + "\n" + ex.format())
return ex
return None
def __callDone(self, fil):
"call done function, if exception, set firstEx if it is None"
try:
fil.done()
except Exception as ex:
ex = ExRunException("Exception on file: " + str(fil), cause=ex)
self.verb.pr(Verb.error, str(ex)+"\n"+ex.format())
return ex
return None
def execute(self):
"execute the rule"
self.verb.enter()
try:
self.run()
except Exception as ex:
ex = ExRunException("Exception executing rule: " + str(self), cause=ex)
self.verb.pr(Verb.error, str(ex)+"\n"+ex.format())
raise
finally:
self.verb.leave()
def finishSucceed(self):
"finish production with atomic install of new output file as actual file"
self.done()
if self.installed:
raise ExRunException("output file already installed: " + self.path)
if self.outPath == None:
raise ExRunException("getOutPath() never called for: " + self.path)
if not os.path.exists(self.outPath):
raise ExRunException("output file as not created: " + self.outPath)
fileOps.atomicInstall(self.outPath, self.path)
self.installed = True
self.outPath = None
def linkProducedBy(self, producedBy):
"set producedBy link, and link back to this object"
if self.producedBy != None:
raise ExRunException("Production " + str(self) +
" producedBy link already set")
if not isinstance(producedBy, Rule):
raise ExRunException(
"Production " + str(self) +
" producedBy can only be linked to a Rule, attempt to link to: "
+ str(producedBy))
self.producedBy = producedBy
producedBy.produces.add(self)
def __connectivityCheck(self):
"""check for cycles and a full connected graph"""
reachable = set()
for entry in self.__getCheckEntries():
self.__cycleCheck(entry, reachable)
if len(reachable) < len(self.nodes):
raise ExRunException("Nodes not reachable from any Target: " +
Node.joinNames(self.nodes - reachable))
elif len(reachable) > len(self.nodes):
raise ExRunException(
"Invalid graph, nodes not added through API: " +
Node.joinNames(reachable - self.nodes))
def __postEvalRuleCheck(self):
"check that a rule build it's productions"
for p in self.rule.produces:
if p.getLocalTime() == None:
if (p.producedBy == None):
raise ExRunException("No rule to build production: " +
str(p))
else:
raise ExRunException("Production not built: " + str(p))
if self.rule.isOutdated():
raise ExRunException("rule didn't update all productions: " +
Node.joinNames(self.rule.getOutdated()))
def execute(self):
"execute the rule"
self.verb.enter()
try:
self.run()
except Exception as ex:
ex = ExRunException("Exception executing rule: " + str(self),
cause=ex)
self.verb.pr(Verb.error, str(ex) + "\n" + ex.format())
raise
finally:
self.verb.leave()
def call(self, cmd):
"run a commands with optional tracing"
firstEx = None
try:
try:
cmd.call(self.verb)
except Exception as ex:
ex = ExRunException("Exception running command: " + str(cmd), cause=ex)
self.verb.pr(Verb.error, str(ex)+"\n"+ex.format())
firstEx = ex
finally:
firstEx = self.__closeFiles(self.requires, firstEx)
firstEx = self.__closeFiles(self.produces, firstEx)
if firstEx != None:
raise firstEx
def call(self, cmd):
"run a commands with optional tracing"
firstEx = None
try:
try:
cmd.call(self.verb)
except Exception as ex:
ex = ExRunException("Exception running command: " + str(cmd),
cause=ex)
self.verb.pr(Verb.error, str(ex) + "\n" + ex.format())
firstEx = ex
finally:
firstEx = self.__closeFiles(self.requires, firstEx)
firstEx = self.__closeFiles(self.produces, firstEx)
if firstEx != None:
raise firstEx
def _transition(self, newState):
"transition to a new state, validating transition"
if newState not in _prodStateTbl[self.state]:
raise ExRunException("invalid Production transition: " +
str(self.state) + " to " + str(newState) +
" on " + self.name)
self.state = newState
def __reportExprErrors(self):
"final error reporting at the end of a run"
if self.verb.enabled(Verb.error):
for ex in self.errors:
self.__reportExprError(ex)
raise ExRunException("Experiment failed: " + str(len(self.errors)) +
" error(s) encountered")
def runCmds(self):
"run commands supplied in the constructor"
if self.cmds == None:
raise ExRunException(
"no commands specified and run() not overridden for CmdRule: "
+ self.name)
for cmd in self.cmds:
self.call(cmd)
def __entrySetup(self, defaultTargetName=None):
"setup entry points info"
self.entryProds = self.__findEntryProductions()
if (len(self.targets) == 0) and (len(self.entryProds) > 0):
if (defaultTargetName != None):
self.addTarget(Target(defaultTargetName, self.entryProds))
else:
raise ExRunException("graph does not contain any targets")
def getTargets(self, targetNames):
"get targets for given names"
targets = []
for tn in targetNames:
t = self.targetsByName.get(tn)
if t == None:
raise ExRunException("no Target named: " + tn)
targets.append(t)
return targets
def linkProduces(self, produces):
"""link in productions produced by this node. Can be single nodes, or
lists or sets of nodes, reverse links are created"""
for p in typeOps.mkiter(produces):
if not isinstance(p, Production):
raise ExRunException(
"Rule " + str(self) +
" produces can only be linked to a Production, attempt to link to: "
+ str(p))
p.linkProducedBy(self)
def linkRequires(self, requires):
"""link in productions or targets required by this node. Can be single nodes,
or lists or sets of nodes"""
for r in typeOps.mkiter(requires):
if not (isinstance(r, Production) or isinstance(r, Target)):
raise ExRunException(
"Target " + str(self) +
" requires can only be linked to a Production or a Target, attempt to link to: "
+ str(r))
self.requires.add(r)
def __addNode(self, desc, node, nodeList, nodeIdx):
"common operations on adding a node"
self.nodes.add(node)
node.graph = self
if node.name in nodeIdx:
raise ExRunException("duplicate " + desc + " name: " + node.name)
nodeList.add(node)
nodeIdx[node.name] = node
node.exrun = self.exrun
node.verb = self.verb
def linkRequires(self, requires):
"""link in productions required by this node. Can be single nodes, or
lists or sets of nodes, reverse links are created"""
for r in typeOps.mkiter(requires):
if not isinstance(r, Production):
raise ExRunException(
"Rule " + str(self) +
" requires can only be linked to a Production, attempt to link to: "
+ str(r))
r.linkRequiredBy(self)
def linkRequiredBy(self, requiredBy):
"""link to rules that require production, can be single rule, lists or
sets of rules, reverse links are created"""
for r in typeOps.mkiter(requiredBy):
if not isinstance(r, Rule):
raise ExRunException(
"Production " + str(self) +
" requiredBy can only be linked to a Rule, attempt to link to: "
+ str(r))
self.requiredBy.add(r)
r.requires.add(self)
def __evalRule(self):
"evaluate a rule"
assert(self.rule.state == RuleState.running)
self.verb.enter(Verb.trace, "eval rule:", self.rule)
try:
self.__preEvalRuleCheck()
self.verb.pr(Verb.details, "run:", self.rule)
self.rule.execute()
self.__finishSucceed()
self.__postEvalRuleCheck()
self.rule.setState(RuleState.ok)
self.verb.leave(Verb.trace, "done rule:", self.rule)
except Exception as ex:
ex = ExRunException("rule error: "+str(self.rule), cause=ex)
self.verb.pr((Verb.trace,Verb.error), str(ex))
self.verb.pr(Verb.error, ex.format())
self.rule.setState(RuleState.failed)
self.__finishFail()
self.verb.leave((Verb.trace,Verb.error), "failed rule:", self.rule)
raise ex
def __finishFail(self):
"""finish up finish up requires/produces on failure, will log errors,
but not fail so original error is not lost"""
for p in self.rule.produces:
try:
p.finishFail()
except Exception as ex:
# FIXME: count these errors
ex = ExRunException("Error in Production.finishFail() for "+p.name,
cause=ex)
self.verb.prall(str(ex))
self.verb.pr(Verb.error, ex.format())
for r in self.rule.requires:
try:
r.finishRequire()
except Exception as ex:
ex = ExRunException("Error in Production.finishRequire() for "+name,
cause=ex)
self.verb.prall(str(ex))
self.verb.pr(Verb.error, +ex.format())
def __addCmdArgFiles(self, cmd, requires, produces):
"""scan a command's arguments for FileIn and FileOut object and add these to
requires or produces"""
for a in cmd:
if isinstance(a, FileIn):
requires.add(a.file)
elif isinstance(a, FileOut):
produces.add(a.file)
elif isinstance(a, File):
raise ExRunException(
"can't use File object in command argument, use FileIn() or FileOut() to generate a reference object"
)
def __evalRule(self):
"evaluate a rule"
assert (self.rule.state == RuleState.running)
self.verb.enter(Verb.trace, "eval rule:", self.rule)
try:
self.__preEvalRuleCheck()
self.verb.pr(Verb.details, "run:", self.rule)
self.rule.execute()
self.__finishSucceed()
self.__postEvalRuleCheck()
self.rule.setState(RuleState.ok)
self.verb.leave(Verb.trace, "done rule:", self.rule)
except Exception as ex:
ex = ExRunException("rule error: " + str(self.rule), cause=ex)
self.verb.pr((Verb.trace, Verb.error), str(ex))
self.verb.pr(Verb.error, ex.format())
self.rule.setState(RuleState.failed)
self.__finishFail()
self.verb.leave((Verb.trace, Verb.error), "failed rule:",
self.rule)
raise ex
def getOutPath(self, autoCompress=True):
"""Get the output name for the file, which is newPath until the rule
terminates. This will also create the output directory for the file,
if it does not exist. One wants to use FileOut() to define
a command argument. This should not be used to get the path to a file
to be opened in the ExRun process, use openOut() instead."""
if self.installed:
raise ExRunException("output file already installed: " + self.path)
if self.outPath == None:
fileOps.ensureFileDir(self.path)
self.outPath = self.exrun.getAtomicPath(self.path)
return self.outPath
def __stateFromRequiresState(self, reqStates):
"compute state from require states, return None if times must be compared"
if ProdState.unknown in reqStates:
raise ExRunException(
"BUG: can't update rule from production in state: " +
str(ProdState.unknown))
elif ProdState.failed in reqStates:
return RuleState.failed
elif (ProdState.bad in reqStates) or (ProdState.blocked in reqStates):
return RuleState.blocked
else:
return None
def __stateFromRuleState(self, rs):
"get state give state of producing rule"
if rs == RuleState.outdated:
return ProdState.outdated
elif rs == RuleState.running:
return ProdState.outdated
elif rs == RuleState.ok:
return ProdState.current
elif rs == RuleState.failed:
return ProdState.failed
elif rs == RuleState.blocked:
return ProdState.blocked
else:
raise ExRunException(
"BUG: can't update production from rule in state: " + str(rs))
def setState(self, state):
"recursively set the state or this rule and parents"
with self.graph.lock:
self.verb.enter(Verb.debug, "Rule.setState: " + str(self.state),
"=>", str(state), self.name) # FIXME
self._transition(state)
if state == RuleState.running:
pstate = None
elif state == RuleState.ok:
pstate = ProdState.current
elif state == RuleState.failed:
pstate = ProdState.failed
elif state == RuleState.blocked:
pstate = ProdState.blocked
else:
raise ExRunException("BUG: Rule.setState() invalid state:" +
str(state))
if pstate != None:
for p in self.produces:
p.setState(pstate)
self.verb.leave() # FIXME
def __finishFail(self):
"""finish up finish up requires/produces on failure, will log errors,
but not fail so original error is not lost"""
for p in self.rule.produces:
try:
p.finishFail()
except Exception as ex:
# FIXME: count these errors
ex = ExRunException("Error in Production.finishFail() for " +
p.name,
cause=ex)
self.verb.prall(str(ex))
self.verb.pr(Verb.error, ex.format())
for r in self.rule.requires:
try:
r.finishRequire()
except Exception as ex:
ex = ExRunException(
"Error in Production.finishRequire() for " + name,
cause=ex)
self.verb.prall(str(ex))
self.verb.pr(Verb.error, +ex.format())
def execute(self):
raise ExRunException("rule should never have been run")
def __init__(self, cycle):
desc = []
for n in cycle:
desc.append(" " + str(n) + " ->")
ExRunException.__init__(self, "cycle detected:\n" + "\n".join(desc))
def __stateCheck(self):
"check for various things after state initialized"
bad = self.__getBadProds()
if bad != None:
raise ExRunException("No rule to build production(s): " +
Node.joinNames(bad))
def __preEvalRuleCheck(self):
"Sanity check before a rule is run"
for r in self.rule.requires:
if r.getLocalTime() == None:
raise ExRunException("require should have been built:" +
str(r))
def getLocalTime(self):
"""Get the modification time of this Production as a floating point number.
If the object does not exist, return None. This is not recurisve (local), it is
simply the time associated with the production. Must be thread-safe."""
raise ExRunException("getLocalTime() not implemented for Production " +
str(type(self)))
def execute(self):
"""Execute the rule, must be implemented by derived class. ExRun will
call the approriate finish*() methods on the produces and requires,
they should not be run by the derived rule class"""
raise ExRunException("execute() not implemented for " +
str(type(self)))
def __init__(self, cycle):
desc = []
for n in cycle:
desc.append(" " + str(n) + " ->")
ExRunException.__init__(self, "cycle detected:\n" + "\n".join(desc))