def __init__(self, graph, mods, folder, doc):
untrustme1 = list()
untrustme2 = list()
dbgstr('Initializing protocol with root: ' + folder)
self._metafolder = folder
self._rootdir = os.getcwd()
self._resmap = dict()
self._doc = doc
self._graphres = resource('graph', os.path.join(folder,'graph.grp'))
untrustme1 = self._updateGraph(graph)
for res in self._getResNames():
newres = resource(res, folder+'/'+res+'.res')
#print res, newres
self._addResource(res, newres)
for mod in self._getNodeNames():
newres = resource(mod, folder+'/'+mod+'.mod')
self._modules[mod] = newres
untrustme2 = self._updateModules(mods)
for mod in untrustme1:
self.untrust(mod)
for mod in untrustme2:
self.untrust(mod)
def _manageGraphChange(self, newGraph):
if newGraph==None:
return
#checking for change in graph structure
#a node is flagged as to-be-untrustest if its inputs have changed
#not untrusted on the fly because of dependancies with untrusts
#from source code changes
oldg = self._graphres.getValue()
untrustme = list()
# import pdb; pdb.set_trace()
for node in newGraph.getNodes():
#check wether node is (not) new
if node in oldg.getNodes():
#check if node inputs have changed
innodes1 = newGraph.getInNodes(node)
innodes2 = oldg.getInNodes(node)
for inNode in innodes1:
if not inNode in innodes2:
dbgstr('Inputs to ' + node + ' have changed, untrusting it.')
untrustme.append(node)
#check if node properties have changed
#cycling through attrib names of node 'node'
# for attrib in [a[1] for a in newGraph._nodeattribs if a[0]==node]:
# if newGraph.getAttrib(node, attrib) != oldg.getAttrib(node, attrib):
# dbgstr('Node attribs of ' + node +
# ' have changed, untrusting it.')
# untrustme.append(node)
return untrustme
def _runNodes(self, nodes, parallel = False):
if parallel and len(nodes)>1:
from multiprocessing import Process, Queue
results = Queue()
tasks = list()
for node in nodes:
nodeparams = self._getNodePar(node)
tasks.append(
Process(target=self._callModPar, args=(node, nodeparams, results,))
)
for task in tasks: task.start()
for task in tasks: task.join()
for task in tasks:
taskres = results.get()
resname = self._buildResName(taskres[0], None, taskres[1])
dbgstr('Requesting add resource: ' + taskres[0], 2)
t=taskres[2]
self._newResource(resname, taskres[1], t)
else:
for node in nodes:
nodeparams = self._getNodePar(node)
taskres = self._callMod(node, nodeparams)
resname = self._buildResName(taskres[1], None, taskres[0])
dbgstr('Requesting add resource: ' + resname, 2)
t=taskres[2]
self._newResource(resname, taskres[0], t)
def undump(self, filtername, verbose = True):
"""Undumps a resource (removes cached version from disk)."""
if type(filtername) != str:
filtername = filtername.__name__
if self._isDumped(filtername):
if verbose:
dbgstr('Undumping resource: ' + str(filtername))
self._getResource(filtername).clearDump()
def _loadMods(self):
if os.path.exists(self.modsToPath()):
mods = pickle.load(open(self.modsToPath(), 'r'))
else:
mods = dict()
dbgstr('Found on disk: ' + str(mods.keys()), 2)
return mods
def clear(self, filtername, verbose = True):
"""Clears a resource."""
if type(filtername) != str:
filtername = filtername.__name__
if self._isAvailable(filtername):
if verbose:
dbgstr('Clearing resource: ' + str(filtername))
self._resmap[filtername].clear()
def _setMod(self, modname, mod):
haschanged = self._modules[modname].changed()
if haschanged:
if type(mod) == File:
mod.update()
self._modules[modname].setValue(mod.getValue())
self._modules[modname].updateFingerprint()
self.untrust(modname)
dependents = self._getOutNodesRecursive(modname)
if dependents != []:
dbgstr('These nodes are dependent: '+str(dependents), 0)
def trust(self, who, what):
"""Assign a resource to a filter without invalidating dependent resources."""
if type(who) != str:
who = who.__name__
if type(who) == str:
self._setMod(who,what)
dbgstr('I\'m assuming that using ' + str(what) + ' for ' + who + ' won\'t have consequences on other nodes.', 0)
elif type(who) == tuple:
self._addResource(who,what)
dbgstr('I\'m assuming that using your object of type ' + str(type(what)) + ' for ' + str(who) + ' won\'t have consequences on other nodes.', 0)
def _update(self, graph, mods):
untrustme1 = list()
untrustme2 = list()
dbgstr('New graph is: ' + str(graph), 3)
dbgstr('New mods are: ' + str(mods), 3)
untrustme1 = self._updateGraph(graph)
untrustme2 = self._updateModules(mods)
for mod in untrustme1:
self.untrust(mod)
for mod in untrustme2:
self.untrust(mod)
def run(self, parallel = False):
"""Provides all leaf (final) resources."""
res = None
allok = True
l = self._getLeaves()
for leaf in l:
resname = leaf
if not self._isAvailable(resname)and not self._isDumped(resname):
allok = False
if allok:
dbgstr('Nothing to be done. Zzz...')
else:
res = self.provide(l, parallel)
return res
def _ChangeME_resToPath(self, res):
if self._getGraph().getAttrib(res[0], 'hashout'):
if res[0] == None: first_part = ''
else: first_part = res[0]
if res[1] == None: second_part = ''
else: second_part = res[1]
if first_part != '' and second_part !='':
mid_part = 'TO'
else: mid_part = ''
fname = first_part + mid_part + second_part + '.dmp'
else:
fname = res[0]
dbgstr('Dump file for resource ' + str(res) + ' is ' + self._metafolder + '/' + fname, 2)
return self._metafolder + '/' + fname
def _updateModules(self, mods):
untrustme = list()
for modname in mods:
if modname in self._modules.keys():
self._modules[modname].setValue(mods[modname])
if self._modules[modname].changed():
untrustme.append(modname)
#self.untrust(self._modules[modname].name())
self._modhelp[modname] = inspect.getdoc(self._modules[modname].getValue())
self._modules[modname].update()
else:
dbgstr('New module: ' + modname)
self._modules[modname] = resource(modname,
self._metafolder+'/'+modname+'.dmp' )
self._addResource(modname, self._modules[modname])
self._modules[modname].setValue(mods[modname])
self._modules[modname].update()
return untrustme
def _loadResource(self, res):
dbgstr('Getting resource ' + str(res) + ' from disk.', 2)
if self._isDumped(res):
dbgstr('Resource ' + str(self._resmap[res]) + ' found in: ' + self._resmap[res].getDumpPath() ,2)
return pickle.load(open(self._resmap[res].getDumpPath(), 'rb'))
else:
dbgstr('Resource ' + str(res) + ' not found on disk! I\'ve been looking for: ' + self._resmap[res].getDumpPath())
def _processRawRes(self, node, rawres):
# # elif type(rawres)==tuple:
# # dbgstr('Raw list are packed in a tuple.')
# #
# # if len(rawres != len(self._getGraph()[node])):
# # raise NameError('When a module returns a tuple, it''s length must be equal to the number of the module''s outputs.')
# #
# # for idx, outnode in enumerate(self._getGraph()[node]):
# # newresname = self._buildResName(node, outnode, rawres)
# # dbgstr('Requesting add resource: ' + str(newresname))
# # if self._isFileMod(node):
# # self._placeFileRes(rawres)
# # self._newResource(newresname, self._updateFilePath(rawres[idx]))
# # else:
# # self._newResource(newresname, rawres[idx])
# #
# # else:
# if self._isFileMod(node):
# if type(rawres)==tuple or type(rawres)==list:
# for rawresi in rawres:
# newresname = self._buildResName(node, None, rawresi)
# dbgstr('Requesting add resource: ' + node, 2)
# self._placeFileRes(rawresi)
# self._newResource(newresname, self._updateFilePath(rawresi))
# else:
# newresname = self._buildResName(node, None, rawres)
# dbgstr('Requesting add resource: ' + node, 2)
# self._placeFileRes(rawres)
# self._newResource(newresname, self._updateFilePath(rawres))
# else:
# newresname = self._buildResName(node, None, rawres)
# dbgstr('Requesting add resource: ' + node, 2)
# self._newResource(newresname, rawres)
newresname = self._buildResName(node, None, rawres)
dbgstr('Requesting add resource: ' + node, 2)
self._newResource(newresname, rawres)
def _isAvailable(self, resname):
dbgstr('Checking resource: ' + str(resname), 3)
if self._getResource(resname).isAvailable():
dbgstr('Available: ' + str(resname), 3)
return True
dbgstr('Unavailable: ' + str(resname), 3)
return False
def provide(self, res, parallel=False):
t = time.time()
if not type(res) == list:
temp = list()
temp.append(res)
res = temp
for i,r in enumerate(res):
if type(res[i]) != str:
res[i] = res[i].__name__
D = self._findDependancies(res)
D.update(res)
states = self._getBestStates(D)
if len(states[1]) > 0:
dbgstr('The following resources will be loaded from disk: '+
self._prettyPrint(states[1]))
for node in states[1]:
self._provideResource(node)
if len(states[2]) > 0:
dbgstr('The following resources will need production: '+
self._prettyPrint(states[2]))
needbuild = states[2]
while True:
canrun = self._findRunnable(needbuild)
if len(canrun) > 1 and parallel:
dbgstr('The following nodes can run in parallel: '+
self._prettyPrint(canrun))
self._runNodes(canrun, parallel)
needbuild=[x for x in needbuild if not x in canrun]
if len(canrun)==0:
break
builtres = list()
for resname in res:
builtres.append(self._resmap[resname].getValue())
if len(states[1]) > 0 or len(states[2]) > 0:
dbgstr('Done in: ' + self._readabletime(time.time() - t) + '.')
if len(builtres) > 1:
return builtres
else:
return builtres[0]
def _newResource(self, resname, resval, t):
dbgstr('Updating resource: ' + resname, 2)
dbgstr('with contents: ' + str(resval), 3)
self._getResource(resname).setValue(resval)
self._getResource(resname).updateFingerprint()
self._getResource(resname)._buildtime = t
self._getResource(resname)._timestamp = time.asctime()
dbgstr('Dumping resource: ' + resname)
self._getResource(resname).dump()
def _callModPar(self, node, nodeparams, queue):
dbgstr('Running node: ' + node)
funct = self._modules[node].getValue()
if len(nodeparams)==0:
dbgstr('No input for: ' + str(node), 2)
t = time.time()
newres = funct()
t = time.time() - t
else:
t = time.time()
newres = funct(*nodeparams)
t = time.time() - t
dbgstr('Produced list:\n\t' + str(newres), 3)
#the following is needed to correctly match node
#and resource when going parallel
queue.put((node, newres, t))
def _getNodePar(self, node):
nodeparams = list()
input_nodes = self._getInNodes(node)
for item in input_nodes:
thisnode_inputs = list()
neededres = item
this_params = self._getResource(neededres)
if type(this_params.getValue())==list:
dbgstr('Resource type is: list.', 2)
for this_param in this_params.getValue():
thisnode_inputs.append(this_param)
elif self._isResFile(this_params):
dbgstr('Resource type is: file.', 2)
thisnode_inputs.append(this_params.getValue())
else:
dbgstr('Resource type is: ' + str(type(this_params.getValue())), 2)
thisnode_inputs.append(this_params.getValue())
if len(thisnode_inputs)==1:
nodeparams.append(thisnode_inputs[0])
else:
nodeparams.append(thisnode_inputs)
return nodeparams
def _setMods(self, mods):
self._modules = mods
for modname in mods.keys():
self._setMod(modname, mods[modname])
dbgstr('Modules are: ' + str(self._modules), 3)
def _callMod(self, node, nodeparams):
t = time.time()
if not self._checkIsFunction(self._modules[node].getValue()):
dbgstr('Node '+node+' is not a function: passing itself.', 2)
newres = self._modules[node].getValue()
elif len(nodeparams)==0:
dbgstr('No input for: ' + str(node), 2)
dbgstr('Running node: ' + node)
newres = self._modules[node].getValue()()
dbgstr('Done.')
dbgstr('Produced list:\n\t' + str(newres), 3)
elif self._getGraph().getAttrib(node, 'hash'):
dbgstr('Inputs are joined.', 2)
dbgstr('Running node: ' + node)
newres = self._modules[node].getValue()(*nodeparams)
dbgstr('Done.', 0)
dbgstr('Produced list:\n\t' + str(newres), 3)
else:
dbgstr('Inputs are hashed.', 2)
for nodeparam in nodeparams:
dbgstr('Running node: ' + node)
newres = self._modules[node].getValue()(nodeparam)
dbgstr('Done.')
dbgstr('Produced list:\n\t' + str(newres), 3)
newresname = self._buildResName(node, None, newres)
dbgstr('Requesting add resource: ' + node, 2)
t = time.time() - t
#self._newResource(newresname, newres, t)
return newres, newresname, t
def _runNode(self, node):
dbgstr('Running node: ' + str(node))
nodeparams = list()
input_nodes = self._getInNodes(node)
for item in input_nodes:
thisnode_inputs = list()
neededres = item
dbgstr('Retreiving resource: ' + neededres)
this_params = self._provideResource(neededres)
if type(this_params.getValue())==list:
dbgstr('Resource type is: list.', 2)
for this_param in this_params.getValue():
thisnode_inputs.append(this_param)
elif self._isResFile(this_params):
dbgstr('Resource type is: file.', 2)
thisnode_inputs.append(this_params.getValue())
else:
dbgstr('Resource type is: ' + str(type(this_params.getValue())), 2)
thisnode_inputs.append(this_params.getValue())
if len(thisnode_inputs)==1:
nodeparams.append(thisnode_inputs[0])
else:
nodeparams.append(thisnode_inputs)
dbgstr('Ready to run: ' + node, 2)
dbgstr('through ' + str(self._getModule(node).getValue()), 2)
dbgstr('on input:\n\t' + str(nodeparams), 3)
newres, newresname = self._callMod(node, nodeparams)
dbgstr('Build took ' + str(self._resmap[newresname]._buildtime))
return newres
def _provideResource(self, resname):
dbgstr('Providing resource: ' + str(resname), 2)
if self._isAvailable(resname):
dbgstr('Found in RAM: ' + str(resname))
dbgstr('Resource content is:\n' + str(self._getResource(resname)) ,4)
return self._getResource(resname)
elif self._isDumped(resname):
dbgstr('Found on disk: ' + str(resname), 2)
self._addResource(resname, self._loadResource(resname))
dbgstr('Resource content is:\n' + str(self._getResource(resname)), 4)
return self._resmap[resname]
else:
dbgstr('Resource not found. I need to run first: ' + str(resname))
self._runNode(resname)
return self._getResource(resname)
def _isResFile(self, res):
isit = res.isFile()
if isit : dbgstr(str(res) + ' is a file.')
return isit
def _getResource(self, resname):
dbgstr('Getting resource: ' + str(resname), 3)
return self._resmap[resname]
def reset(self, filtername):
"""Clears and undump a resource"""
if self._isAvailable(filtername) or self._isDumped(filtername):
dbgstr('Resetting resource: ' + str(filtername))
self.clear(filtername, False)
self.undump(filtername, False)