예제 #1
0
파일: ptl.py 프로젝트: franapoli/pyleaf
    def __init__(self, graph, mods, folder, doc):
        untrustme1 = list()
        untrustme2 = list()
        dbgstr('Initializing protocol with root: ' + folder)
        self._metafolder = folder
        self._rootdir = os.getcwd()
        self._resmap = dict()
        self._doc = doc

        self._graphres = resource('graph', os.path.join(folder,'graph.grp'))
        untrustme1 = self._updateGraph(graph)

        for res in self._getResNames():
            newres = resource(res, folder+'/'+res+'.res')
            #print res, newres
            self._addResource(res, newres)

        for mod in self._getNodeNames():
            newres = resource(mod, folder+'/'+mod+'.mod')
            self._modules[mod] = newres

        untrustme2 = self._updateModules(mods)

        for mod in untrustme1:
            self.untrust(mod)
        for mod in untrustme2:
            self.untrust(mod)
예제 #2
0
파일: ptl.py 프로젝트: franapoli/pyleaf
    def _manageGraphChange(self, newGraph):
        if newGraph==None:
            return
        #checking for change in graph structure
        #a node is flagged as to-be-untrustest if its inputs have changed
        #not untrusted on the fly because of dependancies with untrusts
        #from source code changes
        oldg = self._graphres.getValue()
        untrustme = list()
#        import pdb; pdb.set_trace()

        for node in newGraph.getNodes():
            #check wether node is (not) new
            if node in oldg.getNodes():
                #check if node inputs have changed
                innodes1 = newGraph.getInNodes(node)
                innodes2 = oldg.getInNodes(node)
                for inNode in innodes1:
                    if not inNode in innodes2:
                        dbgstr('Inputs to ' + node + ' have changed, untrusting it.')
                        untrustme.append(node)
                #check if node properties have changed                
                #cycling through attrib names of node 'node'
                # for attrib in [a[1] for a in newGraph._nodeattribs if a[0]==node]:
                #     if newGraph.getAttrib(node, attrib) != oldg.getAttrib(node, attrib):
                #         dbgstr('Node attribs of ' + node +
                #                ' have changed, untrusting it.')
                #         untrustme.append(node)                        

        return untrustme
예제 #3
0
파일: ptl.py 프로젝트: franapoli/pyleaf
    def _runNodes(self, nodes, parallel = False):
        if parallel and len(nodes)>1:
            from multiprocessing import Process, Queue
            results = Queue()
            tasks = list()
            for node in nodes:
                nodeparams = self._getNodePar(node)
                tasks.append(
                    Process(target=self._callModPar, args=(node, nodeparams, results,))
                    )
            for task in tasks: task.start()
            for task in tasks: task.join()

            for task in tasks:
                taskres = results.get()
                resname = self._buildResName(taskres[0], None, taskres[1])
                dbgstr('Requesting add resource: ' + taskres[0], 2)
                t=taskres[2]
                self._newResource(resname, taskres[1], t)
        else:
            for node in nodes:
                nodeparams = self._getNodePar(node)
                taskres = self._callMod(node, nodeparams)
                resname = self._buildResName(taskres[1], None, taskres[0])
                dbgstr('Requesting add resource: ' + resname, 2)
                t=taskres[2]
                self._newResource(resname, taskres[0], t)
예제 #4
0
파일: ptl.py 프로젝트: franapoli/pyleaf
 def undump(self, filtername, verbose = True):
     """Undumps a resource (removes cached version from disk)."""
     if type(filtername) != str:
         filtername = filtername.__name__
     if self._isDumped(filtername):
         if verbose:
             dbgstr('Undumping resource: ' + str(filtername))
         self._getResource(filtername).clearDump()
예제 #5
0
파일: ptl.py 프로젝트: franapoli/pyleaf
 def _loadMods(self):
     if os.path.exists(self.modsToPath()):
         mods = pickle.load(open(self.modsToPath(), 'r'))
     else:
         mods = dict()
     
     dbgstr('Found on disk: ' + str(mods.keys()), 2)
     return mods
예제 #6
0
파일: ptl.py 프로젝트: franapoli/pyleaf
 def clear(self, filtername, verbose = True):
     """Clears a resource.""" 
     if type(filtername) != str:
         filtername = filtername.__name__
     if self._isAvailable(filtername):
         if verbose:
             dbgstr('Clearing resource: ' + str(filtername))
         self._resmap[filtername].clear()
예제 #7
0
파일: ptl.py 프로젝트: franapoli/pyleaf
 def _setMod(self, modname, mod):
     haschanged = self._modules[modname].changed()
     if haschanged:
         if type(mod) == File:
             mod.update()
         self._modules[modname].setValue(mod.getValue())
         self._modules[modname].updateFingerprint()
         self.untrust(modname)
         dependents = self._getOutNodesRecursive(modname)
         if dependents != []:
             dbgstr('These nodes are dependent: '+str(dependents), 0)
예제 #8
0
파일: ptl.py 프로젝트: franapoli/pyleaf
    def trust(self, who, what):
        """Assign a resource to a filter without invalidating dependent resources."""
        if type(who) != str:
            who = who.__name__

        if type(who) == str:
            self._setMod(who,what)
            dbgstr('I\'m assuming that using ' + str(what) + ' for ' + who + ' won\'t have consequences on other nodes.', 0)
        elif type(who) == tuple:
            self._addResource(who,what)        
            dbgstr('I\'m assuming that using your object of type ' + str(type(what)) + ' for ' + str(who) + ' won\'t have consequences on other nodes.', 0)
예제 #9
0
파일: ptl.py 프로젝트: franapoli/pyleaf
 def _update(self, graph, mods):
     untrustme1 = list()
     untrustme2 = list()
     dbgstr('New graph is: ' + str(graph), 3)
     dbgstr('New mods are: ' + str(mods), 3)
     untrustme1 = self._updateGraph(graph)
     untrustme2 = self._updateModules(mods)
     for mod in untrustme1:
         self.untrust(mod)
     for mod in untrustme2:
         self.untrust(mod)
예제 #10
0
파일: ptl.py 프로젝트: franapoli/pyleaf
 def run(self, parallel = False):
     """Provides all leaf (final) resources."""
     res = None
     allok = True
     l = self._getLeaves()
     for leaf in l:
         resname = leaf
         if not self._isAvailable(resname)and not self._isDumped(resname):
             allok = False
     
     if allok:
         dbgstr('Nothing to be done. Zzz...')
     else:
         res = self.provide(l, parallel)
     return res
예제 #11
0
파일: ptl.py 프로젝트: franapoli/pyleaf
 def _ChangeME_resToPath(self, res):
     if self._getGraph().getAttrib(res[0], 'hashout'):
         if res[0] == None: first_part = ''
         else: first_part = res[0]
         if res[1] == None: second_part = ''
         else: second_part = res[1]
         if first_part != '' and second_part !='':
             mid_part = 'TO'
         else: mid_part = ''
     
         fname = first_part + mid_part + second_part + '.dmp'
     else:
         fname = res[0]
         
     dbgstr('Dump file for resource ' + str(res) + ' is ' + self._metafolder + '/' + fname, 2)
     return self._metafolder + '/' + fname
예제 #12
0
파일: ptl.py 프로젝트: franapoli/pyleaf
 def _updateModules(self, mods):
     untrustme = list()
     for modname in mods:
         if modname in self._modules.keys():
             self._modules[modname].setValue(mods[modname])
             if self._modules[modname].changed():
                 untrustme.append(modname)
                 #self.untrust(self._modules[modname].name())
                 self._modhelp[modname] = inspect.getdoc(self._modules[modname].getValue())
             self._modules[modname].update()
         else:
             dbgstr('New module: ' + modname)
             self._modules[modname] = resource(modname,
                 self._metafolder+'/'+modname+'.dmp' )
             self._addResource(modname, self._modules[modname])
             self._modules[modname].setValue(mods[modname])
             self._modules[modname].update()
     return untrustme
예제 #13
0
파일: ptl.py 프로젝트: franapoli/pyleaf
 def _loadResource(self, res):
     dbgstr('Getting resource ' + str(res) + ' from disk.', 2)
     if self._isDumped(res):
         dbgstr('Resource ' + str(self._resmap[res]) + ' found in: ' + self._resmap[res].getDumpPath() ,2)
         return pickle.load(open(self._resmap[res].getDumpPath(), 'rb'))
     else:
         dbgstr('Resource ' + str(res) + ' not found on disk! I\'ve been looking for: ' + self._resmap[res].getDumpPath())
예제 #14
0
파일: ptl.py 프로젝트: franapoli/pyleaf
    def _processRawRes(self, node, rawres):
        

                        
# #        elif type(rawres)==tuple:        
# #            dbgstr('Raw list are packed in a tuple.')
# #            
# #            if len(rawres != len(self._getGraph()[node])):
# #                raise NameError('When a module returns a tuple, it''s length must be equal to the number of the module''s outputs.')
# #                
# #            for idx, outnode in enumerate(self._getGraph()[node]):
# #                newresname = self._buildResName(node, outnode, rawres)                
# #                dbgstr('Requesting add resource: ' + str(newresname))
# #                if self._isFileMod(node):
# #                    self._placeFileRes(rawres)
# #                    self._newResource(newresname, self._updateFilePath(rawres[idx]))
# #                else:
# #                    self._newResource(newresname, rawres[idx])
# #                
# #        else:

        # if self._isFileMod(node):
        #     if type(rawres)==tuple or type(rawres)==list:
        #         for rawresi in rawres:
        #             newresname = self._buildResName(node, None, rawresi)
        #             dbgstr('Requesting add resource: ' + node, 2)
        #             self._placeFileRes(rawresi)
        #             self._newResource(newresname, self._updateFilePath(rawresi))
        #     else:
        #         newresname = self._buildResName(node, None, rawres)
        #         dbgstr('Requesting add resource: ' + node, 2)
        #         self._placeFileRes(rawres)
        #         self._newResource(newresname, self._updateFilePath(rawres))
        # else:
        #     newresname = self._buildResName(node, None, rawres)
        #     dbgstr('Requesting add resource: ' + node, 2)
        #     self._newResource(newresname, rawres)
        newresname = self._buildResName(node, None, rawres)
        dbgstr('Requesting add resource: ' + node, 2)
        self._newResource(newresname, rawres)
예제 #15
0
파일: ptl.py 프로젝트: franapoli/pyleaf
 def _isAvailable(self, resname):
     dbgstr('Checking resource: ' + str(resname), 3)
     if self._getResource(resname).isAvailable():
        dbgstr('Available: ' + str(resname), 3)
        return True
     dbgstr('Unavailable: ' + str(resname), 3)
     return False        
예제 #16
0
파일: ptl.py 프로젝트: franapoli/pyleaf
    def provide(self, res, parallel=False):
        t = time.time()

        if not type(res) == list:
            temp = list()
            temp.append(res)
            res = temp

        for i,r in enumerate(res):
            if type(res[i]) != str:
                res[i] = res[i].__name__

        D = self._findDependancies(res)
        D.update(res)
        states = self._getBestStates(D)

        if len(states[1]) > 0:
            dbgstr('The following resources will be loaded from disk: '+
                   self._prettyPrint(states[1]))
            for node in states[1]:
                self._provideResource(node)

        if len(states[2]) > 0:
            dbgstr('The following resources will need production: '+
                   self._prettyPrint(states[2]))
        needbuild = states[2]

        while True:
            canrun = self._findRunnable(needbuild)
            if len(canrun) > 1 and parallel:
                dbgstr('The following nodes can run in parallel: '+
                       self._prettyPrint(canrun))

            self._runNodes(canrun, parallel)
            needbuild=[x for x in needbuild if not x in canrun]

            if len(canrun)==0:
                break

        builtres = list()
        for resname in res:
            builtres.append(self._resmap[resname].getValue())

        if len(states[1]) > 0 or len(states[2]) > 0:
            dbgstr('Done in: ' + self._readabletime(time.time() - t) + '.')

        if len(builtres) > 1:
            return builtres
        else:
            return builtres[0]
예제 #17
0
파일: ptl.py 프로젝트: franapoli/pyleaf
 def _newResource(self, resname, resval, t):
     dbgstr('Updating resource: ' + resname, 2)
     dbgstr('with contents: ' + str(resval), 3)
     self._getResource(resname).setValue(resval)
     self._getResource(resname).updateFingerprint()
     self._getResource(resname)._buildtime = t
     self._getResource(resname)._timestamp = time.asctime()
     dbgstr('Dumping resource: ' + resname)
     self._getResource(resname).dump()
예제 #18
0
파일: ptl.py 프로젝트: franapoli/pyleaf
    def _callModPar(self, node, nodeparams, queue):
           
        dbgstr('Running node: ' + node)
        funct = self._modules[node].getValue()
        if len(nodeparams)==0:
            dbgstr('No input for: ' + str(node), 2)
            t = time.time()
            newres = funct()
            t = time.time() - t    
        else:
            t = time.time()
            newres = funct(*nodeparams)
            t = time.time() - t

        dbgstr('Produced list:\n\t' + str(newres), 3)

            
        #the following is needed to correctly match node
        #and resource when going parallel
        queue.put((node, newres, t))
예제 #19
0
파일: ptl.py 프로젝트: franapoli/pyleaf
    def _getNodePar(self, node):

        nodeparams = list()        
        input_nodes = self._getInNodes(node)
        for item in input_nodes:
            thisnode_inputs = list()
            neededres = item
            this_params = self._getResource(neededres)
            if type(this_params.getValue())==list:
                dbgstr('Resource type is: list.', 2)
                for this_param in this_params.getValue():
                    thisnode_inputs.append(this_param)
            elif self._isResFile(this_params):
                dbgstr('Resource type is: file.', 2)
                thisnode_inputs.append(this_params.getValue())
            else:
                dbgstr('Resource type is: ' + str(type(this_params.getValue())), 2)
                thisnode_inputs.append(this_params.getValue())
            if len(thisnode_inputs)==1:
                nodeparams.append(thisnode_inputs[0])
            else:
                nodeparams.append(thisnode_inputs)
        
        return nodeparams
예제 #20
0
파일: ptl.py 프로젝트: franapoli/pyleaf
 def _setMods(self, mods):
     self._modules = mods
     for modname in mods.keys():
         self._setMod(modname, mods[modname])
     
     dbgstr('Modules are: ' + str(self._modules), 3)
예제 #21
0
파일: ptl.py 프로젝트: franapoli/pyleaf
    def _callMod(self, node, nodeparams):
        t = time.time()

        if not self._checkIsFunction(self._modules[node].getValue()):
            dbgstr('Node '+node+' is not a function: passing itself.', 2)            
            newres = self._modules[node].getValue()

            
        elif len(nodeparams)==0:
            dbgstr('No input for: ' + str(node), 2)
            dbgstr('Running node: ' + node)
            
            newres = self._modules[node].getValue()()


            dbgstr('Done.')
            dbgstr('Produced list:\n\t' + str(newres), 3)

    
        elif self._getGraph().getAttrib(node, 'hash'):
            dbgstr('Inputs are joined.', 2)
            dbgstr('Running node: ' + node)
            newres = self._modules[node].getValue()(*nodeparams)
            dbgstr('Done.', 0)
            dbgstr('Produced list:\n\t' + str(newres), 3)

            
        else:
            dbgstr('Inputs are hashed.', 2)
            for nodeparam in nodeparams:
                dbgstr('Running node: ' + node)
                newres = self._modules[node].getValue()(nodeparam)
                dbgstr('Done.')
                dbgstr('Produced list:\n\t' + str(newres), 3)

        newresname = self._buildResName(node, None, newres)
        dbgstr('Requesting add resource: ' + node, 2)
        t = time.time() - t
        #self._newResource(newresname, newres, t)
            
        return newres, newresname, t
예제 #22
0
파일: ptl.py 프로젝트: franapoli/pyleaf
    def _runNode(self, node):
        dbgstr('Running node: ' + str(node))
        
        nodeparams = list()        
        input_nodes = self._getInNodes(node)
        for item in input_nodes:
            thisnode_inputs = list()
            neededres = item
            dbgstr('Retreiving resource: ' + neededres)
            this_params = self._provideResource(neededres)
            if type(this_params.getValue())==list:
                dbgstr('Resource type is: list.', 2)
                for this_param in this_params.getValue():
                    thisnode_inputs.append(this_param)
            elif self._isResFile(this_params):
                dbgstr('Resource type is: file.', 2)
                thisnode_inputs.append(this_params.getValue())
            else:
                dbgstr('Resource type is: ' + str(type(this_params.getValue())), 2)
                thisnode_inputs.append(this_params.getValue())
            if len(thisnode_inputs)==1:
                nodeparams.append(thisnode_inputs[0])
            else:
                nodeparams.append(thisnode_inputs)
        
        dbgstr('Ready to run: ' + node, 2)
        dbgstr('through ' + str(self._getModule(node).getValue()), 2)
        dbgstr('on input:\n\t' + str(nodeparams), 3)
        
        newres, newresname = self._callMod(node, nodeparams)
        dbgstr('Build took ' + str(self._resmap[newresname]._buildtime))

        return newres
예제 #23
0
파일: ptl.py 프로젝트: franapoli/pyleaf
 def _provideResource(self, resname):
     dbgstr('Providing resource: ' + str(resname), 2)
     if self._isAvailable(resname):
         dbgstr('Found in RAM: ' + str(resname))
         dbgstr('Resource content is:\n' + str(self._getResource(resname)) ,4)
         return self._getResource(resname)
     elif self._isDumped(resname):
         dbgstr('Found on disk: ' + str(resname), 2)
         self._addResource(resname, self._loadResource(resname))
         dbgstr('Resource content is:\n' + str(self._getResource(resname)), 4)
         return self._resmap[resname]
     else:
         dbgstr('Resource not found. I need to run first: ' + str(resname))
         self._runNode(resname)
         return self._getResource(resname)
예제 #24
0
파일: ptl.py 프로젝트: franapoli/pyleaf
 def _isResFile(self, res):
     isit = res.isFile()
     if isit : dbgstr(str(res) + ' is a file.')
     return isit
예제 #25
0
파일: ptl.py 프로젝트: franapoli/pyleaf
 def _getResource(self, resname):
     dbgstr('Getting resource: ' + str(resname), 3)        
     return self._resmap[resname]
예제 #26
0
파일: ptl.py 프로젝트: franapoli/pyleaf
 def reset(self, filtername):
     """Clears and undump a resource"""
     if self._isAvailable(filtername) or self._isDumped(filtername):
         dbgstr('Resetting resource: ' + str(filtername))
     self.clear(filtername, False)
     self.undump(filtername, False)