def on_prune(self, fgraph, app, reason): """ Remove Apply instance from set which must be computed. """ if app not in self.debug_all_apps: raise ProtocolError("prune without import") self.debug_all_apps.remove(app) # UPDATE self.clients for i, input in enumerate(OrderedSet(app.inputs)): del self.clients[input][app] if getattr(app.op, 'destroy_map', OrderedDict()): self.destroyers.remove(app) # Note: leaving empty client dictionaries in the struct. # Why? It's a pain to remove them. I think they aren't doing any harm, they will be # deleted on_detach(). # UPDATE self.view_i, self.view_o for o_idx, i_idx_list in iteritems( getattr(app.op, 'view_map', OrderedDict())): if len(i_idx_list) > 1: # destroying this output invalidates multiple inputs raise NotImplementedError() o = app.outputs[o_idx] i = app.inputs[i_idx_list[0]] del self.view_i[o] self.view_o[i].remove(o) if not self.view_o[i]: del self.view_o[i] self.stale_droot = True
def register(self, name, obj, *tags): # N.B. obj is not an instance of class Optimizer. # It is an instance of a DB.In the tests for example, # this is not always the case. if not isinstance(obj, (DB, opt.Optimizer, opt.LocalOptimizer)): raise TypeError('Object cannot be registered in OptDB', obj) if name in self.__db__: raise ValueError( 'The name of the object cannot be an existing' ' tag or the name of another existing object.', obj, name) if self.name is not None: tags = tags + (self.name, ) obj.name = name # This restriction is there because in many place we suppose that # something in the DB is there only once. if obj.name in self.__db__: raise ValueError('''You can\'t register the same optimization multiple time in a DB. Tryed to register "%s" again under the new name "%s". Use theano.gof.ProxyDB to work around that''' % (obj.name, name)) self.__db__[name] = OrderedSet([obj]) self._names.add(name) self.__db__[obj.__class__.__name__].add(obj) self.add_tags(name, *tags)
def orderings(self, fgraph): """ Return orderings induced by destructive operations. Raise InconsistencyError when a) attempting to destroy indestructable variable, or b) attempting to destroy a value multiple times, or c) an Apply destroys (illegally) one of its own inputs by aliasing """ rval = OrderedDict() if self.destroyers: # BUILD DATA STRUCTURES # CHECK for multiple destructions during construction of variables droot, impact, __ignore = self.refresh_droot_impact() # check for destruction of constants illegal_destroy = [r for r in droot if getattr(r.tag, 'indestructible', False) or isinstance(r, graph.Constant)] if illegal_destroy: raise InconsistencyError( "Attempting to destroy indestructible variables: %s" % illegal_destroy) # add destroyed variable clients as computational dependencies for app in self.destroyers: # for each destroyed input... for output_idx, input_idx_list in iteritems(app.op.destroy_map): destroyed_idx = input_idx_list[0] destroyed_variable = app.inputs[destroyed_idx] root = droot[destroyed_variable] root_impact = impact[root] # we generally want to put all clients of things which depend on root # as pre-requisites of app. # But, app is itself one such client! # App will always be a client of the node we're destroying # (destroyed_variable, but the tricky thing is when it is also a client of # *another variable* viewing on the root. Generally this is illegal, (e.g., # add_inplace(x, x.T). In some special cases though, the in-place op will # actually be able to work properly with multiple destroyed inputs (e.g, # add_inplace(x, x). An Op that can still work in this case should declare # so via the 'destroyhandler_tolerate_same' attribute or # 'destroyhandler_tolerate_aliased' attribute. # # destroyhandler_tolerate_same should be a list of pairs of the form # [(idx0, idx1), (idx0, idx2), ...] # The first element of each pair is the input index of a destroyed # variable. # The second element of each pair is the index of a different input where # we will permit exactly the same variable to appear. # For example, add_inplace.tolerate_same might be [(0,1)] if the destroyed # input is also allowed to appear as the second argument. # # destroyhandler_tolerate_aliased is the same sort of list of # pairs. # op.destroyhandler_tolerate_aliased = [(idx0, idx1)] tells the # destroyhandler to IGNORE an aliasing between a destroyed # input idx0 and another input idx1. # This is generally a bad idea, but it is safe in some # cases, such as # - the op reads from the aliased idx1 before modifying idx0 # - the idx0 and idx1 are guaranteed not to overlap (e.g. # they are pointed at different rows of a matrix). # # CHECK FOR INPUT ALIASING # OPT: pre-compute this on import tolerate_same = getattr(app.op, 'destroyhandler_tolerate_same', []) assert isinstance(tolerate_same, list) tolerated = OrderedSet(idx1 for idx0, idx1 in tolerate_same if idx0 == destroyed_idx) tolerated.add(destroyed_idx) tolerate_aliased = getattr( app.op, 'destroyhandler_tolerate_aliased', []) assert isinstance(tolerate_aliased, list) ignored = OrderedSet(idx1 for idx0, idx1 in tolerate_aliased if idx0 == destroyed_idx) # print 'tolerated', tolerated # print 'ignored', ignored for i, input in enumerate(app.inputs): if i in ignored: continue if input in root_impact \ and (i not in tolerated or input is not destroyed_variable): raise InconsistencyError("Input aliasing: %s (%i, %i)" % (app, destroyed_idx, i)) # add the rule: app must be preceded by all other Apply instances that # depend on destroyed_input root_clients = OrderedSet() for r in root_impact: assert not [a for a, c in self.clients[r].items() if not c] root_clients.update([a for a, c in self.clients[r].items() if c]) root_clients.remove(app) if root_clients: rval[app] = root_clients return rval
def on_attach(self, fgraph): """ When attaching to a new fgraph, check that 1) This DestroyHandler wasn't already attached to some fgraph (its data structures are only set up to serve one) 2) The FunctionGraph doesn't already have a DestroyHandler. This would result in it validating everything twice, causing compilation to be slower. TODO: WRITEME: what does this do besides the checks? """ ####### Do the checking ########### already_there = False if self.fgraph is fgraph: already_there = True if self.fgraph not in [None, fgraph]: raise Exception("A DestroyHandler instance can only serve" " one FunctionGraph. (Matthew 6:24)") for attr in ('destroyers', 'destroy_handler'): if hasattr(fgraph, attr): already_there = True if already_there: # FunctionGraph.attach_feature catches AlreadyThere # and cancels the attachment raise toolbox.AlreadyThere( "DestroyHandler feature is already present or in" " conflict with another plugin.") ####### end of checking ############ def get_destroyers_of(r): droot, impact, root_destroyer = self.refresh_droot_impact() try: return [root_destroyer[droot[r]]] except Exception: return [] fgraph.destroyers = get_destroyers_of fgraph.destroy_handler = self self.fgraph = fgraph self.destroyers = OrderedSet( ) #set of Apply instances with non-null destroy_map self.view_i = {} # variable -> variable used in calculation self.view_o = { } # variable -> set of variables that use this one as a direct input #clients: how many times does an apply use a given variable self.clients = {} # variable -> apply -> ninputs self.stale_droot = True # IG: It's unclear if this is meant to be included in deployed code. It looks like # it is unnecessary if FunctionGraph is working correctly, so I am commenting uses # of it (for speed) but leaving the commented code in place so it is easy to restore # for debugging purposes. # Note: is there anything like the C preprocessor for python? It would be useful to # just ifdef these things out # self.debug_all_apps = set() if self.do_imports_on_attach: toolbox.Bookkeeper.on_attach(self, fgraph)
def get_impact(root, view_o): impact = OrderedSet() add_impact(root, view_o, impact) return impact