def add(self, edge): """add an edge to this collection.""" (parentnode, childnode) = edge if not self.parent_to_children.has_key(parentnode): self.parent_to_children[parentnode] = util.Set() self.parent_to_children[parentnode].add(childnode) if not self.child_to_parents.has_key(childnode): self.child_to_parents[childnode] = util.Set() self.child_to_parents[childnode].add(parentnode) parentnode.dependencies.add(childnode)
def flush(self, session, objects=None): # this context will track all the objects we want to save/update/delete, # and organize a hierarchical dependency structure. it also handles # communication with the mappers and relationships to fire off SQL # and synchronize attributes between related objects. echo = logging.is_info_enabled(self.logger) flush_context = UOWTransaction(self, session) # create the set of all objects we want to operate upon if objects is not None: # specific list passed in objset = util.Set(objects) else: # or just everything objset = util.Set(self.identity_map.values()).union(self.new) # detect persistent objects that have changes dirty = self.locate_dirty() # store objects whose fate has been decided processed = util.Set() # put all saves/updates into the flush context. detect orphans and throw them into deleted. for obj in self.new.union(dirty).intersection(objset).difference( self.deleted): if obj in processed: continue if object_mapper(obj)._is_orphan(obj): for c in [obj] + list( object_mapper(obj).cascade_iterator('delete', obj)): if c in processed: continue flush_context.register_object(c, isdelete=True) processed.add(c) else: flush_context.register_object(obj) processed.add(obj) # put all remaining deletes into the flush context. for obj in self.deleted: if (objset is not None and not obj in objset) or obj in processed: continue flush_context.register_object(obj, isdelete=True) trans = session.create_transaction(autoflush=False) flush_context.transaction = trans try: flush_context.execute() except: trans.rollback() raise trans.commit() flush_context.post_exec()
def __init__(self, identity_map=None, weak_identity_map=False): if identity_map is not None: self.identity_map = identity_map else: if weak_identity_map: self.identity_map = weakref.WeakValueDictionary() else: self.identity_map = {} self.new = util.Set() #OrderedSet() self.deleted = util.Set() self.logger = logging.instance_logger(self)
def __init__(self, mapper, options): self.mapper = mapper self.options = options self.attributes = {} self.recursion_stack = util.Set() for opt in options: self.accept_option(opt)
def _locate_prop(self, key, start=None): import properties keys = [] seen = util.Set() def search_for_prop(mapper_): if mapper_ in seen: return None seen.add(mapper_) if mapper_.props.has_key(key): prop = mapper_.props[key] if isinstance(prop, properties.SynonymProperty): prop = mapper_.props[prop.name] if isinstance(prop, properties.PropertyLoader): keys.insert(0, prop.key) return prop else: for prop in mapper_.props.values(): if not isinstance(prop, properties.PropertyLoader): continue x = search_for_prop(prop.mapper) if x: keys.insert(0, prop.key) return x else: return None p = search_for_prop(start or self.mapper) if p is None: raise exceptions.InvalidRequestError( "Cant locate property named '%s'" % key) return [keys, p]
def _get_noninheriting_mappers(self): """returns a list of UOWTasks whose mappers are not inheriting from the mapper of another UOWTask. i.e., this returns the root UOWTasks for all the inheritance hierarchies represented in this UOWTransaction.""" mappers = util.Set() for task in self.tasks.values(): base = task.mapper.base_mapper() mappers.add(base) return mappers
def __init__(self, uow, session): self.uow = uow self.session = session # unique list of all the mappers we come across self.mappers = util.Set() self.dependencies = {} self.tasks = {} self.logger = logging.instance_logger(self) self.echo = uow.echo
def append_postupdate(self, obj, post_update_cols): # postupdates are UPDATED immeditely (for now) # convert post_update_cols list to a Set so that __hashcode__ is used to compare columns # instead of __eq__ self.mapper.save_obj([obj], self.uowtransaction, postupdate=True, post_update_cols=util.Set(post_update_cols)) return True
def __init__(self, query, kwargs): self.query = query self.order_by = kwargs.pop('order_by', False) self.from_obj = kwargs.pop('from_obj', []) self.lockmode = kwargs.pop('lockmode', query.lockmode) self.distinct = kwargs.pop('distinct', False) self.limit = kwargs.pop('limit', None) self.offset = kwargs.pop('offset', None) self.eager_loaders = util.Set([x for x in query.mapper._eager_loaders]) self.statement = None super(QueryContext, self).__init__(query.mapper, query.with_options, **kwargs)
def __init__(self, uowtransaction, mapper, circular_parent=None): if not circular_parent: uowtransaction.tasks[mapper] = self # the transaction owning this UOWTask self.uowtransaction = uowtransaction # the Mapper which this UOWTask corresponds to self.mapper = mapper # a dictionary mapping object instances to a corresponding UOWTaskElement. # Each UOWTaskElement represents one instance which is to be saved or # deleted by this UOWTask's Mapper. # in the case of the row-based "circular sort", the UOWTaskElement may # also reference further UOWTasks which are dependent on that UOWTaskElement. self.objects = {} #util.OrderedDict() # a list of UOWDependencyProcessors which are executed after saves and # before deletes, to synchronize data to dependent objects self.dependencies = util.Set() # a list of UOWTasks that are dependent on this UOWTask, which # are to be executed after this UOWTask performs saves and post-save # dependency processing, and before pre-delete processing and deletes self.childtasks = [] # whether this UOWTask is circular, meaning it holds a second # UOWTask that contains a special row-based dependency structure. self.circular = None # for a task thats part of that row-based dependency structure, points # back to the "public facing" task. self.circular_parent = circular_parent # a list of UOWDependencyProcessors are derived from the main # set of dependencies, referencing sub-UOWTasks attached to this # one which represent portions of the total list of objects. # this is used for the row-based "circular sort" self.cyclical_dependencies = util.Set()
def __init__(self, arg=""): values = util.Set([c.strip() for c in arg.split(',')]) self.delete_orphan = "delete-orphan" in values self.delete = "delete" in values or self.delete_orphan or "all" in values self.save_update = "save-update" in values or "all" in values self.merge = "merge" in values or "all" in values self.expunge = "expunge" in values or "all" in values # refresh_expire not really implemented as of yet #self.refresh_expire = "refresh-expire" in values or "all" in values for x in values: if x not in all_cascades: raise exceptions.ArgumentError("Invalid cascade option '%s'" % x)
def __init__(self, attr, obj, current, passive=False): self.attr = attr # get the "original" value. if a lazy load was fired when we got # the 'current' value, this "original" was also populated just # now as well (therefore we have to get it second) orig = obj._state.get('original', None) if orig is not None: original = orig.data.get(attr.key) else: original = None if attr.uselist: self._current = current else: self._current = [current] if attr.uselist: s = util.Set(original or []) self._added_items = [] self._unchanged_items = [] self._deleted_items = [] if current: for a in current: if a in s: self._unchanged_items.append(a) else: self._added_items.append(a) for a in s: if a not in self._unchanged_items: self._deleted_items.append(a) else: if attr.is_equal(current, original): self._unchanged_items = [current] self._added_items = [] self._deleted_items = [] else: self._added_items = [current] if original is not None: self._deleted_items = [original] else: self._deleted_items = [] self._unchanged_items = []
def polymorphic_union(table_map, typecolname, aliasname='p_union'): """create a UNION statement used by a polymorphic mapper. See the SQLAlchemy advanced mapping docs for an example of how this is used.""" colnames = util.Set() colnamemaps = {} types = {} for key in table_map.keys(): table = table_map[key] # mysql doesnt like selecting from a select; make it an alias of the select if isinstance(table, sql.Select): table = table.alias() table_map[key] = table m = {} for c in table.c: colnames.add(c.name) m[c.name] = c types[c.name] = c.type colnamemaps[table] = m def col(name, table): try: return colnamemaps[table][name] except KeyError: return sql.cast(sql.null(), types[name]).label(name) result = [] for type, table in table_map.iteritems(): if typecolname is not None: result.append( sql.select([col(name, table) for name in colnames] + [sql.column("'%s'" % type).label(typecolname)], from_obj=[table])) else: result.append( sql.select([col(name, table) for name in colnames], from_obj=[table])) return sql.union_all(*result).alias(aliasname)
def _find_cycles(self, edges): involved_in_cycles = util.Set() cycles = {} def traverse(node, goal=None, cycle=None): if goal is None: goal = node cycle = [] elif node is goal: return True for (n, key) in edges.edges_by_parent(node): if key in cycle: continue cycle.append(key) if traverse(key, goal, cycle): cycset = util.Set(cycle) for x in cycle: involved_in_cycles.add(x) if cycles.has_key(x): existing_set = cycles[x] [existing_set.add(y) for y in cycset] for y in existing_set: cycles[y] = existing_set cycset = existing_set else: cycles[x] = cycset cycle.pop() for parent in edges.get_parents(): traverse(parent) for cycle in dict([(id(s), s) for s in cycles.values()]).values(): edgecollection = [] for edge in edges: if edge[0] in cycle and edge[1] in cycle: edgecollection.append(edge) yield edgecollection
def traverse(node, goal=None, cycle=None): if goal is None: goal = node cycle = [] elif node is goal: return True for (n, key) in edges.edges_by_parent(node): if key in cycle: continue cycle.append(key) if traverse(key, goal, cycle): cycset = util.Set(cycle) for x in cycle: involved_in_cycles.add(x) if cycles.has_key(x): existing_set = cycles[x] [existing_set.add(y) for y in cycset] for y in existing_set: cycles[y] = existing_set cycset = existing_set else: cycles[x] = cycset cycle.pop()
def merge(self, object, entity_name=None, _recursive=None): """copy the state of the given object onto the persistent object with the same identifier. If there is no persistent instance currently associated with the session, it will be loaded. Return the persistent instance. If the given instance is unsaved, save a copy of and return it as a newly persistent instance. The given instance does not become associated with the session. This operation cascades to associated instances if the association is mapped with cascade="merge". """ if _recursive is None: _recursive = util.Set() mapper = _object_mapper(object) key = getattr(object, '_instance_key', None) if key is None: merged = mapper._create_instance(self) else: if key in self.identity_map: merged = self.identity_map[key] else: merged = self.get(mapper.class_, key[1]) for prop in mapper.props.values(): prop.merge(self, object, merged, _recursive) if key is None: self.save(merged) return merged
def __init__(self): self.columns = util.Set()
def sort(self, allow_self_cycles=True, allow_all_cycles=False): (tuples, allitems) = (self.tuples, self.allitems) #print "\n---------------------------------\n" #print repr([t for t in tuples]) #print repr([a for a in allitems]) #print "\n---------------------------------\n" nodes = {} edges = _EdgeCollection() for item in allitems + [t[0] for t in tuples] + [t[1] for t in tuples]: if not nodes.has_key(item): node = _Node(item) nodes[item] = node for t in tuples: if t[0] is t[1]: if allow_self_cycles: n = nodes[t[0]] n.cycles = util.Set([n]) continue else: raise FlushError("Self-referential dependency detected " + repr(t)) childnode = nodes[t[1]] parentnode = nodes[t[0]] edges.add((parentnode, childnode)) queue = [] for n in nodes.values(): if not edges.has_parents(n): queue.append(n) cycles = {} output = [] while len(nodes) > 0: if len(queue) == 0: # edges remain but no edgeless nodes to remove; this indicates # a cycle if allow_all_cycles: x = self._find_cycles(edges) for cycle in self._find_cycles(edges): lead = cycle[0][0] lead.cycles = util.Set() for edge in cycle: n = edges.remove(edge) lead.cycles.add(edge[0]) lead.cycles.add(edge[1]) if n is not None: queue.append(n) for n in lead.cycles: if n is not lead: n._cyclical = True for (n, k) in list(edges.edges_by_parent(n)): edges.add((lead, k)) edges.remove((n, k)) continue else: # long cycles not allowed raise FlushError("Circular dependency detected " + repr(edges) + repr(queue)) node = queue.pop() if not hasattr(node, '_cyclical'): output.append(node) del nodes[node.item] for childnode in edges.pop_node(node): queue.append(childnode) return self._create_batched_tree(output)
def _sort_circular_dependencies(self, trans, cycles): """for a single task, creates a hierarchical tree of "subtasks" which associate specific dependency actions with individual objects. This is used for a "cyclical" task, or a task where elements of its object list contain dependencies on each other. this is not the normal case; this logic only kicks in when something like a hierarchical tree is being represented.""" allobjects = [] for task in cycles: allobjects += [e.obj for e in task.get_elements(polymorphic=True)] tuples = [] cycles = util.Set(cycles) #print "BEGIN CIRC SORT-------" #print "PRE-CIRC:" #print list(cycles) #[0].dump() # dependency processors that arent part of the cyclical thing # get put here extradeplist = [] # organizes a set of new UOWTasks that will be assembled into # the final tree, for the purposes of holding new UOWDependencyProcessors # which process small sub-sections of dependent parent/child operations dependencies = {} def get_dependency_task(obj, depprocessor): try: dp = dependencies[obj] except KeyError: dp = dependencies.setdefault(obj, {}) try: l = dp[depprocessor] except KeyError: l = UOWTask(self.uowtransaction, depprocessor.targettask.mapper, circular_parent=self) dp[depprocessor] = l return l def dependency_in_cycles(dep): # TODO: make a simpler way to get at the "root inheritance" mapper proctask = trans.get_task_by_mapper( dep.processor.mapper.primary_mapper().base_mapper(), True) targettask = trans.get_task_by_mapper( dep.targettask.mapper.base_mapper(), True) return targettask in cycles and (proctask is not None and proctask in cycles) # organize all original UOWDependencyProcessors by their target task deps_by_targettask = {} for t in cycles: for task in t.polymorphic_tasks(): for dep in task.dependencies: if not dependency_in_cycles(dep): extradeplist.append(dep) for t in dep.targettask.polymorphic_tasks(): l = deps_by_targettask.setdefault(t, []) l.append(dep) object_to_original_task = {} for t in cycles: for task in t.polymorphic_tasks(): for taskelement in task.get_elements(polymorphic=False): obj = taskelement.obj object_to_original_task[obj] = task for dep in deps_by_targettask.get(task, []): # is this dependency involved in one of the cycles ? if not dependency_in_cycles(dep): continue (processor, targettask) = (dep.processor, dep.targettask) isdelete = taskelement.isdelete # list of dependent objects from this object childlist = dep.get_object_dependencies(obj, trans, passive=True) if childlist is None: continue # the task corresponding to saving/deleting of those dependent objects childtask = trans.get_task_by_mapper( processor.mapper.primary_mapper()) childlist = childlist.added_items( ) + childlist.unchanged_items( ) + childlist.deleted_items() for o in childlist: # other object is None. this can occur if the relationship is many-to-one # or one-to-one, and None was set. the "removed" object will be picked # up in this iteration via the deleted_items() part of the collection. if o is None: continue # the other object is not in the UOWTransaction ! but if we are many-to-one, # we need a task in order to attach dependency operations, so establish a "listonly" # task if not childtask.contains_object(o, polymorphic=True): childtask.append(o, listonly=True) object_to_original_task[o] = childtask # create a tuple representing the "parent/child" whosdep = dep.whose_dependent_on_who(obj, o) if whosdep is not None: # append the tuple to the partial ordering. tuples.append(whosdep) # create a UOWDependencyProcessor representing this pair of objects. # append it to a UOWTask if whosdep[0] is obj: get_dependency_task(whosdep[0], dep).append( whosdep[0], isdelete=isdelete) else: get_dependency_task(whosdep[0], dep).append( whosdep[1], isdelete=isdelete) else: get_dependency_task(obj, dep).append( obj, isdelete=isdelete) #print "TUPLES", tuples head = DependencySorter(tuples, allobjects).sort() if head is None: return None #print str(head) # create a tree of UOWTasks corresponding to the tree of object instances # created by the DependencySorter def make_task_tree(node, parenttask, nexttasks): originating_task = object_to_original_task[node.item] t = nexttasks.get(originating_task, None) if t is None: t = UOWTask(self.uowtransaction, originating_task.mapper, circular_parent=self) nexttasks[originating_task] = t parenttask.append( None, listonly=False, isdelete=originating_task.objects[node.item].isdelete, childtask=t) t.append(node.item, originating_task.objects[node.item].listonly, isdelete=originating_task.objects[node.item].isdelete) if dependencies.has_key(node.item): for depprocessor, deptask in dependencies[ node.item].iteritems(): t.cyclical_dependencies.add(depprocessor.branch(deptask)) nd = {} for n in node.children: t2 = make_task_tree(n, t, nd) return t # this is the new "circular" UOWTask which will execute in place of "self" t = UOWTask(self.uowtransaction, self.mapper, circular_parent=self) # stick the non-circular dependencies and child tasks onto the new # circular UOWTask [t.dependencies.add(d) for d in extradeplist] t.childtasks = self.childtasks make_task_tree(head, t, {}) #print t.dump() return t
def __init__(self, item): self.item = item self.dependencies = util.Set() self.children = [] self.cycles = None
# mapper/util.py # Copyright (C) 2005, 2006, 2007 Michael Bayer [email protected] # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php from amilib.sqlalchemy import sql, util, exceptions all_cascades = util.Set([ "delete", "delete-orphan", "all", "merge", "expunge", "save-update", "refresh-expire", "none" ]) class CascadeOptions(object): """keeps track of the options sent to relation().cascade""" def __init__(self, arg=""): values = util.Set([c.strip() for c in arg.split(',')]) self.delete_orphan = "delete-orphan" in values self.delete = "delete" in values or self.delete_orphan or "all" in values self.save_update = "save-update" in values or "all" in values self.merge = "merge" in values or "all" in values self.expunge = "expunge" in values or "all" in values # refresh_expire not really implemented as of yet #self.refresh_expire = "refresh-expire" in values or "all" in values for x in values: if x not in all_cascades: raise exceptions.ArgumentError("Invalid cascade option '%s'" % x)
def locate_dirty(self): return util.Set([ x for x in self.identity_map.values() if x not in self.deleted and attribute_manager.is_modified(x) ])