def _add(self, obj): """ Adds an object to the data store. It will automatically generate the type relationships for the the object as required and store the object itself. """ type_registry = self.type_registry query_args = {} invalidates_types = False if isinstance(obj, PersistableType): # object is a type; create the type and its hierarchy return self._update_types(obj) elif isinstance(obj, Relationship): # object is a relationship obj_type = type(obj) if obj_type in (IsA, DeclaredOn): invalidates_types = True query = get_create_relationship_query(obj, type_registry) else: # object is an instance obj_type = type(obj) type_id = get_type_id(obj_type) if type_id not in type_registry._types_in_db: raise TypeNotPersistedError(type_id) labels = type_registry.get_labels_for_type(obj_type) if labels: node_declaration = 'n:' + ':'.join(labels) else: node_declaration = 'n' query = """ MATCH (cls:PersistableType) WHERE cls.id = {type_id} CREATE (%s {props})-[:INSTANCEOF {rel_props}]->(cls) RETURN n """ % node_declaration query_args = { 'type_id': get_type_id(obj_type), 'rel_props': type_registry.object_to_dict( InstanceOf(None, None), for_db=True), } query_args['props'] = type_registry.object_to_dict( obj, for_db=True) (node_or_rel,) = next(self._execute(query, **query_args)) if invalidates_types: self.invalidate_type_system() set_store_for_object(obj, self) return obj
def _add(self, obj): """ Adds an object to the data store. It will automatically generate the type relationships for the the object as required and store the object itself. """ query_args = {} invalidates_types = False if isinstance(obj, PersistableType): # object is a type; create the type and its hierarchy return self._update_types(obj) elif obj is self.type_system: query = 'CREATE (n {props}) RETURN n' elif isinstance(obj, Relationship): # object is a relationship obj_type = type(obj) if obj_type in (IsA, DeclaredOn): invalidates_types = True query = get_create_relationship_query(obj, self.type_registry) else: # object is an instance obj_type = type(obj) type_id = get_type_id(obj_type) if type_id not in self.type_registry._types_in_db: raise TypeNotPersistedError(type_id) idx_name = get_index_name(PersistableType) query = ( 'START cls=node:%s(id={type_id}) ' 'CREATE (n {props}) -[:INSTANCEOF {rel_props}]-> cls ' 'RETURN n' ) % idx_name query_args = { 'type_id': get_type_id(obj_type), 'rel_props': self.type_registry.object_to_dict( InstanceOf(None, None), for_db=True), } query_args['props'] = self.type_registry.object_to_dict( obj, for_db=True) (node_or_rel,) = next(self._execute(query, **query_args)) if invalidates_types: self.invalidate_type_system() self._index_object(obj, node_or_rel) return obj
def deserialize(self, object_dict): # we don't need to do any translation here; we just need to # pop off any values for translatable fields during deserialization # and put them back afterwards try: type_id = object_dict['__type__'] except KeyError: raise DeserialisationError( 'properties "{}" missing __type__ key'.format(object_dict)) if type_id == get_type_id(PersistableType): return super(Manager, self).deserialize(object_dict) descriptor = self.type_registry.get_descriptor_by_id(type_id) translatables = {} # deserialize a copy so we don't mutate object_dict object_dict_copy = copy.copy(object_dict) for attr_name, attr_type in descriptor.attributes.items(): if isinstance(attr_type, TranslatableString): if attr_name not in object_dict_copy: continue translatables[attr_name] = object_dict_copy.pop(attr_name) obj = super(Manager, self).deserialize(object_dict_copy) for attr_name, value in translatables.items(): setattr(obj, attr_name, value) return obj
def _update_types(self, cls): query, objects, query_args = get_create_types_query( cls, self.type_system.id, self.type_registry) self._execute(query, **query_args) for obj in objects: type_id = get_type_id(obj) self.type_registry._types_in_db.add(type_id) type_constraints = self.type_registry.get_constraints_for_type(obj) for constraint_type_id, constraint_attr_name in type_constraints: self.query( """ CREATE CONSTRAINT ON (type:{type_id}) ASSERT type.{attr_name} IS UNIQUE """.format( type_id=constraint_type_id, attr_name=constraint_attr_name, ) ) # we can't tell whether the CREATE UNIQUE from get_create_types_query # will have any effect, so we must invalidate. self.invalidate_type_system() return cls
def get_message_id(manager, obj): unique_attrs = set() for cls, attr_name in manager.type_registry.get_unique_attrs(type(obj)): value = getattr(obj, attr_name) if value is not None: unique_attrs.add(( get_type_id(cls).lower(), # backwards compat; was index name attr_name, object_to_db_value(value), )) return json.dumps(sorted(unique_attrs))
def update_type(self, tpe, bases): """ Change the bases of the given ``tpe`` """ if not isinstance(tpe, PersistableType): raise UnsupportedTypeError("Object is not a PersistableType") if self.type_registry.is_static_type(tpe): raise CannotUpdateType("Type '{}' is defined in code and cannot" "be updated.".format(get_type_id(tpe))) descriptor = self.type_registry.get_descriptor(tpe) existing_attrs = dict_difference(descriptor.attributes, descriptor.declared_attributes) base_attrs = {} for base in bases: desc = self.type_registry.get_descriptor(base) base_attrs.update(desc.attributes) base_attrs = dict_difference(base_attrs, descriptor.declared_attributes) if existing_attrs != base_attrs: raise CannotUpdateType("Inherited attributes are not identical") match_clauses = [get_match_clause(tpe, 'type', self.type_registry)] create_clauses = [] query_args = {} for index, base in enumerate(bases): name = 'base_{}'.format(index) match = get_match_clause(base, name, self.type_registry) create = "type -[:ISA {%s_props}]-> %s" % (name, name) query_args["{}_props".format(name)] = {'base_index': index} match_clauses.append(match) create_clauses.append(create) query = join_lines( "MATCH", (match_clauses, ','), ", type -[r:ISA]-> ()", "DELETE r", "CREATE", (create_clauses, ','), "RETURN type") try: next(self._execute(query, **query_args)) self.invalidate_type_system() except StopIteration: raise CannotUpdateType("Type or bases not found in the database.") self.reload_types()
def test_amended_indexes(manager): with collector() as collected: class A(Entity): id = Integer(unique=True) class B(Entity): code = String(unique=True) manager.save_collected_classes(collected) amended_registry = get_type_registry_with_base_change( manager, 'B', ('A',)) # confirm that the "amended" indexes of B are now using both id and code amended_indexes = amended_registry.get_unique_attrs(B) assert {(get_type_id(cls), attr) for cls, attr in amended_indexes} == { ('A', 'id'), ('B', 'code') }
def test_type_system_reload(manager_factory, static_types): Thing = static_types['Thing'] manager_factory(skip_setup=True).destroy() manager1 = manager_factory() manager2 = manager_factory() manager1.save(Thing) manager2.reload_types() type_id = get_type_id(Thing) assert manager2.type_registry.get_class_by_id(type_id) == Thing Thing.cls_attr = "cls_attr" manager1.save(Thing) manager2.reload_types() descriptor = manager2.type_registry.get_descriptor_by_id(type_id) assert "cls_attr" in descriptor.class_attributes
def _update_types(self, cls): query, objects, query_args = get_create_types_query( cls, self.type_system, self.type_registry) nodes_or_rels = next(self._execute(query, **query_args)) for obj in objects: type_id = get_type_id(obj) self.type_registry._types_in_db.add(type_id) if is_indexable(obj): index_name = get_index_name(obj) self._conn.get_or_create_index(neo4j.Node, index_name) for obj, node_or_rel in zip(objects, nodes_or_rels): self._index_object(obj, node_or_rel) # we can't tell whether the CREATE UNIQUE from get_create_types_query # will have any effect, so we must invalidate. self.invalidate_type_system() return cls
def test_amended_indexes_same_attr_name(manager): with collector() as collected: class A(Entity): id = Integer(unique=True) class B(Entity): id = String(unique=True) class C(A): pass manager.save_collected_classes(collected) amended_registry = get_type_registry_with_base_change( manager, 'C', ('A', 'B')) # confirm that the "amended" indexes of C are still just A.id amended_indexes = amended_registry.get_unique_attrs(C) assert {(get_type_id(cls), attr) for cls, attr in amended_indexes} == { ('A', 'id'), }
def get_by_unique_attr(self, cls, attr_name, values): """Bulk load entities from a list of values for a unique attribute Returns: A generator (obj1, obj2, ...) corresponding to the `values` list If any values are missing in the index, the corresponding obj is None """ if not hasattr(cls, attr_name): raise ValueError("{} has no attribute {}".format(cls, attr_name)) registry = self.type_registry for declaring_cls, attr in registry.get_unique_attrs(cls): if attr == attr_name: break else: raise ValueError("{}.{} is not unique".format(cls, attr_name)) type_id = get_type_id(cls) query = "MATCH (n:%(label)s {%(attr)s: {id}}) RETURN n" % { 'label': type_id, 'attr': attr_name, } batch = neo4j.ReadBatch(self._conn) for value in values: db_value = object_to_db_value(value) batch.append_cypher(query, params={'id': db_value}) # When upgrading to py2neo 1.6, consider changing this to batch.stream batch_result = batch.submit() # `batch_result` is a list of either one element lists (for matches) # or empty lists. Unpack to flatten (and hydrate to Kaiso objects) result = (self._convert_value(row) for row in batch_result) return result
def get_context(manager, obj, attribute_name): type_id = get_type_id(type(obj)) return "taal:kaiso_field:{}:{}".format(type_id, attribute_name)
def _update(self, persistable, existing, changes): registry = self.type_registry for _, index_attr, _ in registry.get_index_entries(existing): if index_attr in changes: raise NotImplementedError( "We currently don't support changing unique attributes") set_clauses = ', '.join([ 'n.%s={%s}' % (key, key) for key, value in changes.items() if not isinstance(value, dict) ]) if set_clauses: set_clauses = 'SET %s' % set_clauses else: set_clauses = '' if isinstance(persistable, type): query_args = {'type_id': get_type_id(persistable)} class_attr_changes = {k: v for k, v in changes.items() if k != 'attributes'} query_args.update(class_attr_changes) where = [] descr = registry.get_descriptor(persistable) for attr_name in descr.declared_attributes.keys(): where.append('attr.name = {attr_%s}' % attr_name) query_args['attr_%s' % attr_name] = attr_name if where: where = ' OR '.join(where) where = 'WHERE not(%s)' % where else: where = '' index_name = get_index_name(PersistableType) query = join_lines( 'START n=node:%s(id={type_id})' % index_name, set_clauses, 'MATCH attr -[r:DECLAREDON]-> n', where, 'DELETE attr, r', 'RETURN n', ) self._update_types(persistable) else: start_clause = get_start_clause(existing, 'n', registry) query = None if isinstance(persistable, Relationship): old_start = existing.start old_end = existing.end new_start = changes.pop('start', old_start) new_end = changes.pop('end', old_end) if old_start != new_start or old_end != new_end: start_clause = '%s, %s, %s, %s, %s' % ( start_clause, get_start_clause(old_start, 'old_start', registry), get_start_clause(old_end, 'old_end', registry), get_start_clause(new_start, 'new_start', registry), get_start_clause(new_end, 'new_end', registry) ) rel_props = registry.object_to_dict(persistable) query = join_lines( 'START %s' % start_clause, 'DELETE n', 'CREATE new_start -[r:%s {rel_props}]-> new_end' % ( rel_props['__type__'].upper() ), 'RETURN r' ) query_args = {'rel_props': rel_props} if query is None: query = join_lines( 'START %s' % start_clause, set_clauses, 'RETURN n' ) query_args = changes try: (result,) = next(self._execute(query, **query_args)) except StopIteration: # this can happen, if no attributes where changed on a type result = persistable if isinstance(persistable, Relationship): self._index_object(persistable, result) return result
def get_match_clause(obj, name, type_registry): """Return node lookup by index for a match clause using unique attributes Args: obj: An object to create an index lookup. name: The name of the object in the query. Returns: A string with an index lookup for a cypher MATCH clause """ if isinstance(obj, PersistableType): value = get_type_id(obj) return '({name}:PersistableType {{id: {value}}})'.format( name=name, value=json.dumps(object_to_db_value(value)), ) if isinstance(obj, Relationship): if obj.start is None or obj.end is None: raise NoUniqueAttributeError( "{} is missing a start or end node".format(obj) ) neo4j_rel_name = get_neo4j_relationship_name(type(obj)) start_name = '{}__start'.format(name) end_name = '{}__end'.format(name) query = """ {start_clause}, {end_clause}, ({start_name})-[{name}:{neo4j_rel_name}]->({end_name}) """.format( name=name, start_clause=get_match_clause( obj.start, start_name, type_registry, ), end_clause=get_match_clause( obj.end, end_name, type_registry, ), start_name=start_name, end_name=end_name, neo4j_rel_name=neo4j_rel_name, ) return dedent(query) match_params = {} label_classes = set() for cls, attr_name in type_registry.get_unique_attrs(type(obj)): value = getattr(obj, attr_name) if value is not None: label_classes.add(cls) match_params[attr_name] = value if not match_params: raise NoUniqueAttributeError( "{} doesn't have any unique attributes".format(obj) ) match_params_string = inline_parameter_map( dict_to_db_values_dict(match_params) ) labels = ':'.join(get_type_id(cls) for cls in label_classes) return '({name}:{labels} {match_params_string})'.format( name=name, labels=labels, attr_name=attr_name, match_params_string=match_params_string, )
def get_create_types_query(cls, type_system_id, type_registry): """ Returns a CREATE UNIQUE query for an entire type hierarchy. Includes statements that create each type's attributes. Args: cls: An object to create a type hierarchy for. Returns: A tuple containing: (cypher query, classes to create nodes for, the object names). """ hierarchy_lines = [] set_lines = [] classes = {} query_args = { 'type_system_id': type_system_id, } # filter type relationships that we want to persist type_relationships = [] for cls1, rel_cls_idx, cls2 in get_type_relationships(cls): if issubclass(cls2, AttributedBase): type_relationships.append((cls1, rel_cls_idx, cls2)) # process type relationships is_first = True isa_props_counter = 0 for cls1, (rel_cls, base_idx), cls2 in type_relationships: name1 = cls1.__name__ type1 = type(cls1).__name__ node_for_create = ( '(`%(name)s`:%(type)s {' '__type__: {%(name)s__type}, ' 'id: {%(name)s__id}' '})' ) % { 'name': name1, 'type': type1, } create_statement = 'MERGE %s' % node_for_create node_for_ref = '(`%s`)' % name1 if name1 not in classes: classes[name1] = cls1 hierarchy_lines.append(create_statement) if is_first: is_first = False ln = 'MERGE (ts) -[:DEFINES]-> %s' % node_for_ref else: name2 = cls2.__name__ classes[name2] = cls2 rel_name = get_type_id(rel_cls) rel_type = rel_name.upper() prop_name = '%s_%d' % (rel_name, isa_props_counter) isa_props_counter += 1 props = type_registry.object_to_dict(IsA(base_index=base_idx)) query_args[prop_name] = props ln = 'MERGE %s -[%s:%s]-> (`%s`)' % ( node_for_ref, prop_name, rel_type, name2) set_lines.append('SET `%s` = {%s}' % (prop_name, prop_name)) hierarchy_lines.append(ln) # process attributes for name, cls in classes.items(): descriptor = type_registry.get_descriptor(cls) attributes = descriptor.declared_attributes for attr_name, attr in attributes.iteritems(): attr_dict = type_registry.object_to_dict( attr, for_db=True) attr_dict['name'] = attr_name node_contents = [] for entry, value in attr_dict.iteritems(): key = "%s_%s__%s" % (name, attr_name, entry) node_contents.append('%s: {%s}' % (entry, key)) query_args[key] = value ln = 'MERGE ({%s}) -[:DECLAREDON]-> (`%s`)' % ( ', '.join(node_contents), name) hierarchy_lines.append(ln) # processing class attributes for key, cls in classes.iteritems(): # all attributes of the class to be set via the query cls_props = type_registry.object_to_dict(cls, for_db=True) query_args['%s_props' % key] = cls_props set_lines.append('SET `%s` = {%s_props}' % (key, key)) # attributes which uniquely identify the class itself # these are used in the CREATE UNIQUE part of the query query_args['%s__id' % key] = cls_props['id'] query_args['%s__type' % key] = cls_props['__type__'] quoted_names = ('`{}`'.format(cls) for cls in classes.keys()) query = join_lines( 'MATCH (ts:TypeSystem) WHERE ts.id = {type_system_id}', (hierarchy_lines, ''), (set_lines, ''), 'RETURN %s' % ', '.join(quoted_names) ) return query, classes.values(), query_args
def _get_changes(self, persistable): changes = {} existing = None obj_type = type(persistable) registry = self.type_registry if isinstance(persistable, PersistableType): # this is a class, we need to get it and it's attrs idx_name = get_index_name(PersistableType) self._conn.get_or_create_index(neo4j.Node, idx_name) type_id = get_type_id(persistable) query_args = { 'type_id': type_id } query = join_lines( 'START cls=node:%s(id={type_id})' % idx_name, 'MATCH attr -[:DECLAREDON*0..]-> cls', 'RETURN cls, collect(attr.name?)' ) # don't use self.query since we don't want to convert the py2neo # node into an object rows = self._execute(query, **query_args) cls_node, attrs = next(rows, (None, None)) if cls_node is None: # have not found the cls return None, {} existing_cls_attrs = cls_node.get_properties() new_cls_attrs = registry.object_to_dict(persistable) # If any existing keys in "new" are missing in "old", add `None`s. # Unlike instance attributes, we just need to remove the properties # from the node, which we can achieve by setting the values to None for key in set(existing_cls_attrs) - set(new_cls_attrs): new_cls_attrs[key] = None changes = get_changes(old=existing_cls_attrs, new=new_cls_attrs) attrs = set(attrs) modified_attrs = {} descr = registry.get_descriptor(persistable) for name, attr in descr.declared_attributes.items(): if name not in attrs: modified_attrs[name] = attr del_attrs = set(attrs) for name in Descriptor(persistable).attributes.keys(): del_attrs.discard(name) for name in del_attrs: modified_attrs[name] = None if modified_attrs: changes['attributes'] = modified_attrs # we want to return the existing class existing = registry.get_descriptor_by_id(type_id).cls else: existing = self.get(obj_type, **get_attr_filter(persistable, registry)) if existing is not None: existing_props = registry.object_to_dict(existing) props = registry.object_to_dict(persistable) if isinstance(persistable, Relationship): # if the relationship has endoints, also consider # whether those have changed for rel_attr in ['start', 'end']: new = getattr(persistable, rel_attr, None) if new is None: continue ex_rel_attr = getattr(existing, rel_attr) ex_rel_identifier = get_attr_filter(ex_rel_attr, registry) if new != ex_rel_identifier: props[rel_attr] = new if existing_props == props: return existing, {} changes = get_changes(old=existing_props, new=props) return existing, changes
def get(self, cls, **attr_filter): attr_filter = dict_to_db_values_dict(attr_filter) if not attr_filter: return None query_args = {} indexes = attr_filter.items() if issubclass(cls, (Relationship, PersistableType)): idx_name = get_index_name(cls) idx_key, idx_value = indexes[0] if issubclass(cls, Relationship): self._conn.get_or_create_index(neo4j.Relationship, idx_name) start_func = 'relationship' else: self._conn.get_or_create_index(neo4j.Node, idx_name) start_func = 'node' query = 'START nr = %s:%s(%s={idx_value}) RETURN nr' % ( start_func, idx_name, idx_key) query_args['idx_value'] = idx_value elif cls is TypeSystem: idx_name = get_index_name(TypeSystem) query = join_lines( 'START ts=node:%s(id={idx_value})' % idx_name, 'RETURN ts' ) query_args['idx_value'] = self.type_system.id else: idx_where = [] for key, value in indexes: idx_where.append('n.%s! = {%s}' % (key, key)) query_args[key] = value idx_where = ' or '.join(idx_where) idx_name = get_index_name(TypeSystem) query = join_lines( 'START root=node:%s(id={idx_value})' % idx_name, 'MATCH ', ' n -[:INSTANCEOF]-> ()', ' -[:ISA*0..]-> tpe -[:ISA*0..]-> () <-[:DEFINES]- root', 'WHERE %s' % idx_where, ' AND tpe.id = {tpe_id}', 'RETURN n', ) query_args['idx_value'] = self.type_system.id type_id = get_type_id(cls) query_args['tpe_id'] = type_id found = [node for (node,) in self._execute(query, **query_args)] if not found: return None # all the nodes returned should be the same first = found[0] for node in found: if node.id != first.id: raise UniqueConstraintError(( "Multiple nodes ({}) found for unique lookup for " "{}").format(found, cls)) obj = self._convert_value(first) return obj
def _update(self, persistable, existing, changes): registry = self.type_registry set_clauses = ', '.join([ 'n.%s={%s}' % (key, key) for key, value in changes.items() if not isinstance(value, dict) ]) if set_clauses: set_clauses = 'SET %s' % set_clauses else: set_clauses = '' if isinstance(persistable, type): query_args = {'type_id': get_type_id(persistable)} class_attr_changes = {k: v for k, v in changes.items() if k != 'attributes'} query_args.update(class_attr_changes) where = [] descr = registry.get_descriptor(persistable) for attr_name in descr.declared_attributes.keys(): where.append('attr.name = {attr_%s}' % attr_name) query_args['attr_%s' % attr_name] = attr_name if where: where = ' OR '.join(where) where = 'WHERE not(%s)' % where else: where = '' query = join_lines( 'MATCH (n:PersistableType)', 'WHERE n.id = {type_id}', set_clauses, 'WITH n', 'MATCH attr -[r:DECLAREDON]-> n', where, 'DELETE attr, r', 'RETURN n', ) self._update_types(persistable) else: match_clause = get_match_clause(existing, 'n', registry) query = join_lines( 'MATCH %s' % match_clause, set_clauses, 'RETURN n' ) query_args = changes try: (result,) = next(self._execute(query, **query_args)) except StopIteration: # this can happen, if no attributes where changed on a type result = persistable return result
def _get_changes(self, persistable): changes = {} existing = None registry = self.type_registry if isinstance(persistable, PersistableType): if issubclass(persistable, Relationship): # not stored in the db; must be static return None, {} query = """ MATCH {} OPTIONAL MATCH (attr)-[:DECLAREDON*0..]->(cls) RETURN cls, collect(attr.name) """.format(get_match_clause(persistable, 'cls', registry)) # don't use self.query since we don't want to convert the py2neo # node into an object rows = self._execute(query) cls_node, attrs = next(rows, (None, None)) if cls_node is None: # have not found the cls return None, {} existing_cls_attrs = cls_node._properties # Make sure we get a clean view of current data. registry.refresh_type(persistable) new_cls_attrs = registry.object_to_dict(persistable) # If any existing keys in "new" are missing in "old", add `None`s. # Unlike instance attributes, we just need to remove the properties # from the node, which we can achieve by setting the values to None for key in set(existing_cls_attrs) - set(new_cls_attrs): new_cls_attrs[key] = None changes = get_changes(old=existing_cls_attrs, new=new_cls_attrs) attrs = set(attrs) modified_attrs = {} descr = registry.get_descriptor(persistable) for name, attr in descr.declared_attributes.items(): if name not in attrs: modified_attrs[name] = attr del_attrs = set(attrs) for name in Descriptor(persistable).attributes.keys(): del_attrs.discard(name) for name in del_attrs: modified_attrs[name] = None if modified_attrs: changes['attributes'] = modified_attrs # we want to return the existing class type_id = get_type_id(persistable) existing = registry.get_descriptor_by_id(type_id).cls else: try: query = 'MATCH {} RETURN obj'.format( get_match_clause(persistable, 'obj', registry) ) except NoUniqueAttributeError: existing = None else: existing = self.query_single(query) if existing is not None: existing_props = registry.object_to_dict(existing) props = registry.object_to_dict(persistable) if existing_props == props: return existing, {} changes = get_changes(old=existing_props, new=props) return existing, changes