def _results(self): assert log(1, 'called Intersection') resultset = None for query in self.queries: assert log(2, 'resultset is', resultset) assert log(2, 'intersecting with', query) s = set(query()) if resultset is None: resultset = s else: resultset = resultset.intersection(s) assert log(2, 'resultset is finally', resultset) return results(frozenset(resultset))
def copy(src_filename, dest_filename, dest_backend_name, dest_backend_args={}): """Copy internal structures verbatim from a source database to a new destination database. To see progress of the copy operation, turn on tracing as described in `schevo.trace` - `src_filename`: Filename of the source database. Schevo must be able to open the source database using backend autodetection, and by using default backend arguments. - `dest_filename`: Filename of the destination database. This file may exist if it is in the format used by the destination backend, but must not already contain a Schevo database. - `dest_backend_name`: Name of the backend to use when creating the destination database. - `dest_backend_args`: (optional) Arguments to pass to the backend. """ src_backend = new_backend(src_filename) # Make sure the source backend is in the proper format. assert log(1, 'Checking source', src_filename) src_root = src_backend.get_root() if 'SCHEVO' not in src_root: src_backend.close() raise DatabaseDoesNotExist(src_filename) current_format = src_root['SCHEVO']['format'] required_format = 2 if current_format != required_format: src_backend.close() raise DatabaseFormatMismatch(current_format, required_format) # Make sure the destination backend does not have a database. assert log(1, 'Checking destination', src_filename) dest_backend = new_backend( dest_filename, dest_backend_name, dest_backend_args) dest_root = dest_backend.get_root() if 'SCHEVO' in dest_root: src_backend.close() dest_backend.close() raise DatabaseAlreadyExists(dest_filename) assert log(1, 'Start copying structures.') d_btree = dest_backend.BTree d_pdict = dest_backend.PDict d_plist = dest_backend.PList s_btree = src_backend.BTree assert log(2, 'Creating SCHEVO key.') src_SCHEVO = src_root['SCHEVO'] dest_SCHEVO = dest_root['SCHEVO'] = d_pdict() assert log(2, 'Copying lightweight structures.') if 'label' in src_SCHEVO: dest_SCHEVO['label'] = src_SCHEVO['label'] dest_SCHEVO['format'] = 2 dest_SCHEVO['version'] = src_SCHEVO['version'] dest_SCHEVO['schema_source'] = src_SCHEVO['schema_source'] dest_SCHEVO['extent_name_id'] = d_pdict( src_SCHEVO['extent_name_id'].iteritems()) assert log(2, 'Copying extents.') dest_extents = dest_SCHEVO['extents'] = d_pdict() def copy_btree(src): """Used for copying indices structure.""" copy = d_btree() for key, value in src.iteritems(): if isinstance(value, s_btree): value = copy_btree(value) copy[key] = value return copy for extent_id, src_extent in src_SCHEVO['extents'].iteritems(): extent_name = src_extent['name'] assert log(2, 'Creating extent', extent_name) dest_extent = dest_extents[extent_id] = d_pdict() assert log(2, 'Copying lightweight structures for', extent_name) dest_extent['entity_field_ids'] = src_extent['entity_field_ids'] dest_extent['field_id_name'] = d_pdict( src_extent['field_id_name'].iteritems()) dest_extent['field_name_id'] = d_pdict( src_extent['field_name_id'].iteritems()) dest_extent['id'] = src_extent['id'] dest_extent['len'] = src_extent['len'] dest_extent['name'] = src_extent['name'] dest_extent['next_oid'] = src_extent['next_oid'] assert log(2, 'Copying', len(src_extent['entities']), 'entities in', extent_name) dest_entities = dest_extent['entities'] = d_btree() for entity_oid, src_entity in src_extent['entities'].iteritems(): assert log(3, 'Copying', entity_oid) dest_entity = dest_entities[entity_oid] = d_pdict() dest_entity['rev'] = src_entity['rev'] dest_entity['fields'] = d_pdict(src_entity['fields'].iteritems()) dest_entity['link_count'] = src_entity['link_count'] src_links = src_entity['links'] dest_links = dest_entity['links'] = d_pdict() for key, value in src_links.iteritems(): links = dest_links[key] = d_btree() # Do not use update() since schevo.store, durus, and zodb # all have slightly different, incompatible, versions. for k, v in src_links[key].iteritems(): links[k] = v dest_entity['related_entities'] = d_pdict( src_entity['related_entities'].iteritems()) assert log(2, 'Copying indices for', extent_name) dest_extent['index_map'] = d_pdict( (k, d_plist(v)) for k, v in src_extent['index_map'].iteritems() ) dest_extent['normalized_index_map'] = d_pdict( (k, d_plist(v)) for k, v in src_extent['normalized_index_map'].iteritems() ) dest_indices = dest_extent['indices'] = d_btree() for index_spec, src_index_data in src_extent['indices'].iteritems(): unique, src_index_tree = src_index_data dest_indices[index_spec] = (unique, copy_btree(src_index_tree)) assert log(2, 'Done copying', extent_name, '-- committing to disk') dest_backend.commit() # Finalize. assert log(1, 'Close source.') src_backend.close() assert log(1, 'Pack destination.') dest_backend.pack() assert log(1, 'Close destination.') dest_backend.close()
def set_trace(option, opt, value, parser): trace.print_history(value) trace.monitor_level = value trace.log(1, 'Tracing level set to', value)
def _find_entity_oids(self, extent_name, **criteria): """Return list of entity OIDs matching given field value(s).""" assert log(1, extent_name, criteria) extent_map = self._extent_map(extent_name) entity_maps = extent_map['entities'] EntityClass = self._entity_classes[extent_name] if not criteria: # Return all of them. assert log(2, 'Return all oids.') return list(entity_maps.keys()) extent_name_id = self._extent_name_id indices = extent_map['indices'] normalized_index_map = extent_map['normalized_index_map'] entity_field_ids = extent_map['entity_field_ids'] field_name_id = extent_map['field_name_id'] # Convert from field_name:value to field_id:value. field_id_value = {} field_spec = EntityClass._field_spec for field_name, value in criteria.iteritems(): try: field_id = field_name_id[field_name] except KeyError: raise error.FieldDoesNotExist(extent_name, field_name) # Dereference if it's an entity field and not UNASSIGNED. if field_id in entity_field_ids and isinstance(value, Entity): # Dereference entity. other_extent_id = extent_name_id[value._extent.name] other_oid = value._oid value = (other_extent_id, other_oid) else: # Create a field to convert the value. FieldClass = field_spec[field_name] field = FieldClass(None, None) value = field.convert(value) field_id_value[field_id] = value # First, see if the fields given can be found in an index. If # so, use the index to return matches. # # XXX: Should be updated to use partial search via an index, # and brute-force on the subset found via that index. field_ids = tuple(sorted(field_id_value)) assert log(3, 'field_ids', field_ids) len_field_ids = len(field_ids) index_spec = None if field_ids in normalized_index_map: for spec in normalized_index_map[field_ids]: if len(spec) == len_field_ids: index_spec = spec break results = [] if index_spec is not None: # We found an index to use. assert log(2, 'Use index spec:', index_spec) unique, branch = indices[index_spec] match = True for field_id in index_spec: field_value = field_id_value[field_id] if field_value not in branch: # No matches found. match = False break branch = branch[field_value] if match: # Now we're at a leaf that matches all of the # criteria, so return the OIDs in that leaf. results = list(branch.keys()) else: # Fields aren't indexed, so use brute force. assert log(2, 'Use brute force.') append = results.append for oid, entity_map in entity_maps.iteritems(): fields = entity_map['fields'] match = True for field_id, value in field_id_value.iteritems(): if fields.get(field_id, UNASSIGNED) != value: match = False break if match: append(oid) assert log(2, 'Result count', len(results)) return results