def _results(self):
assert log(1, 'called Intersection')
resultset = None
for query in self.queries:
assert log(2, 'resultset is', resultset)
assert log(2, 'intersecting with', query)
s = set(query())
if resultset is None:
resultset = s
else:
resultset = resultset.intersection(s)
assert log(2, 'resultset is finally', resultset)
return results(frozenset(resultset))
def copy(src_filename, dest_filename, dest_backend_name, dest_backend_args={}):
"""Copy internal structures verbatim from a source database to a
new destination database.
To see progress of the copy operation, turn on tracing as
described in `schevo.trace`
- `src_filename`: Filename of the source database. Schevo must be
able to open the source database using backend autodetection,
and by using default backend arguments.
- `dest_filename`: Filename of the destination database. This file
may exist if it is in the format used by the destination
backend, but must not already contain a Schevo database.
- `dest_backend_name`: Name of the backend to use when creating
the destination database.
- `dest_backend_args`: (optional) Arguments to pass to the
backend.
"""
src_backend = new_backend(src_filename)
# Make sure the source backend is in the proper format.
assert log(1, 'Checking source', src_filename)
src_root = src_backend.get_root()
if 'SCHEVO' not in src_root:
src_backend.close()
raise DatabaseDoesNotExist(src_filename)
current_format = src_root['SCHEVO']['format']
required_format = 2
if current_format != required_format:
src_backend.close()
raise DatabaseFormatMismatch(current_format, required_format)
# Make sure the destination backend does not have a database.
assert log(1, 'Checking destination', src_filename)
dest_backend = new_backend(
dest_filename, dest_backend_name, dest_backend_args)
dest_root = dest_backend.get_root()
if 'SCHEVO' in dest_root:
src_backend.close()
dest_backend.close()
raise DatabaseAlreadyExists(dest_filename)
assert log(1, 'Start copying structures.')
d_btree = dest_backend.BTree
d_pdict = dest_backend.PDict
d_plist = dest_backend.PList
s_btree = src_backend.BTree
assert log(2, 'Creating SCHEVO key.')
src_SCHEVO = src_root['SCHEVO']
dest_SCHEVO = dest_root['SCHEVO'] = d_pdict()
assert log(2, 'Copying lightweight structures.')
if 'label' in src_SCHEVO:
dest_SCHEVO['label'] = src_SCHEVO['label']
dest_SCHEVO['format'] = 2
dest_SCHEVO['version'] = src_SCHEVO['version']
dest_SCHEVO['schema_source'] = src_SCHEVO['schema_source']
dest_SCHEVO['extent_name_id'] = d_pdict(
src_SCHEVO['extent_name_id'].iteritems())
assert log(2, 'Copying extents.')
dest_extents = dest_SCHEVO['extents'] = d_pdict()
def copy_btree(src):
"""Used for copying indices structure."""
copy = d_btree()
for key, value in src.iteritems():
if isinstance(value, s_btree):
value = copy_btree(value)
copy[key] = value
return copy
for extent_id, src_extent in src_SCHEVO['extents'].iteritems():
extent_name = src_extent['name']
assert log(2, 'Creating extent', extent_name)
dest_extent = dest_extents[extent_id] = d_pdict()
assert log(2, 'Copying lightweight structures for', extent_name)
dest_extent['entity_field_ids'] = src_extent['entity_field_ids']
dest_extent['field_id_name'] = d_pdict(
src_extent['field_id_name'].iteritems())
dest_extent['field_name_id'] = d_pdict(
src_extent['field_name_id'].iteritems())
dest_extent['id'] = src_extent['id']
dest_extent['len'] = src_extent['len']
dest_extent['name'] = src_extent['name']
dest_extent['next_oid'] = src_extent['next_oid']
assert log(2, 'Copying', len(src_extent['entities']), 'entities in',
extent_name)
dest_entities = dest_extent['entities'] = d_btree()
for entity_oid, src_entity in src_extent['entities'].iteritems():
assert log(3, 'Copying', entity_oid)
dest_entity = dest_entities[entity_oid] = d_pdict()
dest_entity['rev'] = src_entity['rev']
dest_entity['fields'] = d_pdict(src_entity['fields'].iteritems())
dest_entity['link_count'] = src_entity['link_count']
src_links = src_entity['links']
dest_links = dest_entity['links'] = d_pdict()
for key, value in src_links.iteritems():
links = dest_links[key] = d_btree()
# Do not use update() since schevo.store, durus, and zodb
# all have slightly different, incompatible, versions.
for k, v in src_links[key].iteritems():
links[k] = v
dest_entity['related_entities'] = d_pdict(
src_entity['related_entities'].iteritems())
assert log(2, 'Copying indices for', extent_name)
dest_extent['index_map'] = d_pdict(
(k, d_plist(v))
for k, v
in src_extent['index_map'].iteritems()
)
dest_extent['normalized_index_map'] = d_pdict(
(k, d_plist(v))
for k, v
in src_extent['normalized_index_map'].iteritems()
)
dest_indices = dest_extent['indices'] = d_btree()
for index_spec, src_index_data in src_extent['indices'].iteritems():
unique, src_index_tree = src_index_data
dest_indices[index_spec] = (unique, copy_btree(src_index_tree))
assert log(2, 'Done copying', extent_name, '-- committing to disk')
dest_backend.commit()
# Finalize.
assert log(1, 'Close source.')
src_backend.close()
assert log(1, 'Pack destination.')
dest_backend.pack()
assert log(1, 'Close destination.')
dest_backend.close()
def set_trace(option, opt, value, parser):
trace.print_history(value)
trace.monitor_level = value
trace.log(1, 'Tracing level set to', value)
def _find_entity_oids(self, extent_name, **criteria):
"""Return list of entity OIDs matching given field value(s)."""
assert log(1, extent_name, criteria)
extent_map = self._extent_map(extent_name)
entity_maps = extent_map['entities']
EntityClass = self._entity_classes[extent_name]
if not criteria:
# Return all of them.
assert log(2, 'Return all oids.')
return list(entity_maps.keys())
extent_name_id = self._extent_name_id
indices = extent_map['indices']
normalized_index_map = extent_map['normalized_index_map']
entity_field_ids = extent_map['entity_field_ids']
field_name_id = extent_map['field_name_id']
# Convert from field_name:value to field_id:value.
field_id_value = {}
field_spec = EntityClass._field_spec
for field_name, value in criteria.iteritems():
try:
field_id = field_name_id[field_name]
except KeyError:
raise error.FieldDoesNotExist(extent_name, field_name)
# Dereference if it's an entity field and not UNASSIGNED.
if field_id in entity_field_ids and isinstance(value, Entity):
# Dereference entity.
other_extent_id = extent_name_id[value._extent.name]
other_oid = value._oid
value = (other_extent_id, other_oid)
else:
# Create a field to convert the value.
FieldClass = field_spec[field_name]
field = FieldClass(None, None)
value = field.convert(value)
field_id_value[field_id] = value
# First, see if the fields given can be found in an index. If
# so, use the index to return matches.
#
# XXX: Should be updated to use partial search via an index,
# and brute-force on the subset found via that index.
field_ids = tuple(sorted(field_id_value))
assert log(3, 'field_ids', field_ids)
len_field_ids = len(field_ids)
index_spec = None
if field_ids in normalized_index_map:
for spec in normalized_index_map[field_ids]:
if len(spec) == len_field_ids:
index_spec = spec
break
results = []
if index_spec is not None:
# We found an index to use.
assert log(2, 'Use index spec:', index_spec)
unique, branch = indices[index_spec]
match = True
for field_id in index_spec:
field_value = field_id_value[field_id]
if field_value not in branch:
# No matches found.
match = False
break
branch = branch[field_value]
if match:
# Now we're at a leaf that matches all of the
# criteria, so return the OIDs in that leaf.
results = list(branch.keys())
else:
# Fields aren't indexed, so use brute force.
assert log(2, 'Use brute force.')
append = results.append
for oid, entity_map in entity_maps.iteritems():
fields = entity_map['fields']
match = True
for field_id, value in field_id_value.iteritems():
if fields.get(field_id, UNASSIGNED) != value:
match = False
break
if match:
append(oid)
assert log(2, 'Result count', len(results))
return results
