forked from nascheme/durus
/
storage.py
177 lines (149 loc) · 5.27 KB
/
storage.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
"""
$URL$
$Id$
"""
from collections import deque
from durus.serialize import unpack_record, split_oids, extract_class_name
from durus.utils import int8_to_str
import durus.connection
class Storage (object):
"""
This is the interface that Connection requires for Storage.
"""
def __init__(self):
raise RuntimeError("Storage is abstract")
def load(self, oid):
"""Return the record for this oid.
Raises a KeyError if there is no such record.
May also raise a ReadConflictError.
"""
raise NotImplementedError
def begin(self):
"""
Begin a commit.
"""
raise NotImplementedError
def store(self, oid, record):
"""Include this record in the commit underway."""
raise NotImplementedError
def end(self, handle_invalidations=None):
"""Conclude a commit.
This may raise a ConflictError.
"""
raise NotImplementedError
def sync(self):
"""() -> [oid:str]
Return a list of oids that should be invalidated.
"""
raise NotImplementedError
def new_oid(self):
"""() -> oid:str
Return an unused oid. Used by Connection for serializing new persistent
instances.
"""
raise NotImplementedError
def close(self):
"""Clean up as needed.
"""
def get_packer(self):
"""
Return an incremental packer (a generator), or None if this storage
does not support incremental packing.
Used by StorageServer.
"""
return None
def pack(self):
"""If this storage supports it, remove obsolete records."""
return None
def bulk_load(self, oids):
"""(oids:sequence(oid:str)) -> sequence(record:str)
"""
for oid in oids:
yield self.load(oid)
def gen_oid_record(self, start_oid=None, batch_size=100):
"""(start_oid:str = None, batch_size:int = 100) ->
sequence((oid:str, record:str))
Returns a generator for the sequence of (oid, record) pairs.
If a start_oid is given, the resulting sequence follows a
breadth-first traversal of the object graph, starting at the given
start_oid. This uses the storage's bulk_load() method because that
is faster in some cases. The batch_size argument sets the number
of object records loaded on each call to bulk_load().
If no start_oid is given, the sequence may include oids and records
that are not reachable from the root.
"""
if start_oid is None:
start_oid = durus.connection.ROOT_OID
todo = deque([start_oid])
seen = set()
while todo:
batch = []
while todo and len(batch) < batch_size:
oid = todo.popleft()
if oid not in seen:
batch.append(oid)
seen.add(oid)
for record in self.bulk_load(batch):
oid, data, refdata = unpack_record(record)
yield oid, record
for ref in split_oids(refdata):
if ref not in seen:
todo.append(ref)
def gen_referring_oid_record(storage, referred_oid):
"""(storage:Storage, referred_oid:str) -> sequence([oid:str, record:str])
Generate oid, record pairs for all objects that include a
reference to the `referred_oid`.
"""
for oid, record in storage.gen_oid_record():
if referred_oid in split_oids(unpack_record(record)[2]):
yield oid, record
def gen_oid_class(storage, *classes):
"""(storage:Storage, classes:(str)) ->
sequence([(oid:str, class_name:str)])
Generate a sequence of oid, class_name pairs.
If classes are provided, only output pairs for which the
class_name is in `classes`.
"""
for oid, record in storage.gen_oid_record():
class_name = extract_class_name(record)
if not classes or class_name in classes:
yield oid, class_name
def get_census(storage):
"""(storage:Storage) -> {class_name:str, instance_count:int}"""
result = {}
for oid, class_name in gen_oid_class(storage):
result[class_name] = result.get(class_name, 0) + 1
return result
def get_reference_index(storage):
"""(storage:Storage) -> {oid:str : [referring_oid:str]}
Return a full index giving the referring oids for each oid.
This might be large.
"""
result = {}
for oid, record in storage.gen_oid_record():
for ref in split_oids(unpack_record(record)[2]):
result.setdefault(ref, []).append(oid)
return result
class MemoryStorage (Storage):
"""
A concrete Storage that keeps everything in memory.
This may be useful for testing purposes.
"""
def __init__(self):
self.records = {}
self.transaction = None
self.oid = -1
def new_oid(self):
self.oid += 1
return int8_to_str(self.oid)
def load(self, oid):
return self.records[oid]
def begin(self):
self.transaction = {}
def store(self, oid, record):
self.transaction[oid] = record
def end(self, handle_invalidations=None):
self.records.update(self.transaction)
self.transaction = None
def sync(self):
return []