def check_record_pack_unpack(self):
oid = as_bytes('0'*8)
data = as_bytes('sample')
reflist = ['1'*8, '2'*8]
reflist = list(map(as_bytes, reflist))
refs = join_bytes(reflist)
result=unpack_record(pack_record(oid, data, refs))
assert result[0] == oid
assert result[1] == data
assert split_oids(result[2]) == reflist
assert split_oids('') == []
def gen_oid_record(self, start_oid=None, batch_size=100):
"""(start_oid:str = None, batch_size:int = 100) ->
sequence((oid:str, record:str))
Returns a generator for the sequence of (oid, record) pairs.
If a start_oid is given, the resulting sequence follows a
breadth-first traversal of the object graph, starting at the given
start_oid. This uses the storage's bulk_load() method because that
is faster in some cases. The batch_size argument sets the number
of object records loaded on each call to bulk_load().
If no start_oid is given, the sequence may include oids and records
that are not reachable from the root.
"""
if start_oid is None:
start_oid = durus.connection.ROOT_OID
todo = deque([start_oid])
seen = set()
while todo:
batch = []
while todo and len(batch) < batch_size:
oid = todo.popleft()
if oid not in seen:
batch.append(oid)
seen.add(oid)
for record in self.bulk_load(batch):
oid, data, refdata = unpack_record(record)
yield oid, record
for ref in split_oids(refdata):
if ref not in seen:
todo.append(ref)
def gen_reachable_records():
# we order the todo queue by file offset. The hope is that the
# packed file will be mostly the same as the old file in order
# to speed up the rsync delta process.
default_rank = 2**64
pack_todo = [(0, durus.connection.ROOT_OID)]
while pack_todo or self.pack_extra:
if self.pack_extra:
oid = self.pack_extra.pop()
# note we don't check 'index' because it could be an
# object that got updated since the pack began and in
# that case we have to write the new record to the pack
# file
else:
rank, oid = heapq.heappop(pack_todo)
if oid in index:
# we already wrote this object record
continue
record = self.load(oid)
oid2, data, refdata = unpack_record(record)
assert oid == oid2
# ensure we have records for objects referenced
for ref_oid in split_oids(refdata):
item = (self.index.get(ref_oid, default_rank), ref_oid)
heapq.heappush(pack_todo, item)
yield (oid, record)
def handle_B(self, s):
# bulk read of objects
number_of_oids = read_int4(s)
oid_str = read(s, 8 * number_of_oids)
oids = split_oids(oid_str)
for oid in oids:
self._send_load_response(s, oid)
def handle_B(self, s):
# bulk read of objects
number_of_oids = read_int4(s)
oid_str = read(s, 8 * number_of_oids)
oids = split_oids(oid_str)
for oid in oids:
self._send_load_response(s, oid)
def end(self, handle_invalidations=None):
write(self.s, 'C')
n = read_int4(self.s)
oid_list = []
if n != 0:
packed_oids = read(self.s, n * 8)
oid_list = split_oids(packed_oids)
try:
handle_invalidations(oid_list)
except ConflictError:
self.transaction_new_oids.reverse()
self.oid_pool.extend(self.transaction_new_oids)
assert len(self.oid_pool) == len(set(self.oid_pool))
self.begin() # clear out records and transaction_new_oids.
write_int4(self.s, 0) # Tell server we are done.
raise
tdata = []
for oid, record in iteritems(self.records):
tdata.append(int4_to_str(8 + len(record)))
tdata.append(as_bytes(oid))
tdata.append(record)
tdata = join_bytes(tdata)
write_int4_str(self.s, tdata)
self.records.clear()
if len(tdata) > 0:
status = read(self.s, 1)
if status == STATUS_OKAY:
pass
elif status == STATUS_INVALID:
raise WriteConflictError()
else:
raise ProtocolError('server returned invalid status %r' %
status)
def end(self, handle_invalidations=None):
write(self.s, 'C')
n = read_int4(self.s)
oid_list = []
if n != 0:
packed_oids = read(self.s, n*8)
oid_list = split_oids(packed_oids)
try:
handle_invalidations(oid_list)
except ConflictError:
self.transaction_new_oids.reverse()
self.oid_pool.extend(self.transaction_new_oids)
assert len(self.oid_pool) == len(set(self.oid_pool))
self.begin() # clear out records and transaction_new_oids.
write_int4(self.s, 0) # Tell server we are done.
raise
tdata = []
for oid, record in iteritems(self.records):
tdata.append(int4_to_str(8 + len(record)))
tdata.append(as_bytes(oid))
tdata.append(record)
tdata = join_bytes(tdata)
write_int4_str(self.s, tdata)
self.records.clear()
if len(tdata) > 0:
status = read(self.s, 1)
if status == STATUS_OKAY:
pass
elif status == STATUS_INVALID:
raise WriteConflictError()
else:
raise ProtocolError('server returned invalid status %r' % status)
def sync(self):
write(self.s, 'S')
n = read_int4(self.s)
if n == 0:
packed_oids = ''
else:
packed_oids = read(self.s, n*8)
return split_oids(packed_oids)
def gen_referring_oid_record(storage, referred_oid):
"""(storage:Storage, referred_oid:str) -> sequence([oid:str, record:str])
Generate oid, record pairs for all objects that include a
reference to the `referred_oid`.
"""
for oid, record in storage.gen_oid_record():
if referred_oid in split_oids(unpack_record(record)[2]):
yield oid, record
def sync(self):
write(self.s, 'S')
n = read_int4(self.s)
if n == 0:
packed_oids = ''
else:
packed_oids = read(self.s, n * 8)
return split_oids(packed_oids)
def _get_refs(self, oid):
c = self._conn.cursor()
c.execute('SELECT refs FROM objects WHERE id = ?',
(str_to_int8(oid),))
v = c.fetchone()
if v is None:
raise KeyError(oid)
return split_oids(v[0])
def sync(self):
self._send_command('S')
n = read_int4(self.socket)
if n == 0:
packed_oids = ''
else:
packed_oids = read(self.socket, 8 * n)
return split_oids(packed_oids)
def sync(self):
self.s.sendall('S')
n = u32(recv(self.s, 4))
if n == 0:
packed_oids = ''
else:
packed_oids = recv(self.s, n*8)
return split_oids(packed_oids)
def get_reference_index(storage):
"""(storage:Storage) -> {oid:str : [referring_oid:str]}
Return a full index giving the referring oids for each oid.
This might be large.
"""
result = {}
for oid, record in storage.gen_oid_record():
for ref in split_oids(unpack_record(record)[2]):
result.setdefault(ref, []).append(oid)
return result
def handle_bulk_read(self, client, db_name):
# B
log(20, 'Bulk read %s' % db_name)
storage = self.storages[db_name]
number_of_oids = str_to_int4((yield client.read(4)))
oid_str_len = 8 * number_of_oids
oid_str = yield client.read(oid_str_len)
oids = split_oids(oid_str)
for oid in oids:
yield self._send_load_response(client, db_name, storage, oid)
def new_oid(self):
if not self.oid_pool:
batch = self.oid_pool_size
write(self.s, 'M%s' % chr(batch))
self.oid_pool = split_oids(read(self.s, 8 * batch))
self.oid_pool.reverse()
assert len(self.oid_pool) == len(set(self.oid_pool))
oid = self.oid_pool.pop()
assert oid not in self.oid_pool
self.transaction_new_oids.append(oid)
return oid
def new_oid(self):
if not self.oid_pool:
batch = self.oid_pool_size
write(self.s, 'M%s' % chr(batch))
self.oid_pool = split_oids(read(self.s, 8 * batch))
self.oid_pool.reverse()
assert len(self.oid_pool) == len(set(self.oid_pool))
oid = self.oid_pool.pop()
assert oid not in self.oid_pool
self.transaction_new_oids.append(oid)
return oid
def new_oid(self):
if not self.oid_pool:
batch = self.oid_pool_size
self._send_command('M')
write(self.socket, chr(batch))
self.oid_pool = split_oids(read(self.socket, 8 * batch))
self.oid_pool.reverse()
assert len(self.oid_pool) == len(set(self.oid_pool))
oid = self.oid_pool.pop()
assert oid not in self.oid_pool
self.transaction_new_oids.append(oid)
return oid
def gen_reachable_records():
todo = [ROOT_OID]
seen = Set()
while todo:
oid = todo.pop()
if oid in seen:
continue
seen.add(oid)
record = self.load(oid)
record_oid, data, refdata = unpack_record(record)
assert oid == record_oid
todo.extend(split_oids(refdata))
yield oid, record
while self.pack_extra:
oid = self.pack_extra.pop()
yield oid, self.load(oid)
def gen_oid_record(self, start_oid=None, **other):
if start_oid is None:
for item in iteritems(self.shelf):
yield item
else:
todo = [start_oid]
seen = IntSet() # This eventually contains them all.
while todo:
oid = todo.pop()
if str_to_int8(oid) in seen:
continue
seen.add(str_to_int8(oid))
record = self.load(oid)
record_oid, data, refdata = unpack_record(record)
assert oid == record_oid
todo.extend(split_oids(refdata))
yield oid, record
def gen_oid_record(self, start_oid=None, **other):
if start_oid is None:
for item in iteritems(self.shelf):
yield item
else:
todo = [start_oid]
seen = IntSet() # This eventually contains them all.
while todo:
oid = todo.pop()
if str_to_int8(oid) in seen:
continue
seen.add(str_to_int8(oid))
record = self.load(oid)
record_oid, data, refdata = unpack_record(record)
assert oid == record_oid
todo.extend(split_oids(refdata))
yield oid, record
def gen_oid_record(self, start_oid=None, seen=None, **other):
if start_oid is None:
for item in iteritems(self.shelf):
yield item
else:
todo = [start_oid]
if seen is None:
seen = IntSet() # This eventually contains them all.
while todo:
oid = heapq.heappop(todo)
if str_to_int8(oid) in seen:
continue
seen.add(str_to_int8(oid))
record = self.load(oid)
record_oid, data, refdata = unpack_record(record)
assert oid == record_oid
for ref_oid in split_oids(refdata):
heapq.heappush(todo, ref_oid)
yield oid, record
def gen_reachable_records():
pack_todo = [durus.connection.ROOT_OID]
while pack_todo or self.pack_extra:
if self.pack_extra:
oid = self.pack_extra.pop()
# note we don't check 'index' because it could be an
# object that got updated since the pack began and in
# that case we have to write the new record to the pack
# file
else:
oid = heapq.heappop(pack_todo)
if oid in index:
# we already wrote this object record
continue
record = self.load(oid)
oid2, data, refdata = unpack_record(record)
assert oid == oid2
# ensure we have records for objects referenced
for ref_oid in split_oids(refdata):
heapq.heappush(pack_todo, ref_oid)
yield (oid, record)
def end(self, handle_invalidations=None):
self.s.sendall('C')
n = u32(recv(self.s, 4))
if n != 0:
packed_oids = recv(self.s, n*8)
try:
handle_invalidations(split_oids(packed_oids))
except ConflictError:
self.s.sendall(p32(0)) # Tell server we are done.
raise
tdata = []
for oid, record in self.records.iteritems():
tdata.append(p32(8 + len(record)))
tdata.append(oid)
tdata.append(record)
tdata = ''.join(tdata)
self.s.sendall(p32(len(tdata)))
self.s.sendall(tdata)
self.records.clear()
status = recv(self.s, 1)
if status != STATUS_OKAY:
raise ProtocolError, 'server returned invalid status %r' % status