def list_open(self): """List open databases on server. :rtype: List of strings """ write(self.socket, 'E') return list(self._enumerate_database_names())
def end(self, handle_invalidations=None): write(self.s, 'C') n = read_int4(self.s) oid_list = [] if n != 0: packed_oids = read(self.s, n * 8) oid_list = split_oids(packed_oids) try: handle_invalidations(oid_list) except ConflictError: self.transaction_new_oids.reverse() self.oid_pool.extend(self.transaction_new_oids) assert len(self.oid_pool) == len(set(self.oid_pool)) self.begin() # clear out records and transaction_new_oids. write_int4(self.s, 0) # Tell server we are done. raise tdata = [] for oid, record in iteritems(self.records): tdata.append(int4_to_str(8 + len(record))) tdata.append(as_bytes(oid)) tdata.append(record) tdata = join_bytes(tdata) write_int4_str(self.s, tdata) self.records.clear() if len(tdata) > 0: status = read(self.s, 1) if status == STATUS_OKAY: pass elif status == STATUS_INVALID: raise WriteConflictError() else: raise ProtocolError('server returned invalid status %r' % status)
def end(self, handle_invalidations=None): write(self.s, 'C') n = read_int4(self.s) oid_list = [] if n != 0: packed_oids = read(self.s, n*8) oid_list = split_oids(packed_oids) try: handle_invalidations(oid_list) except ConflictError: self.transaction_new_oids.reverse() self.oid_pool.extend(self.transaction_new_oids) assert len(self.oid_pool) == len(set(self.oid_pool)) self.begin() # clear out records and transaction_new_oids. write_int4(self.s, 0) # Tell server we are done. raise tdata = [] for oid, record in iteritems(self.records): tdata.append(int4_to_str(8 + len(record))) tdata.append(as_bytes(oid)) tdata.append(record) tdata = join_bytes(tdata) write_int4_str(self.s, tdata) self.records.clear() if len(tdata) > 0: status = read(self.s, 1) if status == STATUS_OKAY: pass elif status == STATUS_INVALID: raise WriteConflictError() else: raise ProtocolError('server returned invalid status %r' % status)
def list_all(self): """List all databases available on the server. :rtype: list of strings """ write(self.socket, 'A') return list(self._enumerate_database_names())
def store(self, name_value_sequence): """([(str, str)]) -> [(str, int|None, int)] Record all of the items in the sequence. Return a list of triples, each giving a name, an old position (or None if this is a new name), and a new position. """ self.file.seek_end() start = self.file.tell() write_int8(self.file, 0) result = [] index = {} try: for name, value in name_value_sequence: new_position = self.file.tell() old_position = self.get_position(name) index[name] = new_position result.append((name, old_position, new_position)) write_int8(self.file, len(name) + len(value)) write(self.file, name) write(self.file, value) except: # Revert before raising. self.file.seek(start) self.file.truncate() raise end = self.file.tell() self.file.seek(start) write_int8(self.file, end - start - 8) self.file.seek(end) self.memory_index.update(index) return result
def server_protocol(self): """Get the protocol used by the server. :rtype: 4-byte string """ write(self.socket, 'V') return read(self.socket, 4)
def open(self, db_name): """Open the named database. :param db_name: Name of database to open. :ptype db_name: string """ write(self.socket, 'O') self._write_database_name(db_name)
def handle_V(self, s): # Verify protocol version match. client_protocol = read(s, 4) log(10, 'Client Protocol: %s', str_to_int4(client_protocol)) assert len(self.protocol) == 4 write(s, self.protocol) if client_protocol != self.protocol: raise ClientError("Protocol not supported.")
def close(self, db_name): """Close the named database. :param db_name: Name of database to close. :ptype db_name: string """ write(self.socket, 'X') self._write_database_name(db_name)
def sync(self): write(self.s, 'S') n = read_int4(self.s) if n == 0: packed_oids = '' else: packed_oids = read(self.s, n*8) return split_oids(packed_oids)
def sync(self): write(self.s, 'S') n = read_int4(self.s) if n == 0: packed_oids = '' else: packed_oids = read(self.s, n * 8) return split_oids(packed_oids)
def b(self): s = BytesIO() for x in ('', 'a', 'ab', 'a' * 1000): x = as_bytes(x) s.seek(0) write(s, x) s.seek(0) assert x == read(s, len(x))
def b(self): s = BytesIO() for x in ("", "a", "ab", "a" * 1000): x = as_bytes(x) s.seek(0) write(s, x) s.seek(0) assert x == read(s, len(x))
def handle_P(self, s): # pack if self.packer is None: log(20, 'Pack started at %s' % datetime.now()) self.packer = self.storage.get_packer() if self.packer is None: self.storage.pack() log(20, 'Pack completed at %s' % datetime.now()) else: log(20, 'Pack already in progress at %s' % datetime.now()) write(s, STATUS_OKAY)
def quit(self): """Shut down the server process and disconnect. .. todo:: Determine if it is necessary to keep "quit" in the protocol or if a server should only be shut down by process control on the system that the server is running on. """ write(self.socket, 'Q') self.disconnect()
def new_oid(self): if not self.oid_pool: batch = self.oid_pool_size write(self.s, 'M%s' % chr(batch)) self.oid_pool = split_oids(read(self.s, 8 * batch)) self.oid_pool.reverse() assert len(self.oid_pool) == len(set(self.oid_pool)) oid = self.oid_pool.pop() assert oid not in self.oid_pool self.transaction_new_oids.append(oid) return oid
def f(self): class FakeSocket(object): def recv(self, n): if n > 10: return as_bytes('') return as_bytes('x') def send(self, s): return len(s) s = FakeSocket() write(s, 'x' * 2000000) read(s, 8) raises(ShortRead, read, s, 11)
def new_oid(self): if not self.oid_pool: batch = self.oid_pool_size self._send_command('M') write(self.socket, chr(batch)) self.oid_pool = split_oids(read(self.socket, 8 * batch)) self.oid_pool.reverse() assert len(self.oid_pool) == len(set(self.oid_pool)) oid = self.oid_pool.pop() assert oid not in self.oid_pool self.transaction_new_oids.append(oid) return oid
def handle_C(self, s): # commit self._sync_storage() client = self._find_client(s) write_all(s, int4_to_str(len(client.invalid)), join_bytes(client.invalid)) client.invalid.clear() tdata = read_int4_str(s) if len(tdata) == 0: return # client decided not to commit (e.g. conflict) logging_debug = is_logging(10) logging_debug and log(10, 'Committing %s bytes', len(tdata)) self.storage.begin() i = 0 oids = [] while i < len(tdata): rlen = str_to_int4(tdata[i:i + 4]) i += 4 oid = tdata[i:i + 8] record = tdata[i + 8:i + rlen] i += rlen if logging_debug: class_name = extract_class_name(record) log(10, ' oid=%-6s rlen=%-6s %s', str_to_int8(oid), rlen, class_name) self.storage.store(oid, record) oids.append(oid) assert i == len(tdata) oid_set = set(oids) for other_client in self.clients: if other_client is not client: if oid_set.intersection(other_client.unused_oids): raise ClientError("invalid oid: %r" % oid) try: self.storage.end(handle_invalidations=self._handle_invalidations) except ConflictError: log(20, 'Conflict during commit') write(s, STATUS_INVALID) else: self._report_load_record() log(20, 'Committed %3s objects %s bytes at %s', len(oids), len(tdata), datetime.now()) write(s, STATUS_OKAY) client.unused_oids -= oid_set for c in self.clients: if c is not client: c.invalid.update(oids) self.bytes_since_pack += len(tdata) + 8
def handle_C(self, s): # commit self._sync_storage() client = self._find_client(s) write_all(s, int4_to_str(len(client.invalid)), join_bytes(client.invalid)) client.invalid.clear() tdata = read_int4_str(s) if len(tdata) == 0: return # client decided not to commit (e.g. conflict) logging_debug = is_logging(10) logging_debug and log(10, 'Committing %s bytes', len(tdata)) self.storage.begin() i = 0 oids = [] while i < len(tdata): rlen = str_to_int4(tdata[i:i+4]) i += 4 oid = tdata[i:i+8] record = tdata[i+8:i+rlen] i += rlen if logging_debug: class_name = extract_class_name(record) log(10, ' oid=%-6s rlen=%-6s %s', str_to_int8(oid), rlen, class_name) self.storage.store(oid, record) oids.append(oid) assert i == len(tdata) oid_set = set(oids) for other_client in self.clients: if other_client is not client: if oid_set.intersection(other_client.unused_oids): raise ClientError("invalid oid: %r" % oid) try: self.storage.end(handle_invalidations=self._handle_invalidations) except ConflictError: log(20, 'Conflict during commit') write(s, STATUS_INVALID) else: self._report_load_record() log(20, 'Committed %3s objects %s bytes at %s', len(oids), len(tdata), datetime.now()) write(s, STATUS_OKAY) client.unused_oids -= oid_set for c in self.clients: if c is not client: c.invalid.update(oids) self.bytes_since_pack += len(tdata) + 8
def stop_durus(address): socket_address = SocketAddress.new(address) sock = socket_address.get_connected_socket() if sock is None: log(20, "Durus server %s doesn't seem to be running." % str(address)) return False write(sock, 'Q') # graceful exit message sock.close() # Try to wait until the address is free. for attempt in range(20): sleep(0.5) sock = socket_address.get_connected_socket() if sock is None: break sock.close() return True
def destroy(self, db_name): """Destroy the named database. :param db_name: Name of database to destroy. :ptype db_name: string .. note:: The database will not be destroyed if it is currently open. .. warning:: This will permanently delete the file containing the database on the server. """ write(self.socket, 'D') self._write_database_name(db_name)
def generate_shelf(klass, file, items): """(File, [(str, str)]) This returns a generator that writes a new Shelf into file, iterating once through the given items. The use of an iterator makes it possible to build a new Shelf incrementally. """ file.seek_end() if not file.tell() == 0: raise ValueError("Expected %s to be empty." % file) write(file, klass.prefix) if not items: # Just write an empty transaction. write_int8(file, 0) # Write an empty index array. offset_map = OffsetMap(file) else: # Write a transaction here with the given items. transaction_start = file.tell() # Write a placeholder for the length. write_int8(file, 0) # Loop over the items, writing their records. # Keep track of max_key and max_offset. max_key = 0 max_offset = 0 n = 0 for name, value in items: max_key = max(max_key, str_to_int8(name)) max_offset = max(max_offset, file.tell()) write_int8(file, len(name) + len(value)) write(file, name) write(file, value) n += 1 yield n transaction_end = file.tell() # Write the correct transaction length. file.seek(transaction_start) write_int8(file, transaction_end - transaction_start - 8) # Write the empty array with the calculated dimensions. file.seek(transaction_end) for step in OffsetMap.generate(file, max_key, max_offset): yield step offset_map = OffsetMap(file) # Now read through the records and record the offsets in the array. file.seek(transaction_start + 8) while file.tell() < transaction_end: position = file.tell() record_length = read_int8(file) name = read(file, 8) k = str_to_int8(name) offset_map[k] = position file.seek(position + 8 + record_length) n -= 1 yield n for index in offset_map.gen_stitch(): yield index
def _send_load_response(self, s, oid): if oid in self._find_client(s).invalid: write(s, STATUS_INVALID) else: try: record = self.storage.load(oid) except KeyError: log(10, 'KeyError %s', str_to_int8(oid)) write(s, STATUS_KEYERROR) except ReadConflictError: log(10, 'ReadConflictError %s', str_to_int8(oid)) write(s, STATUS_INVALID) else: if is_logging(5): class_name = extract_class_name(record) if class_name in self.load_record: self.load_record[class_name] += 1 else: self.load_record[class_name] = 1 log(4, 'Load %-7s %s', str_to_int8(oid), class_name) write(s, STATUS_OKAY) write_int4_str(s, record)
def load(self, oid): self._send_command('L') write(self.socket, oid) return self._get_load_response(oid)
def pack(self): write(self.s, 'P') status = read(self.s, 1) if status != STATUS_OKAY: raise ProtocolError('server returned invalid status %r' % status)
def close(self): write(self.s, '.') # Closes the server side. self.s.close()
def handle_N(self, s): # new OID write(s, self._new_oids(s, 1)[0])
def handle_M(self, s): # new OIDs count = ord(read(s, 1)) log(10, "oids: %s", count) write(s, join_bytes(self._new_oids(s, count)))
def _write_header(self, fp): fp.seek(0, 2) assert fp.tell() == 0 write(fp, self.MAGIC) write_int8(fp, 0) # index offset
def disconnect(self): """Disconnect from the server.""" if self.socket is not None: write(self.socket, '.') self.socket.close() self.socket = None
def _write_database_name(self, db_name): write_int4(self.socket, len(db_name)) write(self.socket, db_name)