Beispiel #1
0
 def __init__(self, client, db_name):
     self.client = client
     self.db_name = int4_to_str(len(db_name)) + db_name
     self.socket = client.socket
     self.oid_pool = []
     self.oid_pool_size = 32
     self.begin()
Beispiel #2
0
 def end(self, handle_invalidations=None):
     write(self.s, 'C')
     n = read_int4(self.s)
     oid_list = []
     if n != 0:
         packed_oids = read(self.s, n * 8)
         oid_list = split_oids(packed_oids)
         try:
             handle_invalidations(oid_list)
         except ConflictError:
             self.transaction_new_oids.reverse()
             self.oid_pool.extend(self.transaction_new_oids)
             assert len(self.oid_pool) == len(set(self.oid_pool))
             self.begin()  # clear out records and transaction_new_oids.
             write_int4(self.s, 0)  # Tell server we are done.
             raise
     tdata = []
     for oid, record in iteritems(self.records):
         tdata.append(int4_to_str(8 + len(record)))
         tdata.append(as_bytes(oid))
         tdata.append(record)
     tdata = join_bytes(tdata)
     write_int4_str(self.s, tdata)
     self.records.clear()
     if len(tdata) > 0:
         status = read(self.s, 1)
         if status == STATUS_OKAY:
             pass
         elif status == STATUS_INVALID:
             raise WriteConflictError()
         else:
             raise ProtocolError('server returned invalid status %r' %
                                 status)
Beispiel #3
0
 def end(self, handle_invalidations=None):
     write(self.s, 'C')
     n = read_int4(self.s)
     oid_list = []
     if n != 0:
         packed_oids = read(self.s, n*8)
         oid_list = split_oids(packed_oids)
         try:
             handle_invalidations(oid_list)
         except ConflictError:
             self.transaction_new_oids.reverse()
             self.oid_pool.extend(self.transaction_new_oids)
             assert len(self.oid_pool) == len(set(self.oid_pool))
             self.begin() # clear out records and transaction_new_oids.
             write_int4(self.s, 0) # Tell server we are done.
             raise
     tdata = []
     for oid, record in iteritems(self.records):
         tdata.append(int4_to_str(8 + len(record)))
         tdata.append(as_bytes(oid))
         tdata.append(record)
     tdata = join_bytes(tdata)
     write_int4_str(self.s, tdata)
     self.records.clear()
     if len(tdata) > 0:
         status = read(self.s, 1)
         if status == STATUS_OKAY:
             pass
         elif status == STATUS_INVALID:
             raise WriteConflictError()
         else:
             raise ProtocolError('server returned invalid status %r' % status)
Beispiel #4
0
 def bulk_load(self, oids):
     oid_str = join_bytes(oids)
     num_oids, remainder = divmod(len(oid_str), 8)
     assert remainder == 0, remainder
     write_all(self.s, 'B', int4_to_str(num_oids), oid_str)
     records = [self._get_load_response(oid) for oid in oids]
     for record in records:
         yield record
Beispiel #5
0
 def bulk_load(self, oids):
     oid_str = join_bytes(oids)
     num_oids, remainder = divmod(len(oid_str), 8)
     assert remainder == 0, remainder
     write_all(self.s, 'B', int4_to_str(num_oids), oid_str)
     records = [self._get_load_response(oid) for oid in oids]
     for record in records:
         yield record
Beispiel #6
0
 def handle_S(self, s):
     # sync
     client = self._find_client(s)
     self._report_load_record()
     self._sync_storage()
     log(8, 'Sync %s', len(client.invalid))
     write_all(s,
         int4_to_str(len(client.invalid)), join_bytes(client.invalid))
     client.invalid.clear()
Beispiel #7
0
 def handle_S(self, s):
     # sync
     client = self._find_client(s)
     self._report_load_record()
     self._sync_storage()
     log(8, 'Sync %s', len(client.invalid))
     write_all(s, int4_to_str(len(client.invalid)),
               join_bytes(client.invalid))
     client.invalid.clear()
Beispiel #8
0
 def handle_commit(self, client, db_name):
     # C
     log(20, 'Commit %s' % db_name)
     storage = self.storages[db_name]
     self._sync_storage(db_name, storage)
     invalid = client.invalid[db_name]
     yield client.write(int4_to_str(len(invalid)))
     yield client.write(join_bytes(invalid))
     yield client.flush()
     invalid.clear()
     tdata_len = str_to_int4((yield client.read(4)))
     if tdata_len == 0:
         # Client decided not to commit (e.g. conflict)
         return
     tdata = yield client.read(tdata_len)
     logging_debug = is_logging(10)
     logging_debug and log(10, 'Committing %s bytes', tdata_len)
     storage.begin()
     i = 0
     oids = []
     while i < tdata_len:
         rlen = str_to_int4(tdata[i:i+4])
         i += 4
         oid = tdata[i:i+8]
         record = tdata[i+8:i+rlen]
         i += rlen
         if logging_debug:
             class_name = extract_class_name(record)
             log(10, '  oid=%-6s rlen=%-6s %s',
                 str_to_int8(oid), rlen, class_name)
         storage.store(oid, record)
         oids.append(oid)
     assert i == tdata_len
     oid_set = set(oids)
     for c in self.clients:
         if c is not client:
             if oid_set.intersection(c.unused_oids[db_name]):
                 raise ClientError('invalid oid: %r' % oid)
     try:
         handle_invalidations = (
             lambda oids: self._handle_invalidations(db_name, oids))
         storage.end(handle_invalidations=handle_invalidations)
     except ConflictError:
         log(20, 'Conflict during commit')
         yield client.write(STATUS_INVALID)
     else:
         self._report_load_record(storage)
         log(20, 'Committed %3s objects %s bytes at %s',
             len(oids), tdata_len, datetime.now())
         yield client.write(STATUS_OKAY)
         client.unused_oids[db_name] -= oid_set
         for c in self.clients:
             if c is not client:
                 c.invalid[db_name].update(oids)
         storage.d_bytes_since_pack += tdata_len + 8
Beispiel #9
0
 def handle_sync(self, client, db_name):
     # S
     log(20, 'Sync %s' % db_name)
     storage = self.storages[db_name]
     self._report_load_record(storage)
     self._sync_storage(db_name, storage)
     invalid = client.invalid[db_name]
     log(8, 'Sync %s', len(invalid))
     yield client.write(int4_to_str(len(invalid)))
     yield client.write(join_bytes(invalid))
     invalid.clear()
Beispiel #10
0
 def handle_C(self, s):
     # commit
     self._sync_storage()
     client = self._find_client(s)
     write_all(s,
         int4_to_str(len(client.invalid)), join_bytes(client.invalid))
     client.invalid.clear()
     tdata = read_int4_str(s)
     if len(tdata) == 0:
         return # client decided not to commit (e.g. conflict)
     logging_debug = is_logging(10)
     logging_debug and log(10, 'Committing %s bytes', len(tdata))
     self.storage.begin()
     i = 0
     oids = []
     while i < len(tdata):
         rlen = str_to_int4(tdata[i:i+4])
         i += 4
         oid = tdata[i:i+8]
         record = tdata[i+8:i+rlen]
         i += rlen
         if logging_debug:
             class_name = extract_class_name(record)
             log(10, '  oid=%-6s rlen=%-6s %s',
                 str_to_int8(oid), rlen, class_name)
         self.storage.store(oid, record)
         oids.append(oid)
     assert i == len(tdata)
     oid_set = set(oids)
     for other_client in self.clients:
         if other_client is not client:
             if oid_set.intersection(other_client.unused_oids):
                 raise ClientError("invalid oid: %r" % oid)
     try:
         self.storage.end(handle_invalidations=self._handle_invalidations)
     except ConflictError:
         log(20, 'Conflict during commit')
         write(s, STATUS_INVALID)
     else:
         self._report_load_record()
         log(20, 'Committed %3s objects %s bytes at %s',
             len(oids), len(tdata), datetime.now())
         write(s, STATUS_OKAY)
         client.unused_oids -= oid_set
         for c in self.clients:
             if c is not client:
                 c.invalid.update(oids)
         self.bytes_since_pack += len(tdata) + 8
Beispiel #11
0
 def handle_C(self, s):
     # commit
     self._sync_storage()
     client = self._find_client(s)
     write_all(s, int4_to_str(len(client.invalid)),
               join_bytes(client.invalid))
     client.invalid.clear()
     tdata = read_int4_str(s)
     if len(tdata) == 0:
         return  # client decided not to commit (e.g. conflict)
     logging_debug = is_logging(10)
     logging_debug and log(10, 'Committing %s bytes', len(tdata))
     self.storage.begin()
     i = 0
     oids = []
     while i < len(tdata):
         rlen = str_to_int4(tdata[i:i + 4])
         i += 4
         oid = tdata[i:i + 8]
         record = tdata[i + 8:i + rlen]
         i += rlen
         if logging_debug:
             class_name = extract_class_name(record)
             log(10, '  oid=%-6s rlen=%-6s %s', str_to_int8(oid), rlen,
                 class_name)
         self.storage.store(oid, record)
         oids.append(oid)
     assert i == len(tdata)
     oid_set = set(oids)
     for other_client in self.clients:
         if other_client is not client:
             if oid_set.intersection(other_client.unused_oids):
                 raise ClientError("invalid oid: %r" % oid)
     try:
         self.storage.end(handle_invalidations=self._handle_invalidations)
     except ConflictError:
         log(20, 'Conflict during commit')
         write(s, STATUS_INVALID)
     else:
         self._report_load_record()
         log(20, 'Committed %3s objects %s bytes at %s', len(oids),
             len(tdata), datetime.now())
         write(s, STATUS_OKAY)
         client.unused_oids -= oid_set
         for c in self.clients:
             if c is not client:
                 c.invalid.update(oids)
         self.bytes_since_pack += len(tdata) + 8
Beispiel #12
0
 def _send_load_response(self, client, db_name, storage, oid):
     if oid in client.invalid[db_name]:
         yield client.write(STATUS_INVALID)
     else:
         try:
             record = storage.load(oid)
         except KeyError:
             log(10, 'KeyError %s', str_to_int8(oid))
             yield client.write(STATUS_KEYERROR)
         except ReadConflictError:
             log(10, 'ReadConflictError %s', str_to_int8(oid))
             yield client.write(STATUS_INVALID)
         else:
             if is_logging(5):
                 class_name = extract_class_name(record)
                 if class_name in storage.d_load_record:
                     storage.d_load_record[class_name] += 1
                 else:
                     storage.d_load_record[class_name] = 1
                 log(4, 'Load %-7s %s', str_to_int8(oid), class_name)
             yield client.write(STATUS_OKAY)
             yield client.write(int4_to_str(len(record)))
             yield client.write(record)
Beispiel #13
0
 def _handle_enumerate_database_names(self, client, names):
     yield client.write(int4_to_str(len(names)))
     for name in names:
         yield client.write(int4_to_str(len(name)))
         yield client.write(name)
Beispiel #14
0
class StorageServer(object):

    protocol = int4_to_str(1)

    def __init__(self,
                 storage,
                 host=DEFAULT_HOST,
                 port=DEFAULT_PORT,
                 address=None,
                 gcbytes=DEFAULT_GCBYTES):
        self.storage = storage
        self.clients = []
        self.sockets = []
        self.packer = None
        self.address = SocketAddress.new(address or (host, port))
        self.load_record = {}
        self.bytes_since_pack = 0
        self.gcbytes = gcbytes  # Trigger a pack after this many bytes.
        assert isinstance(gcbytes, (int, float))

    def serve(self):
        sock = self.address.get_listening_socket()
        log(20, 'Ready on %s', self.address)
        self.sockets.append(sock)
        try:
            while 1:
                if self.packer is not None:
                    timeout = 0.0
                else:
                    timeout = None
                r, w, e = select.select(self.sockets, [], [], timeout)
                for s in r:
                    if s is sock:
                        # new connection
                        conn, addr = s.accept()
                        self.address.set_connection_options(conn)
                        self.clients.append(_Client(conn, addr))
                        self.sockets.append(conn)
                    else:
                        # command from client
                        try:
                            self.handle(s)
                        except (ClientError, socket.error, socket.timeout,
                                IOError):
                            exc = sys.exc_info()[1]
                            log(10, '%s', ''.join(map(str, exc.args)))
                            self.sockets.remove(s)
                            self.clients.remove(self._find_client(s))
                            s.close()
                if (self.packer is None
                        and 0 < self.gcbytes <= self.bytes_since_pack):
                    self.packer = self.storage.get_packer()
                    if self.packer is not None:
                        log(20, 'gc started at %s' % datetime.now())
                if not r and self.packer is not None:
                    try:
                        pack_step = next(self.packer)
                        if isinstance(pack_step, str):
                            log(15, 'gc ' + pack_step)
                    except StopIteration:
                        log(20, 'gc at %s' % datetime.now())
                        self.packer = None  # done packing
                        self.bytes_since_pack = 0  # reset
        finally:
            self.address.close(sock)

    def handle(self, s):
        command_byte = read(s, 1)[0]
        if type(command_byte) is int:
            command_code = chr(command_byte)
        else:
            command_code = command_byte
        handler = getattr(self, 'handle_%s' % command_code, None)
        if handler is None:
            raise ClientError('No such command code: %r' % command_code)
        handler(s)

    def _find_client(self, s):
        for client in self.clients:
            if client.s is s:
                return client
        assert 0

    def _new_oids(self, s, count):
        oids = []
        while len(oids) < count:
            oid = self.storage.new_oid()
            for client in self.clients:
                if oid in client.invalid:
                    oid = None
                    break
            if oid is not None:
                oids.append(oid)
        self._find_client(s).unused_oids.update(oids)
        return oids

    def handle_N(self, s):
        # new OID
        write(s, self._new_oids(s, 1)[0])

    def handle_M(self, s):
        # new OIDs
        count = ord(read(s, 1))
        log(10, "oids: %s", count)
        write(s, join_bytes(self._new_oids(s, count)))

    def handle_L(self, s):
        # load
        oid = read(s, 8)
        self._send_load_response(s, oid)

    def _send_load_response(self, s, oid):
        if oid in self._find_client(s).invalid:
            write(s, STATUS_INVALID)
        else:
            try:
                record = self.storage.load(oid)
            except KeyError:
                log(10, 'KeyError %s', str_to_int8(oid))
                write(s, STATUS_KEYERROR)
            except ReadConflictError:
                log(10, 'ReadConflictError %s', str_to_int8(oid))
                write(s, STATUS_INVALID)
            else:
                if is_logging(5):
                    class_name = extract_class_name(record)
                    if class_name in self.load_record:
                        self.load_record[class_name] += 1
                    else:
                        self.load_record[class_name] = 1
                    log(4, 'Load %-7s %s', str_to_int8(oid), class_name)
                write(s, STATUS_OKAY)
                write_int4_str(s, record)

    def handle_C(self, s):
        # commit
        self._sync_storage()
        client = self._find_client(s)
        write_all(s, int4_to_str(len(client.invalid)),
                  join_bytes(client.invalid))
        client.invalid.clear()
        tdata = read_int4_str(s)
        if len(tdata) == 0:
            return  # client decided not to commit (e.g. conflict)
        logging_debug = is_logging(10)
        logging_debug and log(10, 'Committing %s bytes', len(tdata))
        self.storage.begin()
        i = 0
        oids = []
        while i < len(tdata):
            rlen = str_to_int4(tdata[i:i + 4])
            i += 4
            oid = tdata[i:i + 8]
            record = tdata[i + 8:i + rlen]
            i += rlen
            if logging_debug:
                class_name = extract_class_name(record)
                log(10, '  oid=%-6s rlen=%-6s %s', str_to_int8(oid), rlen,
                    class_name)
            self.storage.store(oid, record)
            oids.append(oid)
        assert i == len(tdata)
        oid_set = set(oids)
        for other_client in self.clients:
            if other_client is not client:
                if oid_set.intersection(other_client.unused_oids):
                    raise ClientError("invalid oid: %r" % oid)
        try:
            self.storage.end(handle_invalidations=self._handle_invalidations)
        except ConflictError:
            log(20, 'Conflict during commit')
            write(s, STATUS_INVALID)
        else:
            self._report_load_record()
            log(20, 'Committed %3s objects %s bytes at %s', len(oids),
                len(tdata), datetime.now())
            write(s, STATUS_OKAY)
            client.unused_oids -= oid_set
            for c in self.clients:
                if c is not client:
                    c.invalid.update(oids)
            self.bytes_since_pack += len(tdata) + 8

    def _report_load_record(self):
        if self.load_record and is_logging(5):
            log(
                5, '[%s]\n' % getpid() +
                '\n'.join("%8s: %s" % (item[1], item[0])
                          for item in sorted(self.load_record.items())))
            self.load_record.clear()

    def _handle_invalidations(self, oids):
        for c in self.clients:
            c.invalid.update(oids)

    def _sync_storage(self):
        self._handle_invalidations(self.storage.sync())

    def handle_S(self, s):
        # sync
        client = self._find_client(s)
        self._report_load_record()
        self._sync_storage()
        log(8, 'Sync %s', len(client.invalid))
        write_all(s, int4_to_str(len(client.invalid)),
                  join_bytes(client.invalid))
        client.invalid.clear()

    def handle_P(self, s):
        # pack
        if self.packer is None:
            log(20, 'Pack started at %s' % datetime.now())
            self.packer = self.storage.get_packer()
            if self.packer is None:
                self.storage.pack()
                log(20, 'Pack completed at %s' % datetime.now())
        else:
            log(20, 'Pack already in progress at %s' % datetime.now())
        write(s, STATUS_OKAY)

    def handle_B(self, s):
        # bulk read of objects
        number_of_oids = read_int4(s)
        oid_str = read(s, 8 * number_of_oids)
        oids = split_oids(oid_str)
        for oid in oids:
            self._send_load_response(s, oid)

    def handle_Q(self, s):
        # graceful quit
        log(20, 'Quit')
        self.storage.close()
        raise SystemExit

    def handle_V(self, s):
        # Verify protocol version match.
        client_protocol = read(s, 4)
        log(10, 'Client Protocol: %s', str_to_int4(client_protocol))
        assert len(self.protocol) == 4
        write(s, self.protocol)
        if client_protocol != self.protocol:
            raise ClientError("Protocol not supported.")
Beispiel #15
0
def pack_record(oid, data, refs):
    """(oid:str, data:str, refs:str) -> record:str
    """
    return join_bytes([oid, int4_to_str(len(data)), data, refs])
Beispiel #16
0
from durus.serialize import extract_class_name, split_oids
from durus.storage_server import (
    DEFAULT_GCBYTES,
    STATUS_OKAY, STATUS_KEYERROR, STATUS_INVALID,
    ClientError,
    )
from durus.utils import (
    int4_to_str, int8_to_str, str_to_int4, str_to_int8,
    join_bytes,
    )


DEFAULT_HOST = '127.0.0.1'
DEFAULT_PORT = 22972

PROTOCOL = int4_to_str(20001)

EXTENSION = '.durus'


def database_names(path):
    """Return a list of all Durus database names in a given path."""
    for filename in os.listdir(path):
        name, ext = os.path.splitext(filename)
        if ext == EXTENSION:
            yield name


class ConnectedClient(object):

    def __init__(self, client_socket):
Beispiel #17
0
 def check_int4_to_str_str_to_int4(self):
     for x in range(3):
         assert len(int4_to_str(x)) == 4
         assert x == str_to_int4(int4_to_str(x))
Beispiel #18
0
def pack_record(oid, data, refs):
    """(oid:str, data:str, refs:str) -> record:str
    """
    return join_bytes([oid, int4_to_str(len(data)), data, refs])