Пример #1
0
    def __init__(self, **kwargs):
        buffer = kwargs.get("buffer", None)
        self.flags = kwargs.get("flags", b'\x00')
        self.pageCapacity = kwargs.get("pageCapacity", len(buffer))
        self.tupleSize = kwargs.get("tupleSize", None)
        self.bitmap = kwargs.get("bitmap", None)

        if buffer == None:
            raise ValueError(
                "No backing buffer supplied for SlottedPageHeader")

        if self.bitmap == None:
            headerSizeWithoutBitmap = struct.Struct("cHHH").size
            tupleCapacity = math.floor(
                (8 * (self.pageCapacity - headerSizeWithoutBitmap)) /
                (1 + (8 * self.tupleSize)))
            bString = '0b' + ('0' * tupleCapacity)
            self.bitmap = BitArray(bString)

        self.binrepr = struct.Struct("cHHH" +
                                     str(math.ceil(len(self.bitmap))) + 's')
        self.size = self.binrepr.size
        self.freeSpaceOffset = self.size

        buffer[0:self.size] = self.pack()
Пример #2
0
    def unpack(cls, buffer):
        binrepr1 = struct.Struct("cHHH")
        values1 = binrepr1.unpack_from(buffer)

        headerSizeWithoutBitmap = binrepr1.size
        tupleCapacity = math.floor(
            (8 * (values1[3] - headerSizeWithoutBitmap)) / (1 +
                                                            (8 * values1[1])))

        binrepr2 = struct.Struct("cHHH" + str(math.ceil(tupleCapacity)) + 's')
        values2 = binrepr2.unpack_from(buffer)

        bString = '0b' + ('0' * tupleCapacity)
        bitmap = BitArray(bString)

        index = 0
        for bit in values2[4]:
            if index >= tupleCapacity:
                break
            if bit:
                bitmap[index] = '0b1'
            else:
                bitmap[index] = '0b0'
            index = index + 1

        if len(values2) == 5:
            return cls(buffer=buffer,
                       flags=values2[0],
                       tupleSize=values2[1],
                       freeSpaceOffset=values2[2],
                       pageCapacity=values2[3],
                       bitmap=bitmap)
Пример #3
0
  def __init__(self, **kwargs):
    buffer=kwargs.get("buffer", None)
    self.flags           = kwargs.get("flags", b'\x00')
    self.pageCapacity = kwargs.get("pageCapacity", len(buffer))
    self.tupleSize = kwargs.get("tupleSize", None)
    self.bitmap=kwargs.get("bitmap", None)

    if buffer == None:
      raise ValueError("No backing buffer supplied for SlottedPageHeader")

    if self.bitmap == None:
      headerSizeWithoutBitmap = struct.Struct("cHHH").size
      tupleCapacity = math.floor((8*(self.pageCapacity-headerSizeWithoutBitmap))/(1+(8*self.tupleSize)))
      bString = '0b' + ('0' * tupleCapacity)
      self.bitmap = BitArray(bString)
   
    self.binrepr   = struct.Struct("cHHH" + str(math.ceil(len(self.bitmap))) + 's')
    self.size      = self.binrepr.size
    self.freeSpaceOffset = self.size
   
    buffer[0:self.size] = self.pack()
Пример #4
0
class SlottedPageHeader(PageHeader):
  """
  A slotted page header implementation. This should store a slot bitmap
  implemented as a memoryview on the byte buffer backing the page
  associated with this header. Additionally this header object stores
  the number of slots in the array, as well as the index of the next
  available slot.

  The binary representation of this header object is: (numSlots, nextSlot, slotBuffer)

  >>> import io
  >>> buffer = io.BytesIO(bytes(4096))
  >>> ph     = SlottedPageHeader(buffer=buffer.getbuffer(), tupleSize=16)
  >>> ph2    = SlottedPageHeader.unpack(buffer.getbuffer())
  >>> ph == ph2
  True

  ## Dirty bit tests
  >>> ph.isDirty()
  False
  >>> ph.setDirty(True)
  >>> ph.isDirty()
  True
  >>> ph.setDirty(False)
  >>> ph.isDirty()
  False

  ## Tuple count tests
  >>> ph.hasFreeTuple()
  True

  # First tuple allocated should be at the first slot.
  # Notice this is a slot index, not an offset as with contiguous pages.
  >>> ph.nextFreeTuple() == 0
  True

  >>> ph.numTuples()
  1

  >>> tuplesToTest = 10
  >>> [ph.nextFreeTuple() for i in range(0, tuplesToTest)]
  [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
  
  >>> ph.numTuples() == tuplesToTest+1
  True

  >>> ph.hasFreeTuple()
  True

  # Check space utilization
  >>> ph.usedSpace() == (tuplesToTest+1)*ph.tupleSize
  True

  >>> ph.freeSpace() == 4096 - (ph.headerSize() + ((tuplesToTest+1) * ph.tupleSize))
  True

  >>> remainingTuples = int(ph.freeSpace() / ph.tupleSize)

  # Fill the page.
  >>> [ph.nextFreeTuple() for i in range(0, remainingTuples)] # doctest:+ELLIPSIS
  [11, 12, ...]

  >>> ph.hasFreeTuple()
  False

  # No value is returned when trying to exceed the page capacity.
  >>> ph.nextFreeTuple() == None
  True
  
  >>> ph.freeSpace() < ph.tupleSize
  True
  """

  def __init__(self, **kwargs):
    buffer=kwargs.get("buffer", None)
    self.flags           = kwargs.get("flags", b'\x00')
    self.pageCapacity = kwargs.get("pageCapacity", len(buffer))
    self.tupleSize = kwargs.get("tupleSize", None)
    self.bitmap=kwargs.get("bitmap", None)

    if buffer == None:
      raise ValueError("No backing buffer supplied for SlottedPageHeader")

    if self.bitmap == None:
      headerSizeWithoutBitmap = struct.Struct("cHHH").size
      tupleCapacity = math.floor((8*(self.pageCapacity-headerSizeWithoutBitmap))/(1+(8*self.tupleSize)))
      bString = '0b' + ('0' * tupleCapacity)
      self.bitmap = BitArray(bString)
   
    self.binrepr   = struct.Struct("cHHH" + str(math.ceil(len(self.bitmap))) + 's')
    self.size      = self.binrepr.size
    self.freeSpaceOffset = self.size
   
    buffer[0:self.size] = self.pack()
 #   super().__init__(buffer=buffer, flags=kwargs.get("flags", b'\x00'), self.tupleSize)
  
  def __eq__(self, other):
    return (    self.flags == other.flags
            and self.tupleSize == other.tupleSize
            and self.pageCapacity == other.pageCapacity
            and self.freeSpaceOffset == other.freeSpaceOffset 
            and self.bitmap == other.bitmap)

  def __hash__(self):
    return hash((self.flags, self.tupleSize, self.pageCapacity, self.freeSpaceOffset, self.bitmap))

  def headerSize(self):
    return self.size

  # Flag operations.
  def flag(self, mask):
    return (ord(self.flags) & mask) > 0

  def setFlag(self, mask, set):
    if set:
      self.flags = bytes([ord(self.flags) | mask])
    else:
      self.flags = bytes([ord(self.flags) & ~mask])

  # Dirty bit accessors
  def isDirty(self):
    return self.flag(PageHeader.dirtyMask)

  def setDirty(self, dirty):
    self.setFlag(PageHeader.dirtyMask, dirty)

  def numTuples(self):
    return self.bitmap.count(1)

  # Returns the space available in the page associated with this header.
  def freeSpace(self):
    return self.pageCapacity - (self.size + (self.numTuples() * self.tupleSize))

  # Returns the space used in the page associated with this header.
  def usedSpace(self):
    return (self.numTuples() * self.tupleSize)


  # Slot operations.
  def offsetOfSlot(self, slot):
    return slot * self.tupleSize + self.size

  def hasSlot(self, slotIndex):
    return slotIndex < len(self.bitmap)

  def getSlot(self, slotIndex):
    return self.bitmap[slotIndex] == '0b1' 

  def setSlot(self, slotIndex, slot):
    if not self.hasSlot(slotIndex):
      return

    if slot == True:
      self.bitmap[slotIndex] = '0b1'
    elif slot == False:
      self.bitmap[slotIndex] = '0b0'

  def resetSlot(self, slotIndex):
    self.setSlot(slotIndex, False)

  def freeSlots(self):
    freeList = []
    for i in range(len(self.bitmap)):
      if self.bitmap[i] == '0b0':
        freeList.append(i)
    return freeList

  def usedSlots(self):
    usedList = []
    for i in range(len(self.bitmap)):
      if self.bitmap[i] == '0b1':
        usedList.append(i)
    return usedList

  # Tuple allocation operations.
  
  # Returns whether the page has any free space for a tuple.
  def hasFreeTuple(self):
    return self.freeSpace() >= self.tupleSize
    # findTuple = self.bitmap.find('0b0')
    # if findTuple == ():
    #   return False
    # else:
    #   return True

  # Returns the tupleIndex of the next free tuple.
  # This should also "allocate" the tuple, such that any subsequent call
  # does not yield the same tupleIndex.
  def nextFreeTuple(self):
    #nextTuple = self.bitmap.find('0b0')

    #if nextTuple == ():
    #  return None
    
    #headerSizeWithoutBitmap = struct.Struct("cHHH").size
    #tupleCapacity = math.floor((8*(self.pageCapacity-headerSizeWithoutBitmap))/(1+(8*self.tupleSize)))
    #if nextTuple[0] >= tupleCapacity:
    #  return None

    #self.bitmap[nextTuple[0]] = '0b1'
    #return nextTuple[0]
    if self.hasFreeTuple():
      nextTuple = self.bitmap.find('0b0')
      self.bitmap[nextTuple[0]] = '0b1'
      return nextTuple[0]
    else:
      return None

  def nextTupleRange(self):
    start = self.nextFreeTuple()
    end = start + self.tupleSize
    index = (start - self.size)//self.tupleSize
    return (index, start, end)

  # Create a binary representation of a slotted page header.
  # The binary representation should include the slot contents.
  def pack(self):
    byteArray = bytearray(self.bitmap)

    packed = self.binrepr.pack(
              self.flags, self.tupleSize,
              self.freeSpaceOffset, self.pageCapacity, byteArray)
    unpacked = self.binrepr.unpack_from(packed)

    return self.binrepr.pack(
              self.flags, self.tupleSize,
              self.freeSpaceOffset, self.pageCapacity, byteArray)

  # Create a slotted page header instance from a binary representation held in the given buffer.
  @classmethod
  def unpack(cls, buffer):
    binrepr1 = struct.Struct("cHHH")
    values1 = binrepr1.unpack_from(buffer)

    headerSizeWithoutBitmap = binrepr1.size
    tupleCapacity = math.floor((8*(values1[3]-headerSizeWithoutBitmap))/(1+(8*values1[1])))

    binrepr2   = struct.Struct("cHHH" + str(math.ceil(tupleCapacity)) + 's')
    values2 = binrepr2.unpack_from(buffer)

    bString = '0b' + ('0' * tupleCapacity)
    bitmap = BitArray(bString)

    index = 0
    for bit in values2[4]:
      if index >= tupleCapacity:
        break
      if bit:
        bitmap[index] = '0b1'
      else:
        bitmap[index] = '0b0'
      index = index + 1

    if len(values2) == 5:
      return cls(buffer=buffer, flags=values2[0], tupleSize=values2[1],
                 freeSpaceOffset=values2[2], pageCapacity=values2[3], 
                 bitmap=bitmap)
Пример #5
0
class SlottedPageHeader(PageHeader):
    """
  A slotted page header implementation. This should store a slot bitmap
  implemented as a memoryview on the byte buffer backing the page
  associated with this header. Additionally this header object stores
  the number of slots in the array, as well as the index of the next
  available slot.

  The binary representation of this header object is: (numSlots, nextSlot, slotBuffer)

  >>> import io
  >>> buffer = io.BytesIO(bytes(4096))
  >>> ph     = SlottedPageHeader(buffer=buffer.getbuffer(), tupleSize=16)
  >>> ph2    = SlottedPageHeader.unpack(buffer.getbuffer())
  >>> ph == ph2
  True

  ## Dirty bit tests
  >>> ph.isDirty()
  False
  >>> ph.setDirty(True)
  >>> ph.isDirty()
  True
  >>> ph.setDirty(False)
  >>> ph.isDirty()
  False

  ## Tuple count tests
  >>> ph.hasFreeTuple()
  True

  # First tuple allocated should be at the first slot.
  # Notice this is a slot index, not an offset as with contiguous pages.
  >>> ph.nextFreeTuple() == 0
  True

  >>> ph.numTuples()
  1

  >>> tuplesToTest = 10
  >>> [ph.nextFreeTuple() for i in range(0, tuplesToTest)]
  [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
  
  >>> ph.numTuples() == tuplesToTest+1
  True

  >>> ph.hasFreeTuple()
  True

  # Check space utilization
  >>> ph.usedSpace() == (tuplesToTest+1)*ph.tupleSize
  True

  >>> ph.freeSpace() == 4096 - (ph.headerSize() + ((tuplesToTest+1) * ph.tupleSize))
  True

  >>> remainingTuples = int(ph.freeSpace() / ph.tupleSize)

  # Fill the page.
  >>> [ph.nextFreeTuple() for i in range(0, remainingTuples)] # doctest:+ELLIPSIS
  [11, 12, ...]

  >>> ph.hasFreeTuple()
  False

  # No value is returned when trying to exceed the page capacity.
  >>> ph.nextFreeTuple() == None
  True
  
  >>> ph.freeSpace() < ph.tupleSize
  True
  """
    def __init__(self, **kwargs):
        buffer = kwargs.get("buffer", None)
        self.flags = kwargs.get("flags", b'\x00')
        self.pageCapacity = kwargs.get("pageCapacity", len(buffer))
        self.tupleSize = kwargs.get("tupleSize", None)
        self.bitmap = kwargs.get("bitmap", None)

        if buffer == None:
            raise ValueError(
                "No backing buffer supplied for SlottedPageHeader")

        if self.bitmap == None:
            headerSizeWithoutBitmap = struct.Struct("cHHH").size
            tupleCapacity = math.floor(
                (8 * (self.pageCapacity - headerSizeWithoutBitmap)) /
                (1 + (8 * self.tupleSize)))
            bString = '0b' + ('0' * tupleCapacity)
            self.bitmap = BitArray(bString)

        self.binrepr = struct.Struct("cHHH" +
                                     str(math.ceil(len(self.bitmap))) + 's')
        self.size = self.binrepr.size
        self.freeSpaceOffset = self.size

        buffer[0:self.size] = self.pack()
#   super().__init__(buffer=buffer, flags=kwargs.get("flags", b'\x00'), self.tupleSize)

    def __eq__(self, other):
        return (self.flags == other.flags and self.tupleSize == other.tupleSize
                and self.pageCapacity == other.pageCapacity
                and self.freeSpaceOffset == other.freeSpaceOffset
                and self.bitmap == other.bitmap)

    def __hash__(self):
        return hash((self.flags, self.tupleSize, self.pageCapacity,
                     self.freeSpaceOffset, self.bitmap))

    def headerSize(self):
        return self.size

    # Flag operations.
    def flag(self, mask):
        return (ord(self.flags) & mask) > 0

    def setFlag(self, mask, set):
        if set:
            self.flags = bytes([ord(self.flags) | mask])
        else:
            self.flags = bytes([ord(self.flags) & ~mask])

    # Dirty bit accessors
    def isDirty(self):
        return self.flag(PageHeader.dirtyMask)

    def setDirty(self, dirty):
        self.setFlag(PageHeader.dirtyMask, dirty)

    def numTuples(self):
        return self.bitmap.count(1)

    # Returns the space available in the page associated with this header.
    def freeSpace(self):
        return self.pageCapacity - (self.size +
                                    (self.numTuples() * self.tupleSize))

    # Returns the space used in the page associated with this header.
    def usedSpace(self):
        return (self.numTuples() * self.tupleSize)

    # Slot operations.
    def offsetOfSlot(self, slot):
        return slot * self.tupleSize + self.size

    def hasSlot(self, slotIndex):
        return slotIndex < len(self.bitmap)

    def getSlot(self, slotIndex):
        return self.bitmap[slotIndex] == '0b1'

    def setSlot(self, slotIndex, slot):
        if not self.hasSlot(slotIndex):
            return

        if slot == True:
            self.bitmap[slotIndex] = '0b1'
        elif slot == False:
            self.bitmap[slotIndex] = '0b0'

    def resetSlot(self, slotIndex):
        self.setSlot(slotIndex, False)

    def freeSlots(self):
        freeList = []
        for i in range(len(self.bitmap)):
            if self.bitmap[i] == '0b0':
                freeList.append(i)
        return freeList

    def usedSlots(self):
        usedList = []
        for i in range(len(self.bitmap)):
            if self.bitmap[i] == '0b1':
                usedList.append(i)
        return usedList

    # Tuple allocation operations.

    # Returns whether the page has any free space for a tuple.
    def hasFreeTuple(self):
        return self.freeSpace() >= self.tupleSize
        # findTuple = self.bitmap.find('0b0')
        # if findTuple == ():
        #   return False
        # else:
        #   return True

    # Returns the tupleIndex of the next free tuple.
    # This should also "allocate" the tuple, such that any subsequent call
    # does not yield the same tupleIndex.
    def nextFreeTuple(self):
        #nextTuple = self.bitmap.find('0b0')

        #if nextTuple == ():
        #  return None

        #headerSizeWithoutBitmap = struct.Struct("cHHH").size
        #tupleCapacity = math.floor((8*(self.pageCapacity-headerSizeWithoutBitmap))/(1+(8*self.tupleSize)))
        #if nextTuple[0] >= tupleCapacity:
        #  return None

        #self.bitmap[nextTuple[0]] = '0b1'
        #return nextTuple[0]
        if self.hasFreeTuple():
            nextTuple = self.bitmap.find('0b0')
            self.bitmap[nextTuple[0]] = '0b1'
            return nextTuple[0]
        else:
            return None

    def nextTupleRange(self):
        start = self.nextFreeTuple()
        end = start + self.tupleSize
        index = (start - self.size) // self.tupleSize
        return (index, start, end)

    # Create a binary representation of a slotted page header.
    # The binary representation should include the slot contents.
    def pack(self):
        byteArray = bytearray(self.bitmap)

        packed = self.binrepr.pack(self.flags, self.tupleSize,
                                   self.freeSpaceOffset, self.pageCapacity,
                                   byteArray)
        unpacked = self.binrepr.unpack_from(packed)

        return self.binrepr.pack(self.flags, self.tupleSize,
                                 self.freeSpaceOffset, self.pageCapacity,
                                 byteArray)

    # Create a slotted page header instance from a binary representation held in the given buffer.
    @classmethod
    def unpack(cls, buffer):
        binrepr1 = struct.Struct("cHHH")
        values1 = binrepr1.unpack_from(buffer)

        headerSizeWithoutBitmap = binrepr1.size
        tupleCapacity = math.floor(
            (8 * (values1[3] - headerSizeWithoutBitmap)) / (1 +
                                                            (8 * values1[1])))

        binrepr2 = struct.Struct("cHHH" + str(math.ceil(tupleCapacity)) + 's')
        values2 = binrepr2.unpack_from(buffer)

        bString = '0b' + ('0' * tupleCapacity)
        bitmap = BitArray(bString)

        index = 0
        for bit in values2[4]:
            if index >= tupleCapacity:
                break
            if bit:
                bitmap[index] = '0b1'
            else:
                bitmap[index] = '0b0'
            index = index + 1

        if len(values2) == 5:
            return cls(buffer=buffer,
                       flags=values2[0],
                       tupleSize=values2[1],
                       freeSpaceOffset=values2[2],
                       pageCapacity=values2[3],
                       bitmap=bitmap)