def test_pop_element(self):
cache = LRUCache(3)
cache[1] = '1'
cache[2] = '2'
cache[3] = '3'
self.assertEqual("[(1, '1'), (2, '2'), (3, '3')]", str(cache))
self.assertEqual('1', cache.pop(1))
self.assertEqual("[(2, '2'), (3, '3')]", str(cache))
self.assertEqual(2, len(cache))
self.assertRaises(KeyError, cache.pop, 1)
class Layer(object):
"""
Class Layer
Description:
Constructors:
Layer(map, layername)
Methods:
IO operations
--------------
open(mode) Opens a MapSend layer filepair
.lay/.clt for read (mode='r') or
write (mode='w').
close() Closes an opened MapSend layer filepair
read() Read index file and header of layer file
write(outlayername) Read contents of layer that is opened for
read and write a copy of the layer to
layer name outlayername.
"""
def __init__(self,
m,
name,
filename,
layertype=None,
nlevels=0,
fileidentifier=None):
self.map = m
## Copy resolution and reference point from map
self._scale = m.scale # [xscale, yscale]
self._refpoint = m.refpoint # Reference point for conversion to discrete coordinate
self._bbox = None
self._dbbox = None
self.nlevels = nlevels # Cell levels
if m.bboxrec:
self.dbboxrec = m.bboxrec.todiscrete(self._refpoint, self._scale)
self.estimator = None
else:
## Create layer parameter estimator object
self.estimator = LayerParamEstimator(self)
if filename[0:len(m.mapnumstr)] != m.mapnumstr:
filename = m.mapnumstr + filename
if len(filename) > 8:
raise Exception('Length of filename %s must not exceed 8' %
filename)
self.name = name
self.filename = filename
self.cellelementid = 0
self.isopen = False
self.clearCells()
self.mode = None
self.bigendian = m.bigendian
self.writedrc = False # Write DRC file needed by mapsend software
self.nobjects = 0
self.category = 0 # 0=Normal layer, 1=Artificial layer
self.fileidentifier = fileidentifier
self.layertype = layertype
## Statistics from header
self.firstcell = None
self.lastcell = None
## Cell position in layer file (for use in read mode)
self.cellfilepos = {}
self.fhlay = None
self.draworder = 0
self.packed = False
self.packer = None
def __eq__(self, a):
return isinstance(a, Layer) and self.name == a.name
def __hash__(self):
return hash(self.name)
def clearCells(self):
self.modifiedcells = {
} # Dictionary of modified cells keyed by cellnumber
self.cellcache = LRUCache(size=32)
self.cellfilepos = {}
self.cellnumbers = []
def setUnpackTable(self, filename):
self.packed = True
self.packer = layerpacker.LayerPacker(
self.map.mapdir.open(filename).read())
def open(self, mode):
self.mode = mode
if not self.isopen:
if mode == 'r' or mode == 'a':
try:
# First try open the layer as little endian
self.layerfilename = self.filename + ".lay"
self.indexfilename = self.filename + ".clt"
self.fhlay = self.map.mapdir.open(self.layerfilename, "r")
self.bigendian = False
except IOError:
self.layerfilename = self.filename + ".yal"
self.indexfilename = self.filename + ".tlc"
self.fhlay = self.map.mapdir.open(self.layerfilename, "r")
self.bigendian = True
elif mode == 'w':
if not self.bigendian:
self.layerfilename = self.filename + ".lay"
self.indexfilename = self.filename + ".clt"
else:
self.layerfilename = self.filename + ".yal"
self.indexfilename = self.filename + ".tlc"
if not self.map.inmemory:
self.shelffile = tempfile.mktemp()
self.shelf = shelve.open(self.shelffile)
self.fhlay = self.map.mapdir.open(self.layerfilename, "wb")
isopen = True
if self.mode in ('r', 'a'):
self.read_index()
self.read_header()
def optimize(self):
"""Optimize nlevels parameter and calculate bounding box
This function only works when the cell is opened in write only mode
without bounding box.
The function works like this. A proper value for the nlevel parameter and a bounding box
are first estimated.
At the beginning there is only one cell but with the new nlevel value the number of cells
may grow. Hence the cell elements might have to be placed in new cells.
Returns a dictionary of mapping between old and new cellreferences
"""
if self.mode == 'w' and self._bbox == None:
remapdict = {}
logging.debug("Optimizing layer " + self.name)
dbboxrec = self.estimator.calculateDBBox()
if dbboxrec:
self.dbboxrec = dbboxrec
## Get nlevels estimate
self.nlevels = self.estimator.calculateNlevels()
## Adjust bounding box borders to get integer cellsize
if dbboxrec:
self.dbboxrec = dbboxrec
## Update the bounding box of cell 1
self.getCell(1).setbbox()
if self.nlevels > 0:
cellelements = []
cellelementrefs = []
oldcell1 = self.getCell(1)
self.clearCells()
## Loop over the elements in the old cell 1
## The elements need to be accessed in reversed order, otherwise the
## cellelement numbers would change
## during the loop
for i in range(len(oldcell1) - 1, -1, -1):
ce = oldcell1.pop(i)
newcellrefs = self.addCellElement(ce)
remapdict[(oldcell1.cellnum, i)] = newcellrefs[0]
return remapdict
else:
return {}
def close(self):
if (self.mode
== 'w') or (self.mode == 'a') and len(self.modifiedcells) > 0:
## Use estimator to calculate bounding box
if self._bbox == None:
self.optimize()
tmplay = tempfile.NamedTemporaryFile('wb', delete=False)
self.write_header(tmplay)
## The cells must be written in cell number order
self.cellnumbers.sort()
# Merge unchanged cells with modified cells
for cellnum in self.cellnumbers:
if cellnum in self.modifiedcells:
cell = self.modifiedcells[cellnum]
celldata = cell.serialize()
# Update index
self.cellfilepos[cellnum] = [tmplay.tell(), len(celldata)]
tmplay.write(celldata)
else:
self.fhlay.seek(self.cellfilepos[cellnum][0])
celldata = self.fhlay.read(self.cellfilepos[cellnum][1])
self.cellfilepos[cellnum][0] = tmplay.tell()
tmplay.write(celldata)
# Rewind and write the header again with the new cell index statistics
tmplay.seek(0)
self.write_header(tmplay)
# Copy temporary file to layer file
tmplay.close()
tmplay = open(tmplay.name, 'rb')
shutil.copyfileobj(tmplay, self.fhlay)
tmplay.close()
os.unlink(tmplay.name)
# Create index file
fhidx = self.map.mapdir.open(self.indexfilename, "wb")
fhdrc = None
if self.writedrc:
fhdrc = self.map.mapdir.open(self.filename + ".drc", "wb")
if fhdrc:
fhdrc.write(self.pack("I", len(self.cellnumbers)))
for cellnum in self.cellnumbers:
fhidx.write(
self.pack("III", cellnum, self.cellfilepos[cellnum][0],
self.cellfilepos[cellnum][1]))
if fhdrc:
fhdrc.write(
self.pack("III", cellnum, self.cellfilepos[cellnum][0],
self.cellfilepos[cellnum][1]))
fhidx.close()
if fhdrc:
fhdrc.close()
if self.fhlay:
self.fhlay.close()
if self.mode == 'w' and not self.map.inmemory:
os.unlink(self.shelffile)
isopen = False
def read_index(self):
if self.mode in ['r', 'a']:
fhidx = self.map.mapdir.open(self.indexfilename, "r")
self.cellfilepos = {}
self.cellnumbers = []
while 1:
data = fhidx.read(12)
if len(data) == 0: break
[cellnum, offset, cellsize] = self.unpack("3i", data)
self.cellfilepos[cellnum] = [offset, cellsize]
self.cellnumbers.append(cellnum)
def read_header(self):
self.dheader = self.fhlay.read(0x80)
[self.category] = self.unpack("i", self.dheader[4:])
[self.fileidentifier] = self.unpack("H", self.dheader[8:])
tmp = self.unpack("4f", self.dheader[0xa:])
self._bbox = Rec(N.array([tmp[0], tmp[2]]), N.array([tmp[1], tmp[3]]))
[self.nlevels] = self.unpack("h", self.dheader[0x1a:])
[self.nobjects] = self.unpack("i", self.dheader[0x1c:])
[xscale] = self.unpack("d", self.dheader[0x20:])
[yscale] = self.unpack("d", self.dheader[0x28:])
self._scale = N.array([xscale, yscale])
self._refpoint = N.array(self.unpack("2f", self.dheader[0x30:]))
tmp = self.unpack("4i", self.dheader[0x38:])
self._dbbox = Rec(N.array([tmp[0], tmp[1]]), N.array([tmp[2], tmp[3]]))
[self.layertype] = self.unpack("b", self.dheader[0x48:])
[unknown49] = self.unpack("b", self.dheader[0x49:])
[self.largestcellsize] = self.unpack("i", self.dheader[0x4a:])
[self.firstcell] = self.unpack("i", self.dheader[0x4e:])
[self.lastcell] = self.unpack("i", self.dheader[0x52:])
assert (unknown49, 0)
def header_info(self):
info = ""
info += "Category: 0x%x" % self.category + '\n'
info += "Fileidentifier: 0x%x" % self.fileidentifier + '\n'
info += "Number of levels: %d" % self.nlevels + '\n'
info += "Number of elements: %d" % self.nobjects + '\n'
info += "Bbox: %s" % self._bbox + '\n'
info += "Scale: %s" % str(self._scale) + '\n'
info += "Refpoint: %s" % str(self._refpoint) + '\n'
info += "Layer type: 0x%x" % self.layertype + '\n'
info += "Largest cell size: 0x%x" % self.largestcellsize + '\n'
info += "First cell: 0x%x" % self.firstcell + '\n'
info += "Last cell: 0x%x" % self.lastcell + '\n'
info += "Discrete Bbox: %s" % self._dbbox + '\n'
return info
def write_header(self, fh):
header = "MHGO"
header = header + self.pack("i", self.category)
if self.fileidentifier == None:
fileidentifier = 0xc000 | self.map.getLayerIndex(self)
else:
fileidentifier = self.fileidentifier
header = header + self.pack("H", fileidentifier)
header = header + self.pack("4f", self._bbox.minX(), self._bbox.maxX(),
self._bbox.minY(), self._bbox.maxY())
header = header + self.pack("h", self.nlevels)
header = header + self.pack("i", self.nobjects)
header = header + self.pack("d", self._scale[0])
header = header + self.pack("d", self._scale[1])
header = header + self.pack("f", self._refpoint[0])
header = header + self.pack("f", self._refpoint[1])
header = header + self.pack("4i", self._dbbox.minX(),
self._dbbox.minY(), self._dbbox.maxX(),
self._dbbox.maxY())
header = header + self.pack("b", self.layertype)
header = header + self.pack("b", 0)
if len(self.cellfilepos) > 0:
largestcellsize = max([d[1] for d in self.cellfilepos.values()])
else:
largestcellsize = 0
header = header + self.pack("i", largestcellsize)
if len(self.cellnumbers) == 0:
header = header + self.pack("i", 0) # First cell number
header = header + self.pack("i", 0) # Last cell number
else:
header = header + self.pack(
"i", self.cellnumbers[0]) # First cell number
header = header + self.pack(
"i", self.cellnumbers[-1]) # Last cell number
header = header + chr(0) * (128 - len(header))
fh.write(header)
return len(header)
def unpack(self, types, data):
if self.bigendian:
prefix = ">"
else:
prefix = "<"
return struct.unpack(prefix + types,
data[0:struct.calcsize(prefix + types)])
def pack(self, types, *data):
if self.bigendian:
prefix = ">"
else:
prefix = "<"
return struct.pack(prefix + types, *data)
def markCellModified(self, cellnum):
self.modifiedcells[cellnum] = self.getCell(cellnum)
def getCells(self):
for cn in self.cellnumbers:
yield self.getCell(cn)
def getCell(self, cellnum):
if cellnum in self.modifiedcells:
return self.modifiedcells[cellnum]
if cellnum in self.cellcache:
return self.cellcache[cellnum]
# New cell
if self.mode == 'w':
if self.map.inmemory:
cell = CellInMemory(self, cellnum)
elif self.nlevels == 0:
cell = CellShelve(self, cellnum)
else:
cell = CellCommonShelve(self, cellnum, self.shelf)
else:
cell = CellInMemory(self, cellnum)
# Deserialize cell if present in the cell index
if cellnum in self.cellfilepos:
self.fhlay.seek(self.cellfilepos[cellnum][0])
celldata = self.fhlay.read(self.cellfilepos[cellnum][1])
if self.packed:
celldata = self.packer.unpack(celldata)
cell.deSerialize(celldata)
self.cellcache[cellnum] = cell
return cell
def close_cell(self, cellnum):
self.cellcache.pop(cellnum)
def getCellElements(self):
for c in self.getCells():
for s in c.getCellElements():
yield s
def getCellElementsAndRefs(self):
for c in self.getCells():
for nincell, s in enumerate(c.getCellElements()):
yield (s, (c.cellnum, nincell))
def getCellElement(self, cellref):
"""Get cell element from a (cellnum, num_in_cell) pair """
(cellnum, num_in_cell) = cellref
if self.map.debug:
print "Cellcache: " + str(self.cellcache.keys())
cell = self.getCell(cellnum)
try:
cellelement = cell.getCellElement(num_in_cell)
except IndexError:
raise IndexError, "num_in_cell (%d) is outside the # of cellelements (%d) in cell %d, layer %s" % (
num_in_cell, len(cell), cellnum, self.name)
return cellelement
def addCellElement(self, cellelem, cellnum=None):
"""Add cell element to layer. The element might be divided into smaller elements.
Returns list of (cellnum,# in cell) pairs"""
if self.mode in ('r', None):
raise ValueError(
'Layer must be opened in write or append mode to add cell elements'
)
## Calculate the minimum cell that contains the extents of the new element
if self._bbox != None:
if cellnum == None:
cellnum, level, dcellrec = get_best_cell(
self._dbbox, cellelem.dbboxrec.negY(), self.nlevels)
# assert cellelem.bboxrec(self).iscoveredby(self.bboxrec, xmargin=self._scale[0], ymargin=self._scale[1]), "CellElement is outside layer boundaries:" + \
# str(self.bboxrec(self)) + " cellelement:" + str(cellelem.bboxrec(self))
else:
if self.nlevels > 0:
raise ValueError(
'Cannot add cell element to layer with nlevels>0 and no bounding box'
)
cellnum = 1
self.estimator.addCellElement(cellelem)
cellelem.cellnum = cellnum
cell = self.getCell(cellnum)
assert cell.bboxrec == None or \
cellelem.dbboxrec.iscoveredby(cell.dbboxrec), \
"Incorrect cell %d with bbox %s for cell element with bbox %s"%(cellnum, cell.dbboxrec, str(cellelem.dbboxrec))
nincell = cell.addCellElement(cellelem)
assert self.nlevels == 0 or nincell < 2**16
if not cell in self.modifiedcells:
self.modifiedcells[cellnum] = cell
if not cellnum in self.cellnumbers:
self.cellnumbers.append(cellnum)
self.nobjects += 1
return [(cellnum, nincell)]
def updateCellElement(self, cellelementref, cellelement):
"""the updateCellElement must be called when a cell element has been updated"""
self.getCell(cellelementref[0]).updateElement(cellelementref[1],
cellelement)
def getName(self):
return self.name
def getFileName(self):
return self.filename
def getNObjects(self):
return self.nobjects
## Bounding box property
def get_bboxrec(self):
if self._bbox:
return self._bbox.negY()
else:
return None
def set_bboxrec(self, rec):
if self.mode == 'r':
raise ValueError(
"Can't change boundary rectangle in read-only mode")
self.dbboxrec = rec.todiscrete(self._refpoint, self._scale)
# If in append mode all cell elements must be re-added to fit the new
# cell boundaries
if self.mode == 'a':
cellelements = [e for e in self.getCellElements()]
self.clearCells()
for e in cellelements:
self.addCellElement(e)
bboxrec = property(get_bboxrec, set_bboxrec, "Bounding box rectangle")
def get_dbboxrec(self):
if self._dbbox:
return self._dbbox.negY()
else:
return None
def set_dbboxrec(self, drec):
first_time = self._dbbox == None
if not first_time and self.mode == 'r':
raise ValueError(
"Can't change boundary rectangle in read-only mode")
self._dbbox = drec.negY()
self._bbox = self._dbbox.tocontinous(self._refpoint, self._scale)
if self._dbbox.width % (2**self.nlevels) != 0 or self._dbbox.height % (
2**self.nlevels) != 0:
logging.warn(
"bbox should be a multiple of minimum cell size, adjusting bbox borders"
)
n = 2**(self.nlevels + 1)
width = self._dbbox.width
height = self._dbbox.height
width += -width % n
height += -height % n
self._dbbox = Rec(self._dbbox.c1,
self._dbbox.c1 + N.array(width, height))
# If in append mode all cell elements must be re-added to fit the new
# cell boundaries
if not first_time and self.mode == 'a':
cellelements = [e for e in self.getCellElements()]
self.clearCells()
for e in cellelements:
self.addCellElement(e)
dbboxrec = property(get_dbboxrec, set_dbboxrec,
"Bounding box rectangle discrete coordinates")
@property
def refpoint(self):
return self._refpoint
@property
def scale(self):
return self._scale
def getLayerType(self):
return self.layertype
def calc_cell_extents(self, cellnum):
"""
Calculate discrete bounding box of a cell
Note, the extents return is in the internal coordinates with negated Y-values
"""
## Calculate cell level
level = 0
while cellnum > totcells_at_level(level):
level = level + 1
n = 2**level # Number of rows/cols
lbb = self._dbbox
layerwidth = lbb.width
layerheight = lbb.height
layersize = N.array([layerwidth, layerheight])
relcnum = cellnum - (totcells_at_level(level - 1) + 1)
cellsize = layersize / n
if relcnum < n * n:
mincorner = N.array([relcnum % n, relcnum / n]) * cellsize
maxcorner = mincorner + cellsize
else:
relcnum = relcnum - n * n
mincorner = N.array([relcnum % (n + 1), relcnum /
(n + 1)]) * cellsize - cellsize / 2
maxcorner = mincorner + layersize / n
mincorner[N.where(mincorner < 0)] = 0
maxcorner[0] = min(maxcorner[0], layerwidth)
maxcorner[1] = min(maxcorner[1], layerheight)
return Rec(mincorner + lbb.c1, maxcorner + lbb.c1)
def layer_header_nok(self, pcnt):
"""Header check from magsendtool"""
pcnt /= 100.0
rc = 0
if abs(((self._bbox.maxY() - self._bbox.minY()) / self._scale[1]) -
(self._dbbox.maxY() - self._dbbox.minY())) > pcnt * (
self._dbbox.maxY() - self._dbbox.minY()):
rc |= 1
if abs(((self._bbox.maxX() - self._bbox.minX()) / self._scale[0]) -
(self._dbbox.maxX() - self._dbbox.minX())) > pcnt * (
self._dbbox.maxX() - self._dbbox.minX()):
rc |= 2
if self._refpoint[1] != 0.0 and abs(self._bbox.centerY(
) - self._refpoint[1]) / self._scale[1] > 0.75:
rc |= 4
if self._refpoint[0] != 0.0 and abs(self._bbox.centerX(
) - self._refpoint[0]) / self._scale[0] > 0.75:
rc |= 8
return rc
def check(self):
version = 1
if self.layer_header_nok(0.1):
version += 1
if self.layer_header_nok(1.0):
version += 1
if self.layer_header_nok(5.0):
raise ValueError(
'Incorrect layer format rc=%d at 5%% error' %
self.layer_header_nok(5.0))
return version
@property
def ncells(self):
"""Return the number of cells in the layer"""
return len(self.cellnumbers)
@property
def info(self):
res = "Name: " + self.getName() + "\n"
res += "Number of objects: " + str(self.getNObjects()) + "\n"
res += "Number of cells: " + str(len(self.cellnumbers)) + "\n"
res += "Reference point: " + str(self._refpoint) + "\n"
res += "Scale: " + str(self._scale) + "\n"
res += "Boundaries: " + str(self._bbox) + "\n"
res += "Discrete Boundaries: " + str(self._dbbox) + "\n"
if self.fileidentifier:
res += "Identifier: 0x%x\n" % self.fileidentifier
res += "# of levels: %d\n" % self.nlevels
res += "category: %d\n" % self.category
if self.layertype != None:
res += "layertype: %d\n" % self.layertype
res += 'reflat: %f\n' % self._refpoint[1]
res += 'reflon: %f\n' % self._refpoint[0]
if self.firstcell:
res += 'first cell: %d\n' % self.firstcell
if self.lastcell:
res += 'last cell: %d\n' % self.lastcell
return res
def float2discrete(self, points):
"""Convert list of coordinates from floating point to discrete coordinates"""
return ((N.array(points) - self.refpoint) /
self.scale).round().astype(int)
def discrete2float(self, points):
"""Convert list of coordinates from discrete to floating point coordinates"""
return N.array(points) * self.scale + self.refpoint
def __repr__(self):
return self.__class__.__name__ + '(' + self.getName() + ')'
class Layer(object):
"""
Class Layer
Description:
Constructors:
Layer(map, layername)
Methods:
IO operations
--------------
open(mode) Opens a MapSend layer filepair
.lay/.clt for read (mode='r') or
write (mode='w').
close() Closes an opened MapSend layer filepair
read() Read index file and header of layer file
write(outlayername) Read contents of layer that is opened for
read and write a copy of the layer to
layer name outlayername.
"""
def __init__(self, m, name, filename, layertype=None, nlevels=0,
fileidentifier=None):
self.map = m
## Copy resolution and reference point from map
self._scale = m.scale # [xscale, yscale]
self._refpoint = m.refpoint # Reference point for conversion to discrete coordinate
self._bbox = None
self._dbbox = None
self.nlevels = nlevels # Cell levels
if m.bboxrec:
self.dbboxrec = m.bboxrec.todiscrete(self._refpoint, self._scale)
self.estimator = None
else:
## Create layer parameter estimator object
self.estimator = LayerParamEstimator(self)
if filename[0:len(m.mapnumstr)] != m.mapnumstr:
filename = m.mapnumstr + filename
if len(filename) > 8:
raise Exception('Length of filename %s must not exceed 8'%filename)
self.name = name
self.filename = filename
self.cellelementid = 0
self.isopen = False
self.clearCells()
self.mode = None
self.bigendian = m.bigendian
self.writedrc = False # Write DRC file needed by mapsend software
self.nobjects = 0
self.category = 0 # 0=Normal layer, 1=Artificial layer
self.fileidentifier = fileidentifier
self.layertype = layertype
## Statistics from header
self.firstcell = None
self.lastcell = None
## Cell position in layer file (for use in read mode)
self.cellfilepos = {}
self.fhlay = None
self.draworder = 0
self.packed = False
self.packer = None
def __eq__(self, a):
return isinstance(a, Layer) and self.name == a.name
def __hash__(self):
return hash(self.name)
def clearCells(self):
self.modifiedcells = {} # Dictionary of modified cells keyed by cellnumber
self.cellcache = LRUCache(size=32)
self.cellfilepos = {}
self.cellnumbers = []
def setUnpackTable(self, filename):
self.packed = True
self.packer = layerpacker.LayerPacker(self.map.mapdir.open(filename).read())
def open(self, mode):
self.mode = mode
if not self.isopen:
if mode=='r' or mode=='a':
try:
# First try open the layer as little endian
self.layerfilename = self.filename+".lay"
self.indexfilename = self.filename+".clt"
self.fhlay = self.map.mapdir.open(self.layerfilename,"r")
self.bigendian = False
except IOError:
self.layerfilename = self.filename+".yal"
self.indexfilename = self.filename+".tlc"
self.fhlay = self.map.mapdir.open(self.layerfilename,"r")
self.bigendian = True
elif mode=='w':
if not self.bigendian:
self.layerfilename = self.filename+".lay"
self.indexfilename = self.filename+".clt"
else:
self.layerfilename = self.filename+".yal"
self.indexfilename = self.filename+".tlc"
if not self.map.inmemory:
self.shelffile = tempfile.mktemp()
self.shelf = shelve.open(self.shelffile)
self.fhlay = self.map.mapdir.open(self.layerfilename,"wb")
isopen=True
if self.mode in ('r','a'):
self.read_index()
self.read_header()
def optimize(self):
"""Optimize nlevels parameter and calculate bounding box
This function only works when the cell is opened in write only mode
without bounding box.
The function works like this. A proper value for the nlevel parameter and a bounding box
are first estimated.
At the beginning there is only one cell but with the new nlevel value the number of cells
may grow. Hence the cell elements might have to be placed in new cells.
Returns a dictionary of mapping between old and new cellreferences
"""
if self.mode == 'w' and self._bbox == None:
remapdict = {}
logging.debug("Optimizing layer "+self.name)
dbboxrec = self.estimator.calculateDBBox()
if dbboxrec:
self.dbboxrec = dbboxrec
## Get nlevels estimate
self.nlevels = self.estimator.calculateNlevels()
## Adjust bounding box borders to get integer cellsize
if dbboxrec:
self.dbboxrec = dbboxrec
## Update the bounding box of cell 1
self.getCell(1).setbbox()
if self.nlevels > 0:
cellelements = []
cellelementrefs = []
oldcell1 = self.getCell(1)
self.clearCells()
## Loop over the elements in the old cell 1
## The elements need to be accessed in reversed order, otherwise the
## cellelement numbers would change
## during the loop
for i in range(len(oldcell1)-1,-1,-1):
ce = oldcell1.pop(i)
newcellrefs = self.addCellElement(ce)
remapdict[(oldcell1.cellnum, i)] = newcellrefs[0]
return remapdict
else:
return {}
def close(self):
if (self.mode=='w') or (self.mode=='a') and len(self.modifiedcells)>0:
## Use estimator to calculate bounding box
if self._bbox == None:
self.optimize()
tmplay = tempfile.NamedTemporaryFile('wb', delete=False)
self.write_header(tmplay)
## The cells must be written in cell number order
self.cellnumbers.sort()
# Merge unchanged cells with modified cells
for cellnum in self.cellnumbers:
if cellnum in self.modifiedcells:
cell = self.modifiedcells[cellnum]
celldata = cell.serialize()
# Update index
self.cellfilepos[cellnum] = [tmplay.tell(), len(celldata)]
tmplay.write(celldata)
else:
self.fhlay.seek(self.cellfilepos[cellnum][0])
celldata = self.fhlay.read(self.cellfilepos[cellnum][1])
self.cellfilepos[cellnum][0] = tmplay.tell()
tmplay.write(celldata)
# Rewind and write the header again with the new cell index statistics
tmplay.seek(0)
self.write_header(tmplay)
# Copy temporary file to layer file
tmplay.close()
tmplay = open(tmplay.name, 'rb')
shutil.copyfileobj(tmplay, self.fhlay)
tmplay.close()
os.unlink(tmplay.name)
# Create index file
fhidx = self.map.mapdir.open(self.indexfilename,"wb")
fhdrc = None
if self.writedrc:
fhdrc = self.map.mapdir.open(self.filename+".drc", "wb")
if fhdrc:
fhdrc.write( self.pack("I", len(self.cellnumbers)))
for cellnum in self.cellnumbers:
fhidx.write( self.pack("III", cellnum, self.cellfilepos[cellnum][0], self.cellfilepos[cellnum][1]) )
if fhdrc:
fhdrc.write( self.pack("III", cellnum, self.cellfilepos[cellnum][0], self.cellfilepos[cellnum][1]) )
fhidx.close()
if fhdrc:
fhdrc.close()
if self.fhlay:
self.fhlay.close()
if self.mode == 'w' and not self.map.inmemory:
os.unlink(self.shelffile)
isopen=False
def read_index(self):
if self.mode in ['r','a']:
fhidx = self.map.mapdir.open(self.indexfilename, "r")
self.cellfilepos = {}
self.cellnumbers = []
while 1:
data = fhidx.read(12)
if len(data) == 0: break
[cellnum,offset,cellsize] = self.unpack("3i",data)
self.cellfilepos[cellnum] = [offset,cellsize]
self.cellnumbers.append(cellnum)
def read_header(self):
self.dheader = self.fhlay.read(0x80)
[self.category] = self.unpack("i", self.dheader[4:])
[self.fileidentifier] = self.unpack("H", self.dheader[8:])
tmp = self.unpack("4f", self.dheader[0xa:])
self._bbox = Rec(N.array([tmp[0],tmp[2]]),
N.array([tmp[1],tmp[3]]))
[self.nlevels] = self.unpack("h", self.dheader[0x1a:])
[self.nobjects] = self.unpack("i", self.dheader[0x1c:])
[xscale] = self.unpack("d", self.dheader[0x20:])
[yscale] = self.unpack("d", self.dheader[0x28:])
self._scale = N.array([xscale, yscale])
self._refpoint = N.array(self.unpack("2f", self.dheader[0x30:]))
tmp = self.unpack("4i", self.dheader[0x38:])
self._dbbox = Rec(N.array([tmp[0],tmp[1]]),
N.array([tmp[2],tmp[3]]))
[self.layertype] = self.unpack("b", self.dheader[0x48:])
[unknown49] = self.unpack("b", self.dheader[0x49:])
[self.largestcellsize] = self.unpack("i", self.dheader[0x4a:])
[self.firstcell] = self.unpack("i", self.dheader[0x4e:])
[self.lastcell] = self.unpack("i", self.dheader[0x52:])
assert(unknown49, 0)
def header_info(self):
info = ""
info += "Category: 0x%x"%self.category + '\n'
info += "Fileidentifier: 0x%x"%self.fileidentifier +'\n'
info += "Number of levels: %d"%self.nlevels + '\n'
info += "Number of elements: %d"%self.nobjects + '\n'
info += "Bbox: %s"%self._bbox +'\n'
info += "Scale: %s"%str(self._scale) + '\n'
info += "Refpoint: %s"%str(self._refpoint) + '\n'
info += "Layer type: 0x%x"%self.layertype + '\n'
info += "Largest cell size: 0x%x"%self.largestcellsize + '\n'
info += "First cell: 0x%x"%self.firstcell + '\n'
info += "Last cell: 0x%x"%self.lastcell + '\n'
info += "Discrete Bbox: %s"%self._dbbox +'\n'
return info
def write_header(self, fh):
header = "MHGO"
header = header + self.pack("i", self.category)
if self.fileidentifier == None:
fileidentifier = 0xc000 | self.map.getLayerIndex(self)
else:
fileidentifier = self.fileidentifier
header = header + self.pack("H", fileidentifier)
header = header + self.pack("4f", self._bbox.minX(), self._bbox.maxX(),
self._bbox.minY(), self._bbox.maxY())
header = header + self.pack("h", self.nlevels)
header = header + self.pack("i", self.nobjects)
header = header + self.pack("d", self._scale[0])
header = header + self.pack("d", self._scale[1])
header = header + self.pack("f", self._refpoint[0])
header = header + self.pack("f", self._refpoint[1])
header = header + self.pack("4i", self._dbbox.minX(), self._dbbox.minY(), self._dbbox.maxX(), self._dbbox.maxY())
header = header + self.pack("b", self.layertype)
header = header + self.pack("b", 0)
if len(self.cellfilepos)>0:
largestcellsize = max([d[1] for d in self.cellfilepos.values()])
else:
largestcellsize = 0
header = header + self.pack("i", largestcellsize)
if len(self.cellnumbers) == 0:
header = header + self.pack("i", 0) # First cell number
header = header + self.pack("i", 0) # Last cell number
else:
header = header + self.pack("i", self.cellnumbers[0]) # First cell number
header = header + self.pack("i", self.cellnumbers[-1]) # Last cell number
header = header + chr(0)*(128-len(header))
fh.write(header)
return len(header)
def unpack(self,types,data):
if self.bigendian:
prefix=">"
else:
prefix="<"
return struct.unpack(prefix + types,
data[0:struct.calcsize(prefix+types)])
def pack(self, types, *data):
if self.bigendian:
prefix=">"
else:
prefix="<"
return struct.pack(prefix+types, *data)
def markCellModified(self, cellnum):
self.modifiedcells[cellnum] = self.getCell(cellnum)
def getCells(self):
for cn in self.cellnumbers:
yield self.getCell(cn)
def getCell(self, cellnum):
if cellnum in self.modifiedcells:
return self.modifiedcells[cellnum]
if cellnum in self.cellcache:
return self.cellcache[cellnum]
# New cell
if self.mode == 'w':
if self.map.inmemory:
cell = CellInMemory(self, cellnum)
elif self.nlevels == 0:
cell = CellShelve(self, cellnum)
else:
cell = CellCommonShelve(self, cellnum, self.shelf)
else:
cell = CellInMemory(self, cellnum)
# Deserialize cell if present in the cell index
if cellnum in self.cellfilepos:
self.fhlay.seek(self.cellfilepos[cellnum][0])
celldata = self.fhlay.read(self.cellfilepos[cellnum][1])
if self.packed:
celldata = self.packer.unpack(celldata)
cell.deSerialize(celldata)
self.cellcache[cellnum] = cell
return cell
def close_cell(self, cellnum):
self.cellcache.pop(cellnum)
def getCellElements(self):
for c in self.getCells():
for s in c.getCellElements():
yield s
def getCellElementsAndRefs(self):
for c in self.getCells():
for nincell, s in enumerate(c.getCellElements()):
yield (s, (c.cellnum, nincell))
def getCellElement(self, cellref):
"""Get cell element from a (cellnum, num_in_cell) pair """
(cellnum, num_in_cell) = cellref
if self.map.debug:
print "Cellcache: "+str(self.cellcache.keys())
cell = self.getCell(cellnum)
try:
cellelement = cell.getCellElement(num_in_cell)
except IndexError:
raise IndexError,"num_in_cell (%d) is outside the # of cellelements (%d) in cell %d, layer %s"%(num_in_cell,len(cell),cellnum, self.name)
return cellelement
def addCellElement(self, cellelem, cellnum = None):
"""Add cell element to layer. The element might be divided into smaller elements.
Returns list of (cellnum,# in cell) pairs"""
if self.mode in ('r', None):
raise ValueError('Layer must be opened in write or append mode to add cell elements')
## Calculate the minimum cell that contains the extents of the new element
if self._bbox != None:
if cellnum == None:
cellnum, level, dcellrec = get_best_cell(self._dbbox,
cellelem.dbboxrec.negY(),
self.nlevels)
# assert cellelem.bboxrec(self).iscoveredby(self.bboxrec, xmargin=self._scale[0], ymargin=self._scale[1]), "CellElement is outside layer boundaries:" + \
# str(self.bboxrec(self)) + " cellelement:" + str(cellelem.bboxrec(self))
else:
if self.nlevels > 0:
raise ValueError('Cannot add cell element to layer with nlevels>0 and no bounding box')
cellnum = 1
self.estimator.addCellElement(cellelem)
cellelem.cellnum = cellnum
cell = self.getCell(cellnum)
assert cell.bboxrec == None or \
cellelem.dbboxrec.iscoveredby(cell.dbboxrec), \
"Incorrect cell %d with bbox %s for cell element with bbox %s"%(cellnum, cell.dbboxrec, str(cellelem.dbboxrec))
nincell = cell.addCellElement(cellelem)
assert self.nlevels == 0 or nincell < 2**16
if not cell in self.modifiedcells:
self.modifiedcells[cellnum] = cell
if not cellnum in self.cellnumbers:
self.cellnumbers.append(cellnum)
self.nobjects += 1
return [(cellnum, nincell)]
def updateCellElement(self, cellelementref, cellelement):
"""the updateCellElement must be called when a cell element has been updated"""
self.getCell(cellelementref[0]).updateElement(cellelementref[1], cellelement)
def getName(self):
return self.name
def getFileName(self):
return self.filename
def getNObjects(self):
return self.nobjects
## Bounding box property
def get_bboxrec(self):
if self._bbox:
return self._bbox.negY()
else:
return None
def set_bboxrec(self, rec):
if self.mode == 'r':
raise ValueError("Can't change boundary rectangle in read-only mode")
self.dbboxrec = rec.todiscrete(self._refpoint, self._scale)
# If in append mode all cell elements must be re-added to fit the new
# cell boundaries
if self.mode == 'a':
cellelements = [e for e in self.getCellElements()]
self.clearCells()
for e in cellelements:
self.addCellElement(e)
bboxrec = property(get_bboxrec, set_bboxrec, "Bounding box rectangle")
def get_dbboxrec(self):
if self._dbbox:
return self._dbbox.negY()
else:
return None
def set_dbboxrec(self, drec):
first_time = self._dbbox == None
if not first_time and self.mode == 'r':
raise ValueError("Can't change boundary rectangle in read-only mode")
self._dbbox = drec.negY()
self._bbox = self._dbbox.tocontinous(self._refpoint, self._scale)
if self._dbbox.width % (2 ** self.nlevels) != 0 or self._dbbox.height % (2 ** self.nlevels) != 0:
logging.warn("bbox should be a multiple of minimum cell size, adjusting bbox borders")
n = 2 ** (self.nlevels + 1)
width = self._dbbox.width
height = self._dbbox.height
width += -width % n
height += -height % n
self._dbbox = Rec(self._dbbox.c1, self._dbbox.c1 + N.array(width, height))
# If in append mode all cell elements must be re-added to fit the new
# cell boundaries
if not first_time and self.mode == 'a':
cellelements = [e for e in self.getCellElements()]
self.clearCells()
for e in cellelements:
self.addCellElement(e)
dbboxrec = property(get_dbboxrec, set_dbboxrec, "Bounding box rectangle discrete coordinates")
@property
def refpoint(self): return self._refpoint
@property
def scale(self): return self._scale
def getLayerType(self): return self.layertype
def calc_cell_extents(self, cellnum):
"""
Calculate discrete bounding box of a cell
Note, the extents return is in the internal coordinates with negated Y-values
"""
## Calculate cell level
level=0
while cellnum > totcells_at_level(level):
level=level+1
n = 2**level # Number of rows/cols
lbb = self._dbbox
layerwidth = lbb.width
layerheight = lbb.height
layersize = N.array([layerwidth, layerheight])
relcnum = cellnum - (totcells_at_level(level-1)+1)
cellsize = layersize / n
if relcnum < n*n:
mincorner = N.array([relcnum % n, relcnum / n]) * cellsize
maxcorner = mincorner + cellsize
else:
relcnum = relcnum-n*n
mincorner = N.array([relcnum % (n + 1), relcnum / (n + 1)]) * cellsize - cellsize/2
maxcorner = mincorner + layersize / n
mincorner[N.where(mincorner < 0)] = 0
maxcorner[0] = min(maxcorner[0], layerwidth)
maxcorner[1] = min(maxcorner[1], layerheight)
return Rec(mincorner + lbb.c1, maxcorner + lbb.c1)
def layer_header_nok(self, pcnt):
"""Header check from magsendtool"""
pcnt /= 100.0
rc = 0
if abs(((self._bbox.maxY() -self._bbox.minY())/self._scale[1]) - (self._dbbox.maxY() - self._dbbox.minY())) > pcnt * (self._dbbox.maxY() - self._dbbox.minY()):
rc |= 1
if abs(((self._bbox.maxX() -self._bbox.minX())/self._scale[0]) - (self._dbbox.maxX() - self._dbbox.minX())) > pcnt * (self._dbbox.maxX() - self._dbbox.minX()):
rc |= 2
if self._refpoint[1] != 0.0 and abs(self._bbox.centerY() - self._refpoint[1])/self._scale[1] > 0.75:
rc |= 4
if self._refpoint[0] != 0.0 and abs(self._bbox.centerX() - self._refpoint[0])/self._scale[0] > 0.75:
rc |= 8
return rc
def check(self):
version=1
if self.layer_header_nok(0.1):
version+=1
if self.layer_header_nok(1.0):
version+=1
if self.layer_header_nok(5.0):
raise ValueError('Incorrect layer format rc=%d at 5%% error'%self.layer_header_nok(5.0))
return version
@property
def ncells(self):
"""Return the number of cells in the layer"""
return len(self.cellnumbers)
@property
def info(self):
res = "Name: "+self.getName()+"\n"
res += "Number of objects: "+str(self.getNObjects())+"\n"
res += "Number of cells: "+str(len(self.cellnumbers))+"\n"
res += "Reference point: "+str(self._refpoint)+"\n"
res += "Scale: "+str(self._scale)+"\n"
res += "Boundaries: "+str(self._bbox)+"\n"
res += "Discrete Boundaries: "+str(self._dbbox)+"\n"
if self.fileidentifier:
res += "Identifier: 0x%x\n"%self.fileidentifier
res += "# of levels: %d\n"%self.nlevels
res += "category: %d\n"%self.category
if self.layertype != None:
res += "layertype: %d\n"%self.layertype
res += 'reflat: %f\n'%self._refpoint[1]
res += 'reflon: %f\n'%self._refpoint[0]
if self.firstcell:
res += 'first cell: %d\n'%self.firstcell
if self.lastcell:
res += 'last cell: %d\n'%self.lastcell
return res
def float2discrete(self, points):
"""Convert list of coordinates from floating point to discrete coordinates"""
return ((N.array(points) - self.refpoint) / self.scale).round().astype(int)
def discrete2float(self, points):
"""Convert list of coordinates from discrete to floating point coordinates"""
return N.array(points) * self.scale + self.refpoint
def __repr__(self):
return self.__class__.__name__ + '(' + self.getName() + ')'