def test_clean(self):
cache = LRUCache(3)
cache[1] = '1'
cache[2] = '2'
cache[3] = '3'
self.assertEqual(3, len(cache))
cache.clean()
self.assertEqual(0, len(cache))
def test_instanciation_with_timeout_5sec(self):
cache = LRUCache(5, 5)
cache.put('first', 'first')
cache.put('second', 'second')
self.assertEqual(len(cache.keys()), 2)
time.sleep(7)
self.assertEqual(len(cache.keys()), 0)
cache.stop_timer()
def __init__(self):
edr_config = EDRConfig()
try:
with open(self.EDR_FACTIONS_CACHE, 'rb') as handle:
self.factions_cache = pickle.load(handle)
except:
self.factions_cache = LRUCache(edr_config.lru_max_size(),
edr_config.factions_max_age())
def test_get_move_front(self):
cache = LRUCache(2)
cache.set('foo', 10)
cache.set('bar', 20)
cache.get('foo')
cache.set('baz', 30)
self.assertEqual(cache.get('foo'), 10)
self.assertEqual(cache.get('bar'), None)
self.assertEqual(cache.get('baz'), 30)
def test_reset(self):
c = LRUCache(3)
c.put(4, 2)
c.put(1, 1)
c.put(2, 3)
# after reset, nothing existing in the cache
c.reset()
self.assertEqual(c.size(), 0)
self.assertEqual(c.cache(), {})
def test_get_element(self):
cache = LRUCache(3)
cache[1] = '1'
cache[2] = '2'
cache[3] = '3'
self.assertEqual('test', cache.get(11, 'test'))
self.assertEqual("[(1, '1'), (2, '2'), (3, '3')]", str(cache))
self.assertEqual('1', cache.get(1, 'test'))
self.assertEqual("[(2, '2'), (3, '3'), (1, '1')]", str(cache))
self.assertEqual(3, len(cache))
def test_pop_element(self):
cache = LRUCache(3)
cache[1] = '1'
cache[2] = '2'
cache[3] = '3'
self.assertEqual("[(1, '1'), (2, '2'), (3, '3')]", str(cache))
self.assertEqual('1', cache.pop(1))
self.assertEqual("[(2, '2'), (3, '3')]", str(cache))
self.assertEqual(2, len(cache))
self.assertRaises(KeyError, cache.pop, 1)
def test_get_dict_copy_and_keys(self):
cache = LRUCache(3)
cache[1] = '1'
cache[2] = '2'
cache[3] = '3'
self.assertEqual({1: '1', 2: '2', 3: '3'}, cache.dict_copy())
keys = cache.keys()
# Sort keys to make sure they are in the same order on all platforms.
keys.sort()
self.assertEqual([1, 2, 3], keys)
self.assertEqual(3, len(cache))
def test_insert_over_capacity():
cache = LRUCache(3)
cache.put('key1', 'val1')
cache.put('key2', 'val2')
cache.put('key3', 'val3')
cache.put('key4', 'val4')
with pytest.raises(CacheMissException) as ex:
cache.get('key1')
assert 'key1' in str(ex)
assert cache.get('key2') == 'val2'
assert cache.get('key3') == 'val3'
assert cache.get('key4') == 'val4'
def test_keys_appended_to_lru_cache_are_in_correct_order_after_get(self):
context = LRUCache(capacity=5)
context.set("Michael", "Jordan")
context.set("Scotty", "Pippen")
context.get("Michael")
expected = OrderedDict([("Scotty", "Pippen"), ("Michael", "Jordan")])
assert context.cache == expected
def __init__(self, server):
self.server = server
self.timespan = None
self.records_check_interval = None
config = EDRConfig()
try:
with open(self.EDR_LEGAL_RECORDS_CACHE, 'rb') as handle:
self.records = pickle.load(handle)
except:
self.records = LRUCache(config.lru_max_size(), config.legal_records_max_age())
self.timespan = config.legal_records_recent_threshold()
self.records_check_interval = config.legal_records_check_interval()
def test_reset(self):
cache = LRUCache(3,60)
cache.set("a", 34)
cache.reset()
self.assertEqual(cache.keys(), [])
self.assertEqual(cache.values(), [])
self.assertEqual(cache.last_updated, None)
def test_key_is_removed_from_lru_cache_after_accessed_10_times(self):
context = LRUCache(capacity=5)
context.set("Michael", "Jordan")
context.set("Scotty", "Pippen")
for _ in range(11):
context.get("Michael")
expected = OrderedDict([("Scotty", "Pippen")])
assert context.cache == expected
def test_updating_item():
lru_cache = LRUCache(capacity=3)
lru_cache[10] = 5
lru_cache[20] = 7
lru_cache[40] = 9
lru_cache[20] = 10
assert list(lru_cache) == [(20, 10), (40, 9), (10, 5)]
def test_lru_remember_everything(self):
cache = LRUCache(5)
cache[1] = '1'
cache[2] = '2'
cache[3] = '3'
self.assertEqual("[(1, '1'), (2, '2'), (3, '3')]", str(cache))
self.assertEqual(3, len(cache))
def test_getting_middle_item():
lru_cache = LRUCache(capacity=3)
lru_cache[10] = 5
lru_cache[20] = 7
lru_cache[40] = 9
assert lru_cache[20] == 7
assert list(lru_cache) == [(20, 7), (40, 9), (10, 5)]
def test_getting_last_item_2():
lru_cache = LRUCache(capacity=3)
lru_cache[10] = 5
lru_cache[20] = 7
lru_cache[40] = 9
assert lru_cache[10] == 5
assert list(lru_cache) == [(10, 5), (40, 9), (20, 7)]
def __init__(self, cache_dir,
cache_size_unblock=4096, cache_size_block=8192):
"""Cache for block test result.
Two caches for block/unblock are used, so we can set different
cache size for blocked or unblocked url cache.
"""
self.save_trigger = 64
self.cache_dir = cache_dir
self.cache_filename_unblock = os.path.join(cache_dir,
self.filename_unblock)
self.cache_filename_block = os.path.join(cache_dir, self.filename_block)
from lrucache import LRUCache
self.cache_unblock = LRUCache(cache_size_unblock)
self.cache_block = LRUCache(cache_size_block)
def __init__(self, cache_dir,
cache_size_unblock=4096, cache_size_block=8192):
"""Cache for block test result.
Two caches for block/unblock are used, so we can set different
cache size for blocked or unblocked url cache.
"""
self.save_trigger = 64
self.cache_dir = cache_dir
self.cache_filename_unblock = os.path.join(cache_dir,
self.filename_unblock)
self.cache_filename_block = os.path.join(cache_dir, self.filename_block)
from lrucache import LRUCache
self.cache_unblock = LRUCache(cache_size_unblock)
self.cache_block = LRUCache(cache_size_block)
def test_removing_item_from_cache():
lru_cache = LRUCache(capacity=3)
lru_cache[10] = 5
lru_cache[20] = 7
lru_cache[40] = 9
del lru_cache[10]
assert list(lru_cache) == [(40, 9), (20, 7)]
def test_adding_single_item_to_full_cache():
lru_cache = LRUCache(capacity=3)
lru_cache[10] = 5
lru_cache[20] = 5
lru_cache[40] = 7
lru_cache[50] = 9
assert list(lru_cache) == [(50, 9), (40, 7), (20, 5)]
def __getitem__(self, key):
item = None
self.acquire()
try:
item = LRUCache.__getitem__(self, key)
finally:
self.release()
return item
def test_adding_multiple_items_to_full_cache():
lru_cache = LRUCache(capacity=3)
lru_cache[10] = 5
lru_cache[20] = 5
lru_cache[40] = 7
lru_cache[50] = 9
lru_cache[60] = 11
assert list(lru_cache) == [(60, 11), (50, 9), (40, 7)]
def test_lru_keep_fresh(self):
cache = LRUCache(3)
cache[1] = '1'
cache[2] = '2'
cache[3] = '3'
cache[4] = '4'
self.assertEqual("[(2, '2'), (3, '3'), (4, '4')]", str(cache))
self.assertEqual(3, len(cache))
def __getitem__(self, key):
item = None
self.acquire()
try:
item = LRUCache.__getitem__(self, key)
finally:
self.release()
return item
def test_add_item_after_removing():
lru_cache = LRUCache(capacity=3)
lru_cache[10] = 5
lru_cache[20] = 7
lru_cache[40] = 9
del lru_cache[10]
lru_cache[50] = 11
assert list(lru_cache) == [(50, 11), (40, 9), (20, 7)]
def test_basics(self):
cache = LRUCache(5, 60)
self.assertEqual(cache.capacity, 5)
sample = {"test": True, "foo": "bar"}
cache.set("a", sample)
result = cache.get("a")
self.assertEqual(result, sample)
result = cache.get("a")
self.assertDictEqual(result, sample)
cache.evict("a")
result = cache.get("a")
self.assertEqual(result, None)
def test_del_element(self):
cache = LRUCache(4)
cache[1] = '1'
cache[2] = '2'
cache[3] = '3'
cache[4] = '4'
del cache[3]
self.assertEqual("[(1, '1'), (2, '2'), (4, '4')]", str(cache))
self.assertEqual(3, len(cache))
def test_add_multiple_items_after_removing():
lru_cache = LRUCache(capacity=3)
lru_cache[10] = 5
lru_cache[20] = 7
lru_cache[40] = 9
del lru_cache[10]
lru_cache[50] = 11
lru_cache[60] = 13
assert list(lru_cache) == [(60, 13), (50, 11), (40, 9)]
def test_lru_fresh_on_access(self):
cache = LRUCache(3)
cache[1] = '1'
cache[2] = '2'
cache[3] = '3'
# Just access to first element.
_ = cache[1]
self.assertEqual("[(2, '2'), (3, '3'), (1, '1')]", str(cache))
self.assertEqual(3, len(cache))
def setup(self): self.lruc = LRUCache(10) self.lruc2 = LRUCache(10) self.lruc3 = LRUCache(10) self.lruc4 = LRUCache(10) self.lruc5 = LRUCache(10) self.lruc6 = LRUCache(10) self.lruc_test_store= LRUCache(6) self.lruc_test_store_same_item= LRUCache(10)
def test_stale(self):
cache = LRUCache(5, 1)
sample = {"test": True, "foo": "bar"}
key = "a"
cache.set(key, sample)
self.assertFalse(cache.is_stale(key))
time.sleep(1.2)
self.assertTrue(cache.is_stale(key))
self.assertEqual(cache.get("a"), None)
def __init__(self):
self.max = 0
self.previous_max = 0
self.min = 100.0
self.previous_min = 0
self.sum_scanned = 0
self.sum_awarded = 0
self.distribution = {"last_index": 0, "bins": [0]*25}
self.scanned_nb = 0
self.awarded_nb = 0
self.awarded_bounties = deque(maxlen=20)
self.scans = deque(maxlen=20)
self.efficiency = deque(maxlen=20)
self.max_efficiency = 5000000
self.max_normal_bounty = 350000 * 4 # appears to be the highest bounty per faction for NPC and the 4x boost from Nov 30th 2020
now = EDTime.py_epoch_now()
self.start = now
self.current = now
edr_config = EDRConfig()
self.scans_cache = LRUCache(edr_config.lru_max_size(), edr_config.blips_max_age())
self.last = {"timestamp": now, "bounty": None}
def __init__(self, func, targets=None, **kwargs):
if targets is None:
targets = []
elif not hasattr(targets, '__iter__'):
targets = [targets]
self.targets = targets
self.func = func
# Raise errors by default.
self.error = kwargs.pop('error', None)#'raise')
self.name = (
kwargs.get('name') or self.name or
(
self.func.__class__.__name__
if hasattr(func, '__class__') else self.func.__name__
)
)
# self.ids = set()
self.ids = LRUCache(max_size=10000)
self.idx = 0
self.idx_skip = 0
# Make sure we clean up if the program exits before the finalize
# functions are called.
if hasattr(self.func, 'finalize'):
atexit.register(self.finalize)
class TestLRUCache(object):
def setup(self):
self.lruc = LRUCache(10)
self.lruc2 = LRUCache(10)
self.lruc3 = LRUCache(10)
self.lruc4 = LRUCache(10)
self.lruc5 = LRUCache(10)
self.lruc6 = LRUCache(10)
self.lruc_test_store= LRUCache(6)
self.lruc_test_store_same_item= LRUCache(10)
##############################
# test construction
##############################
def test_init(self):
assert 0 == self.lruc._curSize
assert 10 == self.lruc.maxSize
##############################
# test private methods
##############################
def test_exceedesMaxSize(self):
assert True == self.lruc._willExceedesMaxSize(41)
assert False == self.lruc._willExceedesMaxSize(1)
def test_isInCache(self):
self.lruc2._add(Entry('b','boy',3))
assert True==self.lruc2._isInCache('b')
assert False==self.lruc2._isInCache('a') # we didn't put 'a' in this cache
self.lruc2.store('c','cat',3)
assert True ==self.lruc2._isInCache('c')
def test_add_entry(self):
beforeCacheLen = len(self.lruc._cache)
beforePQLen = len(self.lruc._ordering)
ent = Entry('a','apple',5)
self.lruc._add_entry(ent);
assert self.lruc._curSize == 5
assert self.lruc._cache['a'].value=='apple'
afterCacheLen = len(self.lruc._cache)
afterPQLen = len(self.lruc._ordering)
assert beforeCacheLen == beforePQLen ==0
assert afterCacheLen ==afterPQLen ==1
def test_add_entry_correct_priority(self):
ent = Entry('a','apple',5)
ent2 = Entry('b','cat',3)
# add some entries to the cache
self.lruc3._add_entry(ent)
self.lruc3._add_entry(ent2)
# test that they are in the right order
smallest = heapq.heappop(self.lruc3._ordering)
assert smallest == ent
def test_update_entry_correct_priority(self):
ent = Entry('a','apple',5)
time.sleep(0.001)
ent2 = Entry('b','cat',3)
self.lruc4._add_entry(ent)
self.lruc4._add_entry(ent2)
time.sleep(0.001)
ent.touch()
self.lruc4._update_entry(ent)
smallest = heapq.heappop(self.lruc4._ordering)
assert smallest == ent2
@raises(KeyError)
def test_evict(self):
ent = Entry('a','apple',5)
ent2 = Entry('b','boy',3)
self.lruc6._add_entry(ent)
self.lruc6._add_entry(ent2)
self.lruc6._evict()
assert len(self.lruc6._ordering) == len(self.lruc6._cache) == 1
assert_raises(KeyError, self.lruc6.fetch('a'), "a")
assert 'boy' == self.lruc6.fetch('b')
##############################
# test public api methods
##############################
def test_fetch(self):
ent = Entry('a','apple',5)
self.lruc = LRUCache(10)
self.lruc._add_entry(ent)
fetched = self.lruc.fetch(ent.key)
assert fetched =='apple'
@raises(KeyError)
def test_store(self):
assert len(self.lruc_test_store._ordering) == len(self.lruc_test_store._cache) == 0
self.lruc_test_store.store('a','apple',5)
assert len(self.lruc_test_store._ordering) == len(self.lruc_test_store._cache) == 1
self.lruc_test_store.store('b','boy',3)
assert len(self.lruc_test_store._ordering) == len(self.lruc_test_store._cache) == 1 # a got evicted
assert_raises(KeyError, self.lruc6.fetch('a'), "a")
assert 'boy' == self.lruc6.fetch('b') # only b is left
def store_same_value_test_helper(self):
lruc_test_store_same_item = LRUCache(10)
assert len(self.lruc_test_store_same_item._ordering) == len(self.lruc_test_store_same_item._cache) == 0
self.lruc_test_store_same_item.store('b','boy',3)
time.sleep(0.001)
self.lruc_test_store_same_item.store('c','cat',3)
time.sleep(0.001)
self.lruc_test_store_same_item.store('d','dog',3)
time.sleep(0.001)
assert len(self.lruc_test_store_same_item._ordering) == len(self.lruc_test_store_same_item._cache) == 3
boy = self.lruc_test_store_same_item.fetch('b') # update the oldest entry's timestamp
self.lruc_test_store_same_item.store('f','fun',3) # this evicts 'c' - cat, the next least recently used
assert len(self.lruc_test_store_same_item._ordering) == len(self.lruc_test_store_same_item._cache) == 3
# test to see that when trying to store same item again, updates the item's timestamp
def test_store_same_value(self):
self.store_same_value_test_helper()
fun = self.lruc_test_store_same_item.fetch('f')
assert 'fun' == fun
dog = self.lruc_test_store_same_item.fetch('d')
assert 'dog' == dog
boy = self.lruc_test_store_same_item.fetch('b')
assert 'boy' == boy
@raises(KeyError)
def test_store_same_value_with_keyerror(self):
self.store_same_value_test_helper()
cat = self.lruc_test_store_same_item.fetch('c') # raises key error, because 'c' was evicted
def __setitem__(self, key, obj):
self.acquire()
LRUCache.__setitem__(self, key, obj)
self.release()
class BlockCache:
filename_unblock = "lookup-cache-unblock.json"
filename_block = "lookup-cache-block.json"
def __init__(self, cache_dir,
cache_size_unblock=4096, cache_size_block=8192):
"""Cache for block test result.
Two caches for block/unblock are used, so we can set different
cache size for blocked or unblocked url cache.
"""
self.save_trigger = 64
self.cache_dir = cache_dir
self.cache_filename_unblock = os.path.join(cache_dir,
self.filename_unblock)
self.cache_filename_block = os.path.join(cache_dir, self.filename_block)
from lrucache import LRUCache
self.cache_unblock = LRUCache(cache_size_unblock)
self.cache_block = LRUCache(cache_size_block)
def load(self):
self.cache_unblock.load(self.cache_filename_unblock)
self.cache_block.load(self.cache_filename_block)
def make_key(self, url, *args):
key = url + json.dumps(args, sort_keys=True, indent=None,
separators= (',', ':'))
return key
def save(self, force=False):
trigger = 0 if force == True else self.save_trigger
if self.cache_unblock.insert_count > trigger:
self.cache_unblock.save(self.cache_filename_unblock)
self.cache_unblock.reset_insert_count()
if self.cache_block.insert_count > trigger:
self.cache_block.save(self.cache_filename_block)
self.cache_unblock.reset_insert_count()
def __getitem__(self, key):
if key in self.cache_unblock:
ret = self.cache_unblock[key]
else:
ret = self.cache_block[key]
return ret
def __setitem__(self, key, value):
if value:
self.cache_block[key] = value
else:
self.cache_unblock[key] = value
def __contains__(self, key):
return key in self.cache_block or key in self.cache_unblock
def test_fetch(self):
ent = Entry('a','apple',5)
self.lruc = LRUCache(10)
self.lruc._add_entry(ent)
fetched = self.lruc.fetch(ent.key)
assert fetched =='apple'
def test_update_one():
cache = LRUCache(3)
cache.put('key', 'val')
cache.put('key', 'new val')
assert cache.get('key') == 'new val'
class Layer(object):
"""
Class Layer
Description:
Constructors:
Layer(map, layername)
Methods:
IO operations
--------------
open(mode) Opens a MapSend layer filepair
.lay/.clt for read (mode='r') or
write (mode='w').
close() Closes an opened MapSend layer filepair
read() Read index file and header of layer file
write(outlayername) Read contents of layer that is opened for
read and write a copy of the layer to
layer name outlayername.
"""
def __init__(self, m, name, filename, layertype=None, nlevels=0,
fileidentifier=None):
self.map = m
## Copy resolution and reference point from map
self._scale = m.scale # [xscale, yscale]
self._refpoint = m.refpoint # Reference point for conversion to discrete coordinate
self._bbox = None
self._dbbox = None
self.nlevels = nlevels # Cell levels
if m.bboxrec:
self.dbboxrec = m.bboxrec.todiscrete(self._refpoint, self._scale)
self.estimator = None
else:
## Create layer parameter estimator object
self.estimator = LayerParamEstimator(self)
if filename[0:len(m.mapnumstr)] != m.mapnumstr:
filename = m.mapnumstr + filename
if len(filename) > 8:
raise Exception('Length of filename %s must not exceed 8'%filename)
self.name = name
self.filename = filename
self.cellelementid = 0
self.isopen = False
self.clearCells()
self.mode = None
self.bigendian = m.bigendian
self.writedrc = False # Write DRC file needed by mapsend software
self.nobjects = 0
self.category = 0 # 0=Normal layer, 1=Artificial layer
self.fileidentifier = fileidentifier
self.layertype = layertype
## Statistics from header
self.firstcell = None
self.lastcell = None
## Cell position in layer file (for use in read mode)
self.cellfilepos = {}
self.fhlay = None
self.draworder = 0
self.packed = False
self.packer = None
def __eq__(self, a):
return isinstance(a, Layer) and self.name == a.name
def __hash__(self):
return hash(self.name)
def clearCells(self):
self.modifiedcells = {} # Dictionary of modified cells keyed by cellnumber
self.cellcache = LRUCache(size=32)
self.cellfilepos = {}
self.cellnumbers = []
def setUnpackTable(self, filename):
self.packed = True
self.packer = layerpacker.LayerPacker(self.map.mapdir.open(filename).read())
def open(self, mode):
self.mode = mode
if not self.isopen:
if mode=='r' or mode=='a':
try:
# First try open the layer as little endian
self.layerfilename = self.filename+".lay"
self.indexfilename = self.filename+".clt"
self.fhlay = self.map.mapdir.open(self.layerfilename,"r")
self.bigendian = False
except IOError:
self.layerfilename = self.filename+".yal"
self.indexfilename = self.filename+".tlc"
self.fhlay = self.map.mapdir.open(self.layerfilename,"r")
self.bigendian = True
elif mode=='w':
if not self.bigendian:
self.layerfilename = self.filename+".lay"
self.indexfilename = self.filename+".clt"
else:
self.layerfilename = self.filename+".yal"
self.indexfilename = self.filename+".tlc"
if not self.map.inmemory:
self.shelffile = tempfile.mktemp()
self.shelf = shelve.open(self.shelffile)
self.fhlay = self.map.mapdir.open(self.layerfilename,"wb")
isopen=True
if self.mode in ('r','a'):
self.read_index()
self.read_header()
def optimize(self):
"""Optimize nlevels parameter and calculate bounding box
This function only works when the cell is opened in write only mode
without bounding box.
The function works like this. A proper value for the nlevel parameter and a bounding box
are first estimated.
At the beginning there is only one cell but with the new nlevel value the number of cells
may grow. Hence the cell elements might have to be placed in new cells.
Returns a dictionary of mapping between old and new cellreferences
"""
if self.mode == 'w' and self._bbox == None:
remapdict = {}
logging.debug("Optimizing layer "+self.name)
dbboxrec = self.estimator.calculateDBBox()
if dbboxrec:
self.dbboxrec = dbboxrec
## Get nlevels estimate
self.nlevels = self.estimator.calculateNlevels()
## Adjust bounding box borders to get integer cellsize
if dbboxrec:
self.dbboxrec = dbboxrec
## Update the bounding box of cell 1
self.getCell(1).setbbox()
if self.nlevels > 0:
cellelements = []
cellelementrefs = []
oldcell1 = self.getCell(1)
self.clearCells()
## Loop over the elements in the old cell 1
## The elements need to be accessed in reversed order, otherwise the
## cellelement numbers would change
## during the loop
for i in range(len(oldcell1)-1,-1,-1):
ce = oldcell1.pop(i)
newcellrefs = self.addCellElement(ce)
remapdict[(oldcell1.cellnum, i)] = newcellrefs[0]
return remapdict
else:
return {}
def close(self):
if (self.mode=='w') or (self.mode=='a') and len(self.modifiedcells)>0:
## Use estimator to calculate bounding box
if self._bbox == None:
self.optimize()
tmplay = tempfile.NamedTemporaryFile('wb', delete=False)
self.write_header(tmplay)
## The cells must be written in cell number order
self.cellnumbers.sort()
# Merge unchanged cells with modified cells
for cellnum in self.cellnumbers:
if cellnum in self.modifiedcells:
cell = self.modifiedcells[cellnum]
celldata = cell.serialize()
# Update index
self.cellfilepos[cellnum] = [tmplay.tell(), len(celldata)]
tmplay.write(celldata)
else:
self.fhlay.seek(self.cellfilepos[cellnum][0])
celldata = self.fhlay.read(self.cellfilepos[cellnum][1])
self.cellfilepos[cellnum][0] = tmplay.tell()
tmplay.write(celldata)
# Rewind and write the header again with the new cell index statistics
tmplay.seek(0)
self.write_header(tmplay)
# Copy temporary file to layer file
tmplay.close()
tmplay = open(tmplay.name, 'rb')
shutil.copyfileobj(tmplay, self.fhlay)
tmplay.close()
os.unlink(tmplay.name)
# Create index file
fhidx = self.map.mapdir.open(self.indexfilename,"wb")
fhdrc = None
if self.writedrc:
fhdrc = self.map.mapdir.open(self.filename+".drc", "wb")
if fhdrc:
fhdrc.write( self.pack("I", len(self.cellnumbers)))
for cellnum in self.cellnumbers:
fhidx.write( self.pack("III", cellnum, self.cellfilepos[cellnum][0], self.cellfilepos[cellnum][1]) )
if fhdrc:
fhdrc.write( self.pack("III", cellnum, self.cellfilepos[cellnum][0], self.cellfilepos[cellnum][1]) )
fhidx.close()
if fhdrc:
fhdrc.close()
if self.fhlay:
self.fhlay.close()
if self.mode == 'w' and not self.map.inmemory:
os.unlink(self.shelffile)
isopen=False
def read_index(self):
if self.mode in ['r','a']:
fhidx = self.map.mapdir.open(self.indexfilename, "r")
self.cellfilepos = {}
self.cellnumbers = []
while 1:
data = fhidx.read(12)
if len(data) == 0: break
[cellnum,offset,cellsize] = self.unpack("3i",data)
self.cellfilepos[cellnum] = [offset,cellsize]
self.cellnumbers.append(cellnum)
def read_header(self):
self.dheader = self.fhlay.read(0x80)
[self.category] = self.unpack("i", self.dheader[4:])
[self.fileidentifier] = self.unpack("H", self.dheader[8:])
tmp = self.unpack("4f", self.dheader[0xa:])
self._bbox = Rec(N.array([tmp[0],tmp[2]]),
N.array([tmp[1],tmp[3]]))
[self.nlevels] = self.unpack("h", self.dheader[0x1a:])
[self.nobjects] = self.unpack("i", self.dheader[0x1c:])
[xscale] = self.unpack("d", self.dheader[0x20:])
[yscale] = self.unpack("d", self.dheader[0x28:])
self._scale = N.array([xscale, yscale])
self._refpoint = N.array(self.unpack("2f", self.dheader[0x30:]))
tmp = self.unpack("4i", self.dheader[0x38:])
self._dbbox = Rec(N.array([tmp[0],tmp[1]]),
N.array([tmp[2],tmp[3]]))
[self.layertype] = self.unpack("b", self.dheader[0x48:])
[unknown49] = self.unpack("b", self.dheader[0x49:])
[self.largestcellsize] = self.unpack("i", self.dheader[0x4a:])
[self.firstcell] = self.unpack("i", self.dheader[0x4e:])
[self.lastcell] = self.unpack("i", self.dheader[0x52:])
assert(unknown49, 0)
def header_info(self):
info = ""
info += "Category: 0x%x"%self.category + '\n'
info += "Fileidentifier: 0x%x"%self.fileidentifier +'\n'
info += "Number of levels: %d"%self.nlevels + '\n'
info += "Number of elements: %d"%self.nobjects + '\n'
info += "Bbox: %s"%self._bbox +'\n'
info += "Scale: %s"%str(self._scale) + '\n'
info += "Refpoint: %s"%str(self._refpoint) + '\n'
info += "Layer type: 0x%x"%self.layertype + '\n'
info += "Largest cell size: 0x%x"%self.largestcellsize + '\n'
info += "First cell: 0x%x"%self.firstcell + '\n'
info += "Last cell: 0x%x"%self.lastcell + '\n'
info += "Discrete Bbox: %s"%self._dbbox +'\n'
return info
def write_header(self, fh):
header = "MHGO"
header = header + self.pack("i", self.category)
if self.fileidentifier == None:
fileidentifier = 0xc000 | self.map.getLayerIndex(self)
else:
fileidentifier = self.fileidentifier
header = header + self.pack("H", fileidentifier)
header = header + self.pack("4f", self._bbox.minX(), self._bbox.maxX(),
self._bbox.minY(), self._bbox.maxY())
header = header + self.pack("h", self.nlevels)
header = header + self.pack("i", self.nobjects)
header = header + self.pack("d", self._scale[0])
header = header + self.pack("d", self._scale[1])
header = header + self.pack("f", self._refpoint[0])
header = header + self.pack("f", self._refpoint[1])
header = header + self.pack("4i", self._dbbox.minX(), self._dbbox.minY(), self._dbbox.maxX(), self._dbbox.maxY())
header = header + self.pack("b", self.layertype)
header = header + self.pack("b", 0)
if len(self.cellfilepos)>0:
largestcellsize = max([d[1] for d in self.cellfilepos.values()])
else:
largestcellsize = 0
header = header + self.pack("i", largestcellsize)
if len(self.cellnumbers) == 0:
header = header + self.pack("i", 0) # First cell number
header = header + self.pack("i", 0) # Last cell number
else:
header = header + self.pack("i", self.cellnumbers[0]) # First cell number
header = header + self.pack("i", self.cellnumbers[-1]) # Last cell number
header = header + chr(0)*(128-len(header))
fh.write(header)
return len(header)
def unpack(self,types,data):
if self.bigendian:
prefix=">"
else:
prefix="<"
return struct.unpack(prefix + types,
data[0:struct.calcsize(prefix+types)])
def pack(self, types, *data):
if self.bigendian:
prefix=">"
else:
prefix="<"
return struct.pack(prefix+types, *data)
def markCellModified(self, cellnum):
self.modifiedcells[cellnum] = self.getCell(cellnum)
def getCells(self):
for cn in self.cellnumbers:
yield self.getCell(cn)
def getCell(self, cellnum):
if cellnum in self.modifiedcells:
return self.modifiedcells[cellnum]
if cellnum in self.cellcache:
return self.cellcache[cellnum]
# New cell
if self.mode == 'w':
if self.map.inmemory:
cell = CellInMemory(self, cellnum)
elif self.nlevels == 0:
cell = CellShelve(self, cellnum)
else:
cell = CellCommonShelve(self, cellnum, self.shelf)
else:
cell = CellInMemory(self, cellnum)
# Deserialize cell if present in the cell index
if cellnum in self.cellfilepos:
self.fhlay.seek(self.cellfilepos[cellnum][0])
celldata = self.fhlay.read(self.cellfilepos[cellnum][1])
if self.packed:
celldata = self.packer.unpack(celldata)
cell.deSerialize(celldata)
self.cellcache[cellnum] = cell
return cell
def close_cell(self, cellnum):
self.cellcache.pop(cellnum)
def getCellElements(self):
for c in self.getCells():
for s in c.getCellElements():
yield s
def getCellElementsAndRefs(self):
for c in self.getCells():
for nincell, s in enumerate(c.getCellElements()):
yield (s, (c.cellnum, nincell))
def getCellElement(self, cellref):
"""Get cell element from a (cellnum, num_in_cell) pair """
(cellnum, num_in_cell) = cellref
if self.map.debug:
print "Cellcache: "+str(self.cellcache.keys())
cell = self.getCell(cellnum)
try:
cellelement = cell.getCellElement(num_in_cell)
except IndexError:
raise IndexError,"num_in_cell (%d) is outside the # of cellelements (%d) in cell %d, layer %s"%(num_in_cell,len(cell),cellnum, self.name)
return cellelement
def addCellElement(self, cellelem, cellnum = None):
"""Add cell element to layer. The element might be divided into smaller elements.
Returns list of (cellnum,# in cell) pairs"""
if self.mode in ('r', None):
raise ValueError('Layer must be opened in write or append mode to add cell elements')
## Calculate the minimum cell that contains the extents of the new element
if self._bbox != None:
if cellnum == None:
cellnum, level, dcellrec = get_best_cell(self._dbbox,
cellelem.dbboxrec.negY(),
self.nlevels)
# assert cellelem.bboxrec(self).iscoveredby(self.bboxrec, xmargin=self._scale[0], ymargin=self._scale[1]), "CellElement is outside layer boundaries:" + \
# str(self.bboxrec(self)) + " cellelement:" + str(cellelem.bboxrec(self))
else:
if self.nlevels > 0:
raise ValueError('Cannot add cell element to layer with nlevels>0 and no bounding box')
cellnum = 1
self.estimator.addCellElement(cellelem)
cellelem.cellnum = cellnum
cell = self.getCell(cellnum)
assert cell.bboxrec == None or \
cellelem.dbboxrec.iscoveredby(cell.dbboxrec), \
"Incorrect cell %d with bbox %s for cell element with bbox %s"%(cellnum, cell.dbboxrec, str(cellelem.dbboxrec))
nincell = cell.addCellElement(cellelem)
assert self.nlevels == 0 or nincell < 2**16
if not cell in self.modifiedcells:
self.modifiedcells[cellnum] = cell
if not cellnum in self.cellnumbers:
self.cellnumbers.append(cellnum)
self.nobjects += 1
return [(cellnum, nincell)]
def updateCellElement(self, cellelementref, cellelement):
"""the updateCellElement must be called when a cell element has been updated"""
self.getCell(cellelementref[0]).updateElement(cellelementref[1], cellelement)
def getName(self):
return self.name
def getFileName(self):
return self.filename
def getNObjects(self):
return self.nobjects
## Bounding box property
def get_bboxrec(self):
if self._bbox:
return self._bbox.negY()
else:
return None
def set_bboxrec(self, rec):
if self.mode == 'r':
raise ValueError("Can't change boundary rectangle in read-only mode")
self.dbboxrec = rec.todiscrete(self._refpoint, self._scale)
# If in append mode all cell elements must be re-added to fit the new
# cell boundaries
if self.mode == 'a':
cellelements = [e for e in self.getCellElements()]
self.clearCells()
for e in cellelements:
self.addCellElement(e)
bboxrec = property(get_bboxrec, set_bboxrec, "Bounding box rectangle")
def get_dbboxrec(self):
if self._dbbox:
return self._dbbox.negY()
else:
return None
def set_dbboxrec(self, drec):
first_time = self._dbbox == None
if not first_time and self.mode == 'r':
raise ValueError("Can't change boundary rectangle in read-only mode")
self._dbbox = drec.negY()
self._bbox = self._dbbox.tocontinous(self._refpoint, self._scale)
if self._dbbox.width % (2 ** self.nlevels) != 0 or self._dbbox.height % (2 ** self.nlevels) != 0:
logging.warn("bbox should be a multiple of minimum cell size, adjusting bbox borders")
n = 2 ** (self.nlevels + 1)
width = self._dbbox.width
height = self._dbbox.height
width += -width % n
height += -height % n
self._dbbox = Rec(self._dbbox.c1, self._dbbox.c1 + N.array(width, height))
# If in append mode all cell elements must be re-added to fit the new
# cell boundaries
if not first_time and self.mode == 'a':
cellelements = [e for e in self.getCellElements()]
self.clearCells()
for e in cellelements:
self.addCellElement(e)
dbboxrec = property(get_dbboxrec, set_dbboxrec, "Bounding box rectangle discrete coordinates")
@property
def refpoint(self): return self._refpoint
@property
def scale(self): return self._scale
def getLayerType(self): return self.layertype
def calc_cell_extents(self, cellnum):
"""
Calculate discrete bounding box of a cell
Note, the extents return is in the internal coordinates with negated Y-values
"""
## Calculate cell level
level=0
while cellnum > totcells_at_level(level):
level=level+1
n = 2**level # Number of rows/cols
lbb = self._dbbox
layerwidth = lbb.width
layerheight = lbb.height
layersize = N.array([layerwidth, layerheight])
relcnum = cellnum - (totcells_at_level(level-1)+1)
cellsize = layersize / n
if relcnum < n*n:
mincorner = N.array([relcnum % n, relcnum / n]) * cellsize
maxcorner = mincorner + cellsize
else:
relcnum = relcnum-n*n
mincorner = N.array([relcnum % (n + 1), relcnum / (n + 1)]) * cellsize - cellsize/2
maxcorner = mincorner + layersize / n
mincorner[N.where(mincorner < 0)] = 0
maxcorner[0] = min(maxcorner[0], layerwidth)
maxcorner[1] = min(maxcorner[1], layerheight)
return Rec(mincorner + lbb.c1, maxcorner + lbb.c1)
def layer_header_nok(self, pcnt):
"""Header check from magsendtool"""
pcnt /= 100.0
rc = 0
if abs(((self._bbox.maxY() -self._bbox.minY())/self._scale[1]) - (self._dbbox.maxY() - self._dbbox.minY())) > pcnt * (self._dbbox.maxY() - self._dbbox.minY()):
rc |= 1
if abs(((self._bbox.maxX() -self._bbox.minX())/self._scale[0]) - (self._dbbox.maxX() - self._dbbox.minX())) > pcnt * (self._dbbox.maxX() - self._dbbox.minX()):
rc |= 2
if self._refpoint[1] != 0.0 and abs(self._bbox.centerY() - self._refpoint[1])/self._scale[1] > 0.75:
rc |= 4
if self._refpoint[0] != 0.0 and abs(self._bbox.centerX() - self._refpoint[0])/self._scale[0] > 0.75:
rc |= 8
return rc
def check(self):
version=1
if self.layer_header_nok(0.1):
version+=1
if self.layer_header_nok(1.0):
version+=1
if self.layer_header_nok(5.0):
raise ValueError('Incorrect layer format rc=%d at 5%% error'%self.layer_header_nok(5.0))
return version
@property
def ncells(self):
"""Return the number of cells in the layer"""
return len(self.cellnumbers)
@property
def info(self):
res = "Name: "+self.getName()+"\n"
res += "Number of objects: "+str(self.getNObjects())+"\n"
res += "Number of cells: "+str(len(self.cellnumbers))+"\n"
res += "Reference point: "+str(self._refpoint)+"\n"
res += "Scale: "+str(self._scale)+"\n"
res += "Boundaries: "+str(self._bbox)+"\n"
res += "Discrete Boundaries: "+str(self._dbbox)+"\n"
if self.fileidentifier:
res += "Identifier: 0x%x\n"%self.fileidentifier
res += "# of levels: %d\n"%self.nlevels
res += "category: %d\n"%self.category
if self.layertype != None:
res += "layertype: %d\n"%self.layertype
res += 'reflat: %f\n'%self._refpoint[1]
res += 'reflon: %f\n'%self._refpoint[0]
if self.firstcell:
res += 'first cell: %d\n'%self.firstcell
if self.lastcell:
res += 'last cell: %d\n'%self.lastcell
return res
def float2discrete(self, points):
"""Convert list of coordinates from floating point to discrete coordinates"""
return ((N.array(points) - self.refpoint) / self.scale).round().astype(int)
def discrete2float(self, points):
"""Convert list of coordinates from discrete to floating point coordinates"""
return N.array(points) * self.scale + self.refpoint
def __repr__(self):
return self.__class__.__name__ + '(' + self.getName() + ')'
def clearCells(self):
self.modifiedcells = {} # Dictionary of modified cells keyed by cellnumber
self.cellcache = LRUCache(size=32)
self.cellfilepos = {}
self.cellnumbers = []
def test_insert_many():
cache = LRUCache(3)
cache.put('key', 'val')
cache.put('key2', 'val2')
cache.put('key3', 'val3')
assert cache.get('key3') == 'val3'
number_of_connections = int ( sys.argv[3] )
size_of_cache = int ( sys.argv[4] )
else:
print "Usage: \"python cache-routine.py server_name port_number number_of_connections size_of_cache\": Insufficient parameters"
exit()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# for reusing the same port without waiting for TIME_WAIT to expire
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((server_name, port_number))
# Will listen to only one connection at a time
s.listen( number_of_connections )
LRU = LRUCache( size_of_cache )
while True:
conn, addr = s.accept()
data = conn.recv(1024)
print "DEBUG: " + data
# convert the received data into JSON
json_data = json.loads(data)
"""
fetch the request method
GET denotes a GET cache request to check for hit/miss
POST denotes a SAVE request once a miss happens
"""
request_method = json_data["method"]
def test_insert_one():
cache = LRUCache(3)
cache.put('key', 'val')
assert cache.get('key') == 'val'
class CoroutineProcessor(object):
name = None
log_rate = 1000
log_rate_skip = 50
def __init__(self, func, targets=None, **kwargs):
if targets is None:
targets = []
elif not hasattr(targets, '__iter__'):
targets = [targets]
self.targets = targets
self.func = func
# Raise errors by default.
self.error = kwargs.pop('error', None)#'raise')
self.name = (
kwargs.get('name') or self.name or
(
self.func.__class__.__name__
if hasattr(func, '__class__') else self.func.__name__
)
)
# self.ids = set()
self.ids = LRUCache(max_size=10000)
self.idx = 0
self.idx_skip = 0
# Make sure we clean up if the program exits before the finalize
# functions are called.
if hasattr(self.func, 'finalize'):
atexit.register(self.finalize)
@coroutine
def run(self, *args, **kwargs):
self.targets = [ t.run(*args, **kwargs) for t in self.targets ]
# =============================================================
def send(res):
if isinstance(res, dict) or not hasattr(res, '__iter__'):
res = [res]
for item in res:
###############
LOGGER.debug(pformat(item))
###############
_id = item.get('id')
# ----------------------------------------
if self.idx % self.log_rate == 0:
LOGGER.debug(
"[{0}] ({1}) {2}".format(self.idx, self.name, _id)
)
# ----------------------------------------
self.idx += 1
if _id:
exists = self.ids.get(_id)
# if _id in self.ids:
if exists:
LOGGER.warning(
"[{0}] ({1}) Document already processed!".format(
_id, self.name
))
else:
# self.ids.add(_id)
self.ids[_id] = 1
# Stop processing this document?
meta = item.get('__meta__', {}) or {}
procs = set(meta.get('processors', []) or [])
procs.add(self.name)
meta['processors'] = list(procs)
item['__meta__'] = meta
if meta.get('skip'):
# -----------------------------------
if self.idx_skip % self.log_rate_skip == 0:
LOGGER.info("[{0}] ({1}) Skipped ({2}). Current: {3} ".format(
self.idx_skip, self.name, _id, pformat(item['__meta__'])
))
# -----------------------------------
self.idx_skip += 1
continue
# Pass the item to the next stage
# in the pipeline.
for target in self.targets:
target.send(item)
# =============================================================
try:
while True:
doc = (yield)
###############
LOGGER.debug(pformat(doc))
###############
if doc is StopIteration:
raise StopIteration
try:
res = self.func(doc, *args, **kwargs)
except Exception as e:
if self.error == 'raise':
raise
else:
msg = """
Error processing document:
Task: {}
Exception: {}
Traceback: {}
""".format(self.name, e, traceback.format_exc())
LOGGER.error(textwrap.dedent(msg))
if DEBUG:
self.to_pickle(doc)
else:
if res:
if res is StopIteration:
raise StopIteration
else:
send(res)
except StopIteration as e:
res = self.finalize()
if res:
# Process final batch, if any.
send(res)
for target in self.targets:
try:
target.send(StopIteration)
except StopIteration:
pass
except GeneratorExit:
LOGGER.warning("[{0}] Quitting...".format(self.name))
def to_pickle(self, doc):
with open(os.path.join(self.path_pickle, self.name or 'UNK'), 'ab') as f:
pickle.dump(doc, f)
def get_state(self):
try:
self.func.get_state()
except AttributeError as e:
raise NotImplementedError
def to_df(self):
try:
self.func.to_df()
except AttributeError as e:
raise NotImplementedError
def to_csv(self, _id=None):
try:
self.func.to_csv(_id=_id)
except AttributeError as e:
raise NotImplementedError
def to_json(self):
try:
self.func.to_json()
except AttributeError as e:
raise NotImplementedError
def finalize(self):
try:
res = self.func.finalize()
except AttributeError as e:
########################
LOGGER.error(
"""
Processor doesn't have a 'finalize' method: {}
""".format(
self.func.name if hasattr(self.func, 'name')
else self.func.__name__
)
)
########################
raise NotImplementedError
except Exception as e:
msg = """
Error finalizing task:
Processor: {}
Exception: {}
Traceback: {}
""".format(self.name, e, traceback.format_exc())
LOGGER.error(textwrap.dedent(msg))
else:
return res
def __eq__(self, other):
eq = (
other and
self.__class__ is other.__class__ and
self.__dict__ == other.__dict__
)
return eq
def __or__(self, other):
if isinstance(other, CoroutineProcessor):
return CoroutineProcessor(
self.func, self.targets + [other], error=self.error
)
elif callable(other):
return CoroutineProcessor(
self.func,
self.targets + [CoroutineProcessor(other, error=self.error)]
)
else:
raise Exception(
"{0} is not composable with {0}".format(self, other)
)
def __getitem__(self, item):
return self.targets.__getitem__(item)
def __len__(self):
return len(self.targets)
def __repr__(self):
return "{0}({1})".format(
self.__class__.__name__,
", ".join(repr(prs) for prs in self.targets)
)
def __call__(self, *args, **kwargs):
return self.run(*args, **kwargs)
def __init__(self, num):
LRUCache.__init__(self, num)
self.lock = threading.RLock()
def __setitem__(self, key, obj):
self.acquire()
try:
LRUCache.__setitem__(self, key, obj)
finally:
self.release()
