def test_file_lock(self):
# Test a lock that becomes free during a waiting lock() call.
class Lock(threading.Thread):
def __init__(self, lock_file):
threading.Thread.__init__(self)
self.lock_file = lock_file
self.lock = FileLock(self.lock_file)
def run(self):
self.lock.lock()
time.sleep(0.2)
self.lock.unlock()
lock_thread = Lock(self.lock_file)
start_time = time.time()
lock_thread.start()
# wait until thread got the locked
while not lock_thread.lock._locked:
time.sleep(0.001)
# one lock that times out
assert_locked(self.lock_file)
# one lock that will get it after some time
l = FileLock(self.lock_file, timeout=0.3, step=0.001)
l.lock()
locked_for = time.time() - start_time
assert locked_for - 0.2 <= 0.1, 'locking took to long?! (rerun if not sure)'
#cleanup
l.unlock()
lock_thread.join()
def test_file_lock(self):
# Test a lock that becomes free during a waiting lock() call.
class Lock(threading.Thread):
def __init__(self, lock_file):
threading.Thread.__init__(self)
self.lock_file = lock_file
self.lock = FileLock(self.lock_file)
def run(self):
self.lock.lock()
time.sleep(0.2)
self.lock.unlock()
lock_thread = Lock(self.lock_file)
start_time = time.time()
lock_thread.start()
# wait until thread got the locked
while not lock_thread.lock._locked:
time.sleep(0.001)
# one lock that times out
assert_locked(self.lock_file)
# one lock that will get it after some time
l = FileLock(self.lock_file, timeout=0.3, step=0.001)
l.lock()
locked_for = time.time() - start_time
assert locked_for - 0.2 <=0.1, 'locking took to long?! (rerun if not sure)'
#cleanup
l.unlock()
lock_thread.join()
def lock(p=None):
l = FileLock(lock_file, timeout=60)
l.lock()
counter = int(open(count_file).read())
open(count_file, 'w').write(str(counter+1))
time.sleep(0.001)
l.unlock()
def lock(p=None):
l = FileLock(lock_file, timeout=60)
l.lock()
counter = int(open(count_file).read())
open(count_file, 'w').write(str(counter+1))
time.sleep(0.001)
l.unlock()
def assert_locked(lock_file, timeout=0.02, step=0.001):
assert os.path.exists(lock_file)
l = FileLock(lock_file, timeout=timeout, step=step)
try:
l.lock()
assert False, 'file was not locked'
except LockTimeout:
pass
def assert_locked(lock_file, timeout=0.02, step=0.001):
assert os.path.exists(lock_file)
l = FileLock(lock_file, timeout=timeout, step=step)
try:
l.lock()
assert False, 'file was not locked'
except LockTimeout:
pass
def lock(args):
lock_file, count_file = args
l = FileLock(lock_file.strpath, timeout=60)
l.lock()
counter = int(count_file.read())
count_file.write(str(counter+1))
time.sleep(0.001)
l.unlock()
class Lock(threading.Thread):
def __init__(self, lock_file):
threading.Thread.__init__(self)
self.lock_file = lock_file
self.lock = FileLock(self.lock_file)
def run(self):
self.lock.lock()
time.sleep(0.2)
self.lock.unlock()
class Lock(threading.Thread):
def __init__(self, lock_file):
threading.Thread.__init__(self)
self.lock_file = lock_file
self.lock = FileLock(self.lock_file)
def run(self):
self.lock.lock()
time.sleep(0.2)
self.lock.unlock()
def test_remove_on_unlock(self):
l = FileLock(self.lock_file, remove_on_unlock=True)
l.lock()
assert os.path.exists(self.lock_file)
l.unlock()
assert not os.path.exists(self.lock_file)
l.lock()
assert os.path.exists(self.lock_file)
os.remove(self.lock_file)
assert not os.path.exists(self.lock_file)
# ignore removed lock
l.unlock()
assert not os.path.exists(self.lock_file)
def test_remove_on_unlock(self):
l = FileLock(self.lock_file, remove_on_unlock=True)
l.lock()
assert os.path.exists(self.lock_file)
l.unlock()
assert not os.path.exists(self.lock_file)
l.lock()
assert os.path.exists(self.lock_file)
os.remove(self.lock_file)
assert not os.path.exists(self.lock_file)
# ignore removed lock
l.unlock()
assert not os.path.exists(self.lock_file)
def test_lock_cleanup(self):
old_lock_file = os.path.join(self.lock_dir, 'lock_old.lck')
l = FileLock(old_lock_file)
l.lock()
l.unlock()
mtime = os.stat(old_lock_file).st_mtime
mtime -= 7*60
os.utime(old_lock_file, (mtime, mtime))
l = self._create_lock()
l.unlock()
assert os.path.exists(old_lock_file)
assert os.path.exists(self.lock_file)
cleanup_lockdir(self.lock_dir)
assert not os.path.exists(old_lock_file)
assert os.path.exists(self.lock_file)
def test_lock_cleanup(self):
old_lock_file = os.path.join(self.lock_dir, 'lock_old.lck')
l = FileLock(old_lock_file)
l.lock()
l.unlock()
mtime = os.stat(old_lock_file).st_mtime
mtime -= 7 * 60
os.utime(old_lock_file, (mtime, mtime))
l = self._create_lock()
l.unlock()
assert os.path.exists(old_lock_file)
assert os.path.exists(self.lock_file)
cleanup_lockdir(self.lock_dir)
assert not os.path.exists(old_lock_file)
assert os.path.exists(self.lock_file)
def test_remove_on_unlock(self):
l = FileLock(self.lock_file, remove_on_unlock=True)
l.lock()
assert os.path.exists(self.lock_file)
l.unlock()
if is_win: # not removed on windows
assert os.path.exists(self.lock_file)
else:
assert not os.path.exists(self.lock_file)
if is_win:
# not possible to remove lock file when lock is held
pass
else:
l.lock()
assert os.path.exists(self.lock_file)
os.remove(self.lock_file)
assert not os.path.exists(self.lock_file)
# ignore removed lock
l.unlock()
assert not os.path.exists(self.lock_file)
class BundleDataV2(object):
BUNDLE_CACHE = {}
BUNDLESIZE = 128 # Quadkey Base
BUNDLESIZE2 = BUNDLESIZE**2 # Tiles in Bundle
INDEXSIZE = BUNDLESIZE2 * 8
SLACKSPACE = 4 # Slack Space
BUNDLE_BYTEORDER = "<" # little-endian
BUNDLE_HEADER_FORMAT = "4I3Q6I"
BUNDLE_INDEX_FORMAT = "%iQ" % BUNDLESIZE2
DEFAULT_BUNDLE_HEADER = collections.OrderedDict([
("version", 3),
("numRecords", BUNDLESIZE2),
("maxRecordSize", 0),
("offsetSize", 5),
("slackSpace", SLACKSPACE),
("fileSize", 64 + INDEXSIZE),
("userHeaderOffset", 40),
("userHeaderSize", 20 + INDEXSIZE),
("legacy1", 3),
("legacy2", 16),
("legacy3", BUNDLESIZE2),
("legacy4", 5),
("indexSize", INDEXSIZE)
])
def __init__(self, bundle, mode="read"):
self.mode = mode
self.filename = bundle + '.bundle'
self.filelock = FileLock(bundle + '.lck')
self.header = collections.OrderedDict(self.DEFAULT_BUNDLE_HEADER)
self.index = [4] * self.BUNDLESIZE2 # empty index
if not os.path.exists(self.filename):
self._init_bundle()
cache = self.BUNDLE_CACHE.get(self.filename, {})
if cache and cache['stat'][-4:] == os.stat(self.filename)[-4:]:
# log.debug("use BUNDLE_CACHE")
self.index = cache['index']
self.header = cache['header']
else:
self.__read_header()
self.__read_index()
self.BUNDLE_CACHE[self.filename] = {"stat": os.fstat(
self.filehandle.fileno()), "index": self.index, "header": self.header}
def _init_bundle(self):
log.info("Init Bundle %s" % self.filename)
ensure_directory(self.filename)
write_atomic(self.filename, struct.pack(*[self.BUNDLE_BYTEORDER + self.BUNDLE_HEADER_FORMAT + self.BUNDLE_INDEX_FORMAT] +
list(self.header.values()) +
self.index # empty index
))
def __read_header(self):
self.filehandle.seek(0)
header = struct.unpack(
self.BUNDLE_BYTEORDER + self.BUNDLE_HEADER_FORMAT, self.filehandle.read(64))
for value, key in zip(header, self.header.keys()):
self.header[key] = value
assert self.header["version"] is 3
assert self.header["fileSize"] == os.fstat(
self.filehandle.fileno()).st_size
def __read_index(self):
self.filehandle.seek(64)
self.index = list(struct.unpack(
self.BUNDLE_BYTEORDER + self.BUNDLE_INDEX_FORMAT, self.filehandle.read(self.header['indexSize'])))
def __tile_pos(self, col, row, level):
return (row % self.BUNDLESIZE) * self.BUNDLESIZE + col % self.BUNDLESIZE
def __get_tile_from_index(self, col, row, level):
tile_pos = self.__tile_pos(col, row, level)
index_value = self.index[tile_pos]
if index_value <= 4:
return 0, -1
size = index_value >> self.header['userHeaderOffset']
offset = index_value - (size << self.header['userHeaderOffset'])
return size, offset
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
return False
def tile_size(self, tile):
size, unused = self.__get_tile_from_index(*tile)
return size
def load_tile(self, tile):
size, offset = self.__get_tile_from_index(*tile.coord)
if size <= 0 or offset == -1:
return False
self.filehandle.seek(offset)
tile.source = ImageSource(BytesIO(self.filehandle.read(size)))
return True
def store_tile(self, data, tile):
size = len(data)
self.filehandle.seek(0, os.SEEK_END)
self.filehandle.write(struct.pack("<I", size))
self.index[self.__tile_pos(
*tile.coord)] = self.filehandle.tell() + (size << self.header['userHeaderOffset'])
self.filehandle.write(data)
self.header['fileSize'] = self.filehandle.tell()
self.header['maxRecordSize'] = max(self.header['maxRecordSize'], size)
self.filehandle.seek(0)
self.filehandle.write(struct.pack(
*[self.BUNDLE_BYTEORDER + self.BUNDLE_HEADER_FORMAT + self.BUNDLE_INDEX_FORMAT] + list(self.header.values()) + self.index))
return True
def remove_tile(self, tile):
self.index[self.__tile_pos(*tile.coord)] = 4
self.filehandle.seek(0)
self.filehandle.write(struct.pack(
*[self.BUNDLE_BYTEORDER + self.BUNDLE_HEADER_FORMAT + self.BUNDLE_INDEX_FORMAT] + list(self.header.values()) + self.index))
return True
def close(self):
if hasattr(self, '_%s__filehandle' % self.__class__.__name__) and not self.__filehandle.closed:
log.debug("close")
self.__filehandle.close()
if self.mode == "write":
self.BUNDLE_CACHE[self.filename] = {}
self.filelock.unlock()
@property
def filehandle(self):
if not hasattr(self, '_%s__filehandle' % self.__class__.__name__):
log.debug("open")
if self.mode == "write":
self.filelock.lock()
self.__filehandle = open(self.filename, mode="r+b")
else:
self.__filehandle = open(self.filename, mode="rb")
return self.__filehandle
def _create_lock(self):
lock = FileLock(self.lock_file)
lock.lock()
return lock
def _create_lock(self):
lock = FileLock(self.lock_file)
lock.lock()
return lock
