def test_file_lock(self):
# Test a lock that becomes free during a waiting lock() call.
class Lock(threading.Thread):
def __init__(self, lock_file):
threading.Thread.__init__(self)
self.lock_file = lock_file
self.lock = FileLock(self.lock_file)
def run(self):
self.lock.lock()
time.sleep(0.2)
self.lock.unlock()
lock_thread = Lock(self.lock_file)
start_time = time.time()
lock_thread.start()
# wait until thread got the locked
while not lock_thread.lock._locked:
time.sleep(0.001)
# one lock that times out
assert_locked(self.lock_file)
# one lock that will get it after some time
l = FileLock(self.lock_file, timeout=0.3, step=0.001)
l.lock()
locked_for = time.time() - start_time
assert locked_for - 0.2 <=0.1, 'locking took to long?! (rerun if not sure)'
#cleanup
l.unlock()
lock_thread.join()
def lock(p=None):
l = FileLock(lock_file, timeout=60)
l.lock()
counter = int(open(count_file).read())
open(count_file, 'w').write(str(counter+1))
time.sleep(0.001)
l.unlock()
def assert_locked(lock_file, timeout=0.02, step=0.001):
assert os.path.exists(lock_file)
l = FileLock(lock_file, timeout=timeout, step=step)
try:
l.lock()
assert False, 'file was not locked'
except LockTimeout:
pass
def assert_locked(lock_file, timeout=0.02, step=0.001):
assert os.path.exists(lock_file)
l = FileLock(lock_file, timeout=timeout, step=step)
try:
l.lock()
assert False, 'file was not locked'
except LockTimeout:
pass
class Lock(threading.Thread):
def __init__(self, lock_file):
threading.Thread.__init__(self)
self.lock_file = lock_file
self.lock = FileLock(self.lock_file)
def run(self):
self.lock.lock()
time.sleep(0.2)
self.lock.unlock()
class Lock(threading.Thread):
def __init__(self, lock_file):
threading.Thread.__init__(self)
self.lock_file = lock_file
self.lock = FileLock(self.lock_file)
def run(self):
self.lock.lock()
time.sleep(0.2)
self.lock.unlock()
def test_remove_on_unlock(self):
l = FileLock(self.lock_file, remove_on_unlock=True)
l.lock()
assert os.path.exists(self.lock_file)
l.unlock()
assert not os.path.exists(self.lock_file)
l.lock()
assert os.path.exists(self.lock_file)
os.remove(self.lock_file)
assert not os.path.exists(self.lock_file)
# ignore removed lock
l.unlock()
assert not os.path.exists(self.lock_file)
def ensure_mbtile(self):
if not os.path.exists(self.mbtile_file):
with FileLock(self.mbtile_file + '.init.lck',
remove_on_unlock=REMOVE_ON_UNLOCK):
if not os.path.exists(self.mbtile_file):
ensure_directory(self.mbtile_file)
self._initialize_mbtile()
def config(self, user, uuid=None, max_uuids=50):
conf_filename = os.path.join(self.cache_dir, (uuid or user) + '.yaml')
if self._requires_reconf(conf_filename):
with FileLock(conf_filename + '.lck'):
if self._requires_reconf(conf_filename):
log.debug('(re)configure %s for %s', (uuid or "all"), user)
if uuid:
params = tile_params(user, uuid, self.cartodb_domain)
if params is None:
return
conf = mapproxy_config([params], user=user)
else:
layers = []
try:
for uuid in user_uuids(user, max_uuids=max_uuids, cartodb_domain=self.cartodb_domain):
try:
params = tile_params(user, uuid, self.cartodb_domain)
if params:
layers.append(params)
else:
log.warn("found no layer for %s %s", user, uuid)
except RequestError as ex:
log.warn("faild to query tiler for %s %s: %s", user, uuid, ex)
except RequestError as ex:
log.warn("failed to get user uuids for user %s: %s", user, ex)
if not layers:
return
conf = mapproxy_config(layers, user=user)
write_mapproxy_config(conf, conf_filename)
return conf_filename
def ensure_mbtile(self):
if not os.path.exists(self.mbtile_file):
with FileLock(os.path.join(self.lock_dir, 'init.lck'),
timeout=self.lock_timeout,
remove_on_unlock=True):
if not os.path.exists(self.mbtile_file):
ensure_directory(self.mbtile_file)
self._initialize_mbtile()
def lock(self, tile):
"""
Returns a lock object for this tile.
"""
lock_filename = self.lock_filename(tile)
cleanup_lockdir(self.lock_dir, force=False)
return FileLock(lock_filename, timeout=self.lock_timeout,
remove_on_unlock=True)
def ensure_mbtile(self):
if not os.path.exists(self.mbtile_file):
with FileLock(os.path.join(os.path.dirname(self.mbtile_file),
'init.lck'),
remove_on_unlock=True):
if not os.path.exists(self.mbtile_file):
ensure_directory(self.mbtile_file)
self._initialize_mbtile()
def test_remove_on_unlock(self):
l = FileLock(self.lock_file, remove_on_unlock=True)
l.lock()
assert os.path.exists(self.lock_file)
l.unlock()
assert not os.path.exists(self.lock_file)
l.lock()
assert os.path.exists(self.lock_file)
os.remove(self.lock_file)
assert not os.path.exists(self.lock_file)
# ignore removed lock
l.unlock()
assert not os.path.exists(self.lock_file)
def remove_tile(self, tile):
if tile.coord is None:
return True
with FileLock(self.lock_filename):
with self.index().readwrite() as idx:
x, y = self._rel_tile_coord(tile.coord)
idx.remove_tile_offset(x, y)
return True
def lock(self, tile):
"""
Returns a lock object for this tile.
"""
if getattr(self, 'locking_disabled', False):
return DummyLock()
lock_filename = self.lock_filename(tile)
cleanup_lockdir(self.lock_dir, force=False)
return FileLock(lock_filename, timeout=self.lock_timeout,
remove_on_unlock=True)
def ensure_gpkg(self):
if not os.path.isfile(self.geopackage_file):
with FileLock(self.geopackage_file + '.init.lck',
remove_on_unlock=REMOVE_ON_UNLOCK):
ensure_directory(self.geopackage_file)
self._initialize_gpkg()
else:
if not self.check_gpkg():
ensure_directory(self.geopackage_file)
self._initialize_gpkg()
def remove_tile(self, tile):
if tile.coord is None:
return True
with FileLock(self.lock_filename):
idx = BundleIndex(self.base_filename + BUNDLEX_EXT)
x, y = self._rel_tile_coord(tile.coord)
idx.remove_tile_offset(x, y)
return True
def remove_tile(self, tile):
if tile.coord is None:
return True
self._init_index()
with FileLock(self.lock_filename):
with self._readwrite() as fh:
x, y = self._rel_tile_coord(tile.coord)
self._update_tile_offset(fh, x, y, 0, 0)
return True
def test_remove_on_unlock(self):
l = FileLock(self.lock_file, remove_on_unlock=True)
l.lock()
assert os.path.exists(self.lock_file)
l.unlock()
if is_win: # not removed on windows
assert os.path.exists(self.lock_file)
else:
assert not os.path.exists(self.lock_file)
if is_win:
# not possible to remove lock file when lock is held
pass
else:
l.lock()
assert os.path.exists(self.lock_file)
os.remove(self.lock_file)
assert not os.path.exists(self.lock_file)
# ignore removed lock
l.unlock()
assert not os.path.exists(self.lock_file)
def test_file_lock(self):
# Test a lock that becomes free during a waiting lock() call.
class Lock(threading.Thread):
def __init__(self, lock_file):
threading.Thread.__init__(self)
self.lock_file = lock_file
self.lock = FileLock(self.lock_file)
def run(self):
self.lock.lock()
time.sleep(0.2)
self.lock.unlock()
lock_thread = Lock(self.lock_file)
start_time = time.time()
lock_thread.start()
# wait until thread got the locked
while not lock_thread.lock._locked:
time.sleep(0.001)
# one lock that times out
assert_locked(self.lock_file)
# one lock that will get it after some time
l = FileLock(self.lock_file, timeout=0.3, step=0.001)
l.lock()
locked_for = time.time() - start_time
assert locked_for - 0.2 <= 0.1, 'locking took to long?! (rerun if not sure)'
#cleanup
l.unlock()
lock_thread.join()
def lock(p=None):
l = FileLock(lock_file, timeout=60)
l.lock()
counter = int(open(count_file).read())
open(count_file, 'w').write(str(counter+1))
time.sleep(0.001)
l.unlock()
def lock(args):
lock_file, count_file = args
l = FileLock(lock_file.strpath, timeout=60)
l.lock()
counter = int(count_file.read())
count_file.write(str(counter+1))
time.sleep(0.001)
l.unlock()
def to_csv(csv_config_file, cap_type, cap_url, layer_name, system_id, dimensions=None):
dimensions = serialize_dimensions(dimensions)
id = urlparse(cap_url).netloc + '_' + layer_name + '_' + system_id
if dimensions:
id += '_' + dimensions
id = re.sub('[^A-Za-z0-9-_]', '_', id)
with FileLock(csv_config_file + '.lck'):
records = read_csv(csv_config_file)
records[id] = (cap_type, cap_url, layer_name, system_id, dimensions)
write_csv(csv_config_file, records)
return id
def test_lock_cleanup(self):
old_lock_file = os.path.join(self.lock_dir, 'lock_old.lck')
l = FileLock(old_lock_file)
l.lock()
l.unlock()
mtime = os.stat(old_lock_file).st_mtime
mtime -= 7 * 60
os.utime(old_lock_file, (mtime, mtime))
l = self._create_lock()
l.unlock()
assert os.path.exists(old_lock_file)
assert os.path.exists(self.lock_file)
cleanup_lockdir(self.lock_dir)
assert not os.path.exists(old_lock_file)
assert os.path.exists(self.lock_file)
def store_tile(self, tile):
if tile.stored:
return True
with tile_buffer(tile) as buf:
data = buf.read()
with FileLock(self.lock_filename):
bundle = BundleData(self.base_filename + BUNDLE_EXT, self.offset)
idx = BundleIndex(self.base_filename + BUNDLEX_EXT)
x, y = self._rel_tile_coord(tile.coord)
offset = idx.tile_offset(x, y)
offset, size = bundle.append_tile(data, prev_offset=offset)
idx.update_tile_offset(x, y, offset=offset, size=size)
return True
def store_tiles(self, tiles):
self._init_index()
tiles_data = []
for t in tiles:
if t.stored:
continue
with tile_buffer(t) as buf:
data = buf.read()
tiles_data.append((t.coord, data))
with FileLock(self.lock_filename):
with self._readwrite() as fh:
for tile_coord, data in tiles_data:
self._store_tile(fh, tile_coord, data)
return True
def test_lock_cleanup(self):
old_lock_file = os.path.join(self.lock_dir, 'lock_old.lck')
l = FileLock(old_lock_file)
l.lock()
l.unlock()
mtime = os.stat(old_lock_file).st_mtime
mtime -= 7*60
os.utime(old_lock_file, (mtime, mtime))
l = self._create_lock()
l.unlock()
assert os.path.exists(old_lock_file)
assert os.path.exists(self.lock_file)
cleanup_lockdir(self.lock_dir)
assert not os.path.exists(old_lock_file)
assert os.path.exists(self.lock_file)
def __init__(self, bundle, mode="read"):
self.mode = mode
self.filename = bundle + '.bundle'
self.filelock = FileLock(bundle + '.lck')
self.header = collections.OrderedDict(self.DEFAULT_BUNDLE_HEADER)
self.index = [4] * self.BUNDLESIZE2 # empty index
if not os.path.exists(self.filename):
self._init_bundle()
cache = self.BUNDLE_CACHE.get(self.filename, {})
if cache and cache['stat'][-4:] == os.stat(self.filename)[-4:]:
# log.debug("use BUNDLE_CACHE")
self.index = cache['index']
self.header = cache['header']
else:
self.__read_header()
self.__read_index()
self.BUNDLE_CACHE[self.filename] = {"stat": os.fstat(
self.filehandle.fileno()), "index": self.index, "header": self.header}
def store_tiles(self, tiles):
tiles_data = []
for t in tiles:
if t.stored:
continue
with tile_buffer(t) as buf:
data = buf.read()
tiles_data.append((t.coord, data))
with FileLock(self.lock_filename):
with self.data().readwrite() as bundle:
with self.index().readwrite() as idx:
for tile_coord, data in tiles_data:
x, y = self._rel_tile_coord(tile_coord)
offset = idx.tile_offset(x, y)
offset, size = bundle.append_tile(data,
prev_offset=offset)
idx.update_tile_offset(x, y, offset=offset, size=size)
return True
def __init__(self, lock_file):
threading.Thread.__init__(self)
self.lock_file = lock_file
self.lock = FileLock(self.lock_file)
def __init__(self, lock_file):
threading.Thread.__init__(self)
self.lock_file = lock_file
self.lock = FileLock(self.lock_file)
class BundleDataV2(object):
BUNDLE_CACHE = {}
BUNDLESIZE = 128 # Quadkey Base
BUNDLESIZE2 = BUNDLESIZE**2 # Tiles in Bundle
INDEXSIZE = BUNDLESIZE2 * 8
SLACKSPACE = 4 # Slack Space
BUNDLE_BYTEORDER = "<" # little-endian
BUNDLE_HEADER_FORMAT = "4I3Q6I"
BUNDLE_INDEX_FORMAT = "%iQ" % BUNDLESIZE2
DEFAULT_BUNDLE_HEADER = collections.OrderedDict([
("version", 3),
("numRecords", BUNDLESIZE2),
("maxRecordSize", 0),
("offsetSize", 5),
("slackSpace", SLACKSPACE),
("fileSize", 64 + INDEXSIZE),
("userHeaderOffset", 40),
("userHeaderSize", 20 + INDEXSIZE),
("legacy1", 3),
("legacy2", 16),
("legacy3", BUNDLESIZE2),
("legacy4", 5),
("indexSize", INDEXSIZE)
])
def __init__(self, bundle, mode="read"):
self.mode = mode
self.filename = bundle + '.bundle'
self.filelock = FileLock(bundle + '.lck')
self.header = collections.OrderedDict(self.DEFAULT_BUNDLE_HEADER)
self.index = [4] * self.BUNDLESIZE2 # empty index
if not os.path.exists(self.filename):
self._init_bundle()
cache = self.BUNDLE_CACHE.get(self.filename, {})
if cache and cache['stat'][-4:] == os.stat(self.filename)[-4:]:
# log.debug("use BUNDLE_CACHE")
self.index = cache['index']
self.header = cache['header']
else:
self.__read_header()
self.__read_index()
self.BUNDLE_CACHE[self.filename] = {"stat": os.fstat(
self.filehandle.fileno()), "index": self.index, "header": self.header}
def _init_bundle(self):
log.info("Init Bundle %s" % self.filename)
ensure_directory(self.filename)
write_atomic(self.filename, struct.pack(*[self.BUNDLE_BYTEORDER + self.BUNDLE_HEADER_FORMAT + self.BUNDLE_INDEX_FORMAT] +
list(self.header.values()) +
self.index # empty index
))
def __read_header(self):
self.filehandle.seek(0)
header = struct.unpack(
self.BUNDLE_BYTEORDER + self.BUNDLE_HEADER_FORMAT, self.filehandle.read(64))
for value, key in zip(header, self.header.keys()):
self.header[key] = value
assert self.header["version"] is 3
assert self.header["fileSize"] == os.fstat(
self.filehandle.fileno()).st_size
def __read_index(self):
self.filehandle.seek(64)
self.index = list(struct.unpack(
self.BUNDLE_BYTEORDER + self.BUNDLE_INDEX_FORMAT, self.filehandle.read(self.header['indexSize'])))
def __tile_pos(self, col, row, level):
return (row % self.BUNDLESIZE) * self.BUNDLESIZE + col % self.BUNDLESIZE
def __get_tile_from_index(self, col, row, level):
tile_pos = self.__tile_pos(col, row, level)
index_value = self.index[tile_pos]
if index_value <= 4:
return 0, -1
size = index_value >> self.header['userHeaderOffset']
offset = index_value - (size << self.header['userHeaderOffset'])
return size, offset
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
return False
def tile_size(self, tile):
size, unused = self.__get_tile_from_index(*tile)
return size
def load_tile(self, tile):
size, offset = self.__get_tile_from_index(*tile.coord)
if size <= 0 or offset == -1:
return False
self.filehandle.seek(offset)
tile.source = ImageSource(BytesIO(self.filehandle.read(size)))
return True
def store_tile(self, data, tile):
size = len(data)
self.filehandle.seek(0, os.SEEK_END)
self.filehandle.write(struct.pack("<I", size))
self.index[self.__tile_pos(
*tile.coord)] = self.filehandle.tell() + (size << self.header['userHeaderOffset'])
self.filehandle.write(data)
self.header['fileSize'] = self.filehandle.tell()
self.header['maxRecordSize'] = max(self.header['maxRecordSize'], size)
self.filehandle.seek(0)
self.filehandle.write(struct.pack(
*[self.BUNDLE_BYTEORDER + self.BUNDLE_HEADER_FORMAT + self.BUNDLE_INDEX_FORMAT] + list(self.header.values()) + self.index))
return True
def remove_tile(self, tile):
self.index[self.__tile_pos(*tile.coord)] = 4
self.filehandle.seek(0)
self.filehandle.write(struct.pack(
*[self.BUNDLE_BYTEORDER + self.BUNDLE_HEADER_FORMAT + self.BUNDLE_INDEX_FORMAT] + list(self.header.values()) + self.index))
return True
def close(self):
if hasattr(self, '_%s__filehandle' % self.__class__.__name__) and not self.__filehandle.closed:
log.debug("close")
self.__filehandle.close()
if self.mode == "write":
self.BUNDLE_CACHE[self.filename] = {}
self.filelock.unlock()
@property
def filehandle(self):
if not hasattr(self, '_%s__filehandle' % self.__class__.__name__):
log.debug("open")
if self.mode == "write":
self.filelock.lock()
self.__filehandle = open(self.filename, mode="r+b")
else:
self.__filehandle = open(self.filename, mode="rb")
return self.__filehandle
def _create_lock(self):
lock = FileLock(self.lock_file)
lock.lock()
return lock
if (time.time() - mtime) > self.max_cache_time:
log.debug('removing cached tilelimit for %s %s', user_token,
name)
os.unlink(cache_file)
except EnvironmentError, ex:
if ex.errno != errno.ENOENT:
raise
try:
with open(cache_file, 'rb') as f:
return self.deserialize(f.read())
except EnvironmentError, ex:
if ex.errno != errno.ENOENT:
raise
with FileLock(cache_file + '.lck', remove_on_unlock=True):
if os.path.exists(cache_file):
# created while we were waiting for the lock
return self.load(user_token, name)
data = self.create(user_token, name)
self.cache(user_token, name, data)
return data
def create(self, user_token, name):
raise NotImplementedError
def serialize(self, date):
raise NotImplementedError
def deserialize(self, date):
def count_up():
with FileLock(self.lock_file, timeout=60):
with open(count_file, 'r+b') as f:
counter = int(f.read().strip())
f.seek(0)
f.write(str(counter + 1).encode('utf-8'))
def _create_lock(self):
lock = FileLock(self.lock_file)
lock.lock()
return lock
