def lock(p=None):
l = FileLock(lock_file, timeout=60)
l.lock()
counter = int(open(count_file).read())
open(count_file, 'w').write(str(counter+1))
time.sleep(0.001)
l.unlock()
def ensure_mbtile(self):
if not os.path.exists(self.mbtile_file):
with FileLock(self.mbtile_file + '.init.lck',
remove_on_unlock=REMOVE_ON_UNLOCK):
if not os.path.exists(self.mbtile_file):
ensure_directory(self.mbtile_file)
self._initialize_mbtile()
def config(self, user, uuid=None, max_uuids=50):
conf_filename = os.path.join(self.cache_dir, (uuid or user) + '.yaml')
if self._requires_reconf(conf_filename):
with FileLock(conf_filename + '.lck'):
if self._requires_reconf(conf_filename):
log.debug('(re)configure %s for %s', (uuid or "all"), user)
if uuid:
params = tile_params(user, uuid, self.cartodb_domain)
if params is None:
return
conf = mapproxy_config([params], user=user)
else:
layers = []
try:
for uuid in user_uuids(user, max_uuids=max_uuids, cartodb_domain=self.cartodb_domain):
try:
params = tile_params(user, uuid, self.cartodb_domain)
if params:
layers.append(params)
else:
log.warn("found no layer for %s %s", user, uuid)
except RequestError as ex:
log.warn("faild to query tiler for %s %s: %s", user, uuid, ex)
except RequestError as ex:
log.warn("failed to get user uuids for user %s: %s", user, ex)
if not layers:
return
conf = mapproxy_config(layers, user=user)
write_mapproxy_config(conf, conf_filename)
return conf_filename
def test_file_lock(self):
# Test a lock that becomes free during a waiting lock() call.
class Lock(threading.Thread):
def __init__(self, lock_file):
threading.Thread.__init__(self)
self.lock_file = lock_file
self.lock = FileLock(self.lock_file)
def run(self):
self.lock.lock()
time.sleep(0.2)
self.lock.unlock()
lock_thread = Lock(self.lock_file)
start_time = time.time()
lock_thread.start()
# wait until thread got the locked
while not lock_thread.lock._locked:
time.sleep(0.001)
# one lock that times out
assert_locked(self.lock_file)
# one lock that will get it after some time
l = FileLock(self.lock_file, timeout=0.3, step=0.001)
l.lock()
locked_for = time.time() - start_time
assert locked_for - 0.2 <= 0.1, 'locking took to long?! (rerun if not sure)'
#cleanup
l.unlock()
lock_thread.join()
def lock(args):
lock_file, count_file = args
l = FileLock(lock_file.strpath, timeout=60)
l.lock()
counter = int(count_file.read())
count_file.write(str(counter+1))
time.sleep(0.001)
l.unlock()
def assert_locked(lock_file, timeout=0.02, step=0.001):
assert os.path.exists(lock_file)
l = FileLock(lock_file, timeout=timeout, step=step)
try:
l.lock()
assert False, 'file was not locked'
except LockTimeout:
pass
def lock(self, tile):
"""
Returns a lock object for this tile.
"""
lock_filename = self.lock_filename(tile)
cleanup_lockdir(self.lock_dir, force=False)
return FileLock(lock_filename, timeout=self.lock_timeout,
remove_on_unlock=True)
def ensure_mbtile(self):
if not os.path.exists(self.mbtile_file):
with FileLock(os.path.join(self.lock_dir, 'init.lck'),
timeout=self.lock_timeout,
remove_on_unlock=True):
if not os.path.exists(self.mbtile_file):
ensure_directory(self.mbtile_file)
self._initialize_mbtile()
def ensure_mbtile(self):
if not os.path.exists(self.mbtile_file):
with FileLock(os.path.join(os.path.dirname(self.mbtile_file),
'init.lck'),
remove_on_unlock=True):
if not os.path.exists(self.mbtile_file):
ensure_directory(self.mbtile_file)
self._initialize_mbtile()
def ensure_gpkg(self):
if not os.path.isfile(self.geopackage_file):
with FileLock(self.geopackage_file + '.init.lck',
remove_on_unlock=REMOVE_ON_UNLOCK):
ensure_directory(self.geopackage_file)
self._initialize_gpkg()
else:
if not self.check_gpkg():
ensure_directory(self.geopackage_file)
self._initialize_gpkg()
def remove_tile(self, tile):
if tile.coord is None:
return True
with FileLock(self.lock_filename):
with self.index().readwrite() as idx:
x, y = self._rel_tile_coord(tile.coord)
idx.remove_tile_offset(x, y)
return True
def remove_tile(self, tile):
if tile.coord is None:
return True
with FileLock(self.lock_filename):
idx = BundleIndex(self.base_filename + BUNDLEX_EXT)
x, y = self._rel_tile_coord(tile.coord)
idx.remove_tile_offset(x, y)
return True
def lock(self, tile):
"""
Returns a lock object for this tile.
"""
if getattr(self, 'locking_disabled', False):
return DummyLock()
lock_filename = self.lock_filename(tile)
cleanup_lockdir(self.lock_dir, force=False)
return FileLock(lock_filename, timeout=self.lock_timeout,
remove_on_unlock=True)
def remove_tile(self, tile):
if tile.coord is None:
return True
self._init_index()
with FileLock(self.lock_filename):
with self._readwrite() as fh:
x, y = self._rel_tile_coord(tile.coord)
self._update_tile_offset(fh, x, y, 0, 0)
return True
def test_remove_on_unlock(self):
l = FileLock(self.lock_file, remove_on_unlock=True)
l.lock()
assert os.path.exists(self.lock_file)
l.unlock()
assert not os.path.exists(self.lock_file)
l.lock()
assert os.path.exists(self.lock_file)
os.remove(self.lock_file)
assert not os.path.exists(self.lock_file)
# ignore removed lock
l.unlock()
assert not os.path.exists(self.lock_file)
def to_csv(csv_config_file, cap_type, cap_url, layer_name, system_id, dimensions=None):
dimensions = serialize_dimensions(dimensions)
id = urlparse(cap_url).netloc + '_' + layer_name + '_' + system_id
if dimensions:
id += '_' + dimensions
id = re.sub('[^A-Za-z0-9-_]', '_', id)
with FileLock(csv_config_file + '.lck'):
records = read_csv(csv_config_file)
records[id] = (cap_type, cap_url, layer_name, system_id, dimensions)
write_csv(csv_config_file, records)
return id
def store_tile(self, tile):
if tile.stored:
return True
with tile_buffer(tile) as buf:
data = buf.read()
with FileLock(self.lock_filename):
bundle = BundleData(self.base_filename + BUNDLE_EXT, self.offset)
idx = BundleIndex(self.base_filename + BUNDLEX_EXT)
x, y = self._rel_tile_coord(tile.coord)
offset = idx.tile_offset(x, y)
offset, size = bundle.append_tile(data, prev_offset=offset)
idx.update_tile_offset(x, y, offset=offset, size=size)
return True
def store_tiles(self, tiles):
self._init_index()
tiles_data = []
for t in tiles:
if t.stored:
continue
with tile_buffer(t) as buf:
data = buf.read()
tiles_data.append((t.coord, data))
with FileLock(self.lock_filename):
with self._readwrite() as fh:
for tile_coord, data in tiles_data:
self._store_tile(fh, tile_coord, data)
return True
def test_lock_cleanup(self):
old_lock_file = os.path.join(self.lock_dir, 'lock_old.lck')
l = FileLock(old_lock_file)
l.lock()
l.unlock()
mtime = os.stat(old_lock_file).st_mtime
mtime -= 7 * 60
os.utime(old_lock_file, (mtime, mtime))
l = self._create_lock()
l.unlock()
assert os.path.exists(old_lock_file)
assert os.path.exists(self.lock_file)
cleanup_lockdir(self.lock_dir)
assert not os.path.exists(old_lock_file)
assert os.path.exists(self.lock_file)
def store_tiles(self, tiles):
tiles_data = []
for t in tiles:
if t.stored:
continue
with tile_buffer(t) as buf:
data = buf.read()
tiles_data.append((t.coord, data))
with FileLock(self.lock_filename):
with self.data().readwrite() as bundle:
with self.index().readwrite() as idx:
for tile_coord, data in tiles_data:
x, y = self._rel_tile_coord(tile_coord)
offset = idx.tile_offset(x, y)
offset, size = bundle.append_tile(data,
prev_offset=offset)
idx.update_tile_offset(x, y, offset=offset, size=size)
return True
def test_remove_on_unlock(self):
l = FileLock(self.lock_file, remove_on_unlock=True)
l.lock()
assert os.path.exists(self.lock_file)
l.unlock()
if is_win: # not removed on windows
assert os.path.exists(self.lock_file)
else:
assert not os.path.exists(self.lock_file)
if is_win:
# not possible to remove lock file when lock is held
pass
else:
l.lock()
assert os.path.exists(self.lock_file)
os.remove(self.lock_file)
assert not os.path.exists(self.lock_file)
# ignore removed lock
l.unlock()
assert not os.path.exists(self.lock_file)
def __init__(self, lock_file):
threading.Thread.__init__(self)
self.lock_file = lock_file
self.lock = FileLock(self.lock_file)
if (time.time() - mtime) > self.max_cache_time:
log.debug('removing cached tilelimit for %s %s', user_token,
name)
os.unlink(cache_file)
except EnvironmentError, ex:
if ex.errno != errno.ENOENT:
raise
try:
with open(cache_file, 'rb') as f:
return self.deserialize(f.read())
except EnvironmentError, ex:
if ex.errno != errno.ENOENT:
raise
with FileLock(cache_file + '.lck', remove_on_unlock=True):
if os.path.exists(cache_file):
# created while we were waiting for the lock
return self.load(user_token, name)
data = self.create(user_token, name)
self.cache(user_token, name, data)
return data
def create(self, user_token, name):
raise NotImplementedError
def serialize(self, date):
raise NotImplementedError
def deserialize(self, date):
def _create_lock(self):
lock = FileLock(self.lock_file)
lock.lock()
return lock
def count_up():
with FileLock(self.lock_file, timeout=60):
with open(count_file, 'r+b') as f:
counter = int(f.read().strip())
f.seek(0)
f.write(str(counter + 1).encode('utf-8'))
