def test_file_lock(self): # Test a lock that becomes free during a waiting lock() call. class Lock(threading.Thread): def __init__(self, lock_file): threading.Thread.__init__(self) self.lock_file = lock_file self.lock = FileLock(self.lock_file) def run(self): self.lock.lock() time.sleep(0.2) self.lock.unlock() lock_thread = Lock(self.lock_file) start_time = time.time() lock_thread.start() # wait until thread got the locked while not lock_thread.lock._locked: time.sleep(0.001) # one lock that times out assert_locked(self.lock_file) # one lock that will get it after some time l = FileLock(self.lock_file, timeout=0.3, step=0.001) l.lock() locked_for = time.time() - start_time assert locked_for - 0.2 <= 0.1, 'locking took to long?! (rerun if not sure)' #cleanup l.unlock() lock_thread.join()
def test_file_lock(self): # Test a lock that becomes free during a waiting lock() call. class Lock(threading.Thread): def __init__(self, lock_file): threading.Thread.__init__(self) self.lock_file = lock_file self.lock = FileLock(self.lock_file) def run(self): self.lock.lock() time.sleep(0.2) self.lock.unlock() lock_thread = Lock(self.lock_file) start_time = time.time() lock_thread.start() # wait until thread got the locked while not lock_thread.lock._locked: time.sleep(0.001) # one lock that times out assert_locked(self.lock_file) # one lock that will get it after some time l = FileLock(self.lock_file, timeout=0.3, step=0.001) l.lock() locked_for = time.time() - start_time assert locked_for - 0.2 <=0.1, 'locking took to long?! (rerun if not sure)' #cleanup l.unlock() lock_thread.join()
def lock(p=None): l = FileLock(lock_file, timeout=60) l.lock() counter = int(open(count_file).read()) open(count_file, 'w').write(str(counter+1)) time.sleep(0.001) l.unlock()
def assert_locked(lock_file, timeout=0.02, step=0.001): assert os.path.exists(lock_file) l = FileLock(lock_file, timeout=timeout, step=step) try: l.lock() assert False, 'file was not locked' except LockTimeout: pass
def lock(args): lock_file, count_file = args l = FileLock(lock_file.strpath, timeout=60) l.lock() counter = int(count_file.read()) count_file.write(str(counter+1)) time.sleep(0.001) l.unlock()
class Lock(threading.Thread): def __init__(self, lock_file): threading.Thread.__init__(self) self.lock_file = lock_file self.lock = FileLock(self.lock_file) def run(self): self.lock.lock() time.sleep(0.2) self.lock.unlock()
def test_remove_on_unlock(self): l = FileLock(self.lock_file, remove_on_unlock=True) l.lock() assert os.path.exists(self.lock_file) l.unlock() assert not os.path.exists(self.lock_file) l.lock() assert os.path.exists(self.lock_file) os.remove(self.lock_file) assert not os.path.exists(self.lock_file) # ignore removed lock l.unlock() assert not os.path.exists(self.lock_file)
def test_lock_cleanup(self): old_lock_file = os.path.join(self.lock_dir, 'lock_old.lck') l = FileLock(old_lock_file) l.lock() l.unlock() mtime = os.stat(old_lock_file).st_mtime mtime -= 7*60 os.utime(old_lock_file, (mtime, mtime)) l = self._create_lock() l.unlock() assert os.path.exists(old_lock_file) assert os.path.exists(self.lock_file) cleanup_lockdir(self.lock_dir) assert not os.path.exists(old_lock_file) assert os.path.exists(self.lock_file)
def test_lock_cleanup(self): old_lock_file = os.path.join(self.lock_dir, 'lock_old.lck') l = FileLock(old_lock_file) l.lock() l.unlock() mtime = os.stat(old_lock_file).st_mtime mtime -= 7 * 60 os.utime(old_lock_file, (mtime, mtime)) l = self._create_lock() l.unlock() assert os.path.exists(old_lock_file) assert os.path.exists(self.lock_file) cleanup_lockdir(self.lock_dir) assert not os.path.exists(old_lock_file) assert os.path.exists(self.lock_file)
def test_remove_on_unlock(self): l = FileLock(self.lock_file, remove_on_unlock=True) l.lock() assert os.path.exists(self.lock_file) l.unlock() if is_win: # not removed on windows assert os.path.exists(self.lock_file) else: assert not os.path.exists(self.lock_file) if is_win: # not possible to remove lock file when lock is held pass else: l.lock() assert os.path.exists(self.lock_file) os.remove(self.lock_file) assert not os.path.exists(self.lock_file) # ignore removed lock l.unlock() assert not os.path.exists(self.lock_file)
class BundleDataV2(object): BUNDLE_CACHE = {} BUNDLESIZE = 128 # Quadkey Base BUNDLESIZE2 = BUNDLESIZE**2 # Tiles in Bundle INDEXSIZE = BUNDLESIZE2 * 8 SLACKSPACE = 4 # Slack Space BUNDLE_BYTEORDER = "<" # little-endian BUNDLE_HEADER_FORMAT = "4I3Q6I" BUNDLE_INDEX_FORMAT = "%iQ" % BUNDLESIZE2 DEFAULT_BUNDLE_HEADER = collections.OrderedDict([ ("version", 3), ("numRecords", BUNDLESIZE2), ("maxRecordSize", 0), ("offsetSize", 5), ("slackSpace", SLACKSPACE), ("fileSize", 64 + INDEXSIZE), ("userHeaderOffset", 40), ("userHeaderSize", 20 + INDEXSIZE), ("legacy1", 3), ("legacy2", 16), ("legacy3", BUNDLESIZE2), ("legacy4", 5), ("indexSize", INDEXSIZE) ]) def __init__(self, bundle, mode="read"): self.mode = mode self.filename = bundle + '.bundle' self.filelock = FileLock(bundle + '.lck') self.header = collections.OrderedDict(self.DEFAULT_BUNDLE_HEADER) self.index = [4] * self.BUNDLESIZE2 # empty index if not os.path.exists(self.filename): self._init_bundle() cache = self.BUNDLE_CACHE.get(self.filename, {}) if cache and cache['stat'][-4:] == os.stat(self.filename)[-4:]: # log.debug("use BUNDLE_CACHE") self.index = cache['index'] self.header = cache['header'] else: self.__read_header() self.__read_index() self.BUNDLE_CACHE[self.filename] = {"stat": os.fstat( self.filehandle.fileno()), "index": self.index, "header": self.header} def _init_bundle(self): log.info("Init Bundle %s" % self.filename) ensure_directory(self.filename) write_atomic(self.filename, struct.pack(*[self.BUNDLE_BYTEORDER + self.BUNDLE_HEADER_FORMAT + self.BUNDLE_INDEX_FORMAT] + list(self.header.values()) + self.index # empty index )) def __read_header(self): self.filehandle.seek(0) header = struct.unpack( self.BUNDLE_BYTEORDER + self.BUNDLE_HEADER_FORMAT, self.filehandle.read(64)) for value, key in zip(header, self.header.keys()): self.header[key] = value assert self.header["version"] is 3 assert self.header["fileSize"] == os.fstat( self.filehandle.fileno()).st_size def __read_index(self): self.filehandle.seek(64) self.index = list(struct.unpack( self.BUNDLE_BYTEORDER + self.BUNDLE_INDEX_FORMAT, self.filehandle.read(self.header['indexSize']))) def __tile_pos(self, col, row, level): return (row % self.BUNDLESIZE) * self.BUNDLESIZE + col % self.BUNDLESIZE def __get_tile_from_index(self, col, row, level): tile_pos = self.__tile_pos(col, row, level) index_value = self.index[tile_pos] if index_value <= 4: return 0, -1 size = index_value >> self.header['userHeaderOffset'] offset = index_value - (size << self.header['userHeaderOffset']) return size, offset def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() return False def tile_size(self, tile): size, unused = self.__get_tile_from_index(*tile) return size def load_tile(self, tile): size, offset = self.__get_tile_from_index(*tile.coord) if size <= 0 or offset == -1: return False self.filehandle.seek(offset) tile.source = ImageSource(BytesIO(self.filehandle.read(size))) return True def store_tile(self, data, tile): size = len(data) self.filehandle.seek(0, os.SEEK_END) self.filehandle.write(struct.pack("<I", size)) self.index[self.__tile_pos( *tile.coord)] = self.filehandle.tell() + (size << self.header['userHeaderOffset']) self.filehandle.write(data) self.header['fileSize'] = self.filehandle.tell() self.header['maxRecordSize'] = max(self.header['maxRecordSize'], size) self.filehandle.seek(0) self.filehandle.write(struct.pack( *[self.BUNDLE_BYTEORDER + self.BUNDLE_HEADER_FORMAT + self.BUNDLE_INDEX_FORMAT] + list(self.header.values()) + self.index)) return True def remove_tile(self, tile): self.index[self.__tile_pos(*tile.coord)] = 4 self.filehandle.seek(0) self.filehandle.write(struct.pack( *[self.BUNDLE_BYTEORDER + self.BUNDLE_HEADER_FORMAT + self.BUNDLE_INDEX_FORMAT] + list(self.header.values()) + self.index)) return True def close(self): if hasattr(self, '_%s__filehandle' % self.__class__.__name__) and not self.__filehandle.closed: log.debug("close") self.__filehandle.close() if self.mode == "write": self.BUNDLE_CACHE[self.filename] = {} self.filelock.unlock() @property def filehandle(self): if not hasattr(self, '_%s__filehandle' % self.__class__.__name__): log.debug("open") if self.mode == "write": self.filelock.lock() self.__filehandle = open(self.filename, mode="r+b") else: self.__filehandle = open(self.filename, mode="rb") return self.__filehandle
def _create_lock(self): lock = FileLock(self.lock_file) lock.lock() return lock