def test_double_reader_writer(self): lock = fasteners.ReaderWriterLock() activated = collections.deque() active = threading.Event() def double_reader(): with lock.read_lock(): active.set() while not lock.has_pending_writers: time.sleep(0.001) with lock.read_lock(): activated.append(lock.owner) def happy_writer(): with lock.write_lock(): activated.append(lock.owner) reader = _daemon_thread(double_reader) reader.start() active.wait(WAIT_TIMEOUT) self.assertTrue(active.is_set()) writer = _daemon_thread(happy_writer) writer.start() reader.join() writer.join() self.assertEqual(2, len(activated)) self.assertEqual(['r', 'w'], list(activated))
def test_no_double_writers(self): lock = fasteners.ReaderWriterLock() watch = _utils.StopWatch(duration=5) watch.start() dups = collections.deque() active = collections.deque() def acquire_check(me): with lock.write_lock(): if len(active) >= 1: dups.append(me) dups.extend(active) active.append(me) try: time.sleep(random.random() / 100) finally: active.remove(me) def run(): me = threading.current_thread() while not watch.expired(): acquire_check(me) threads = [] for i in range(0, self.THREAD_COUNT): t = _daemon_thread(run) threads.append(t) t.start() while threads: t = threads.pop() t.join() self.assertEqual([], list(dups)) self.assertEqual([], list(active))
def __init__(self, tickerid='MARKET:SYMBOL'): super().__init__() self.tickerid = tickerid self.candles = {} self.lock = fasteners.ReaderWriterLock() self.queues = {} self.start_threads()
def test_writer_chaotic(self): lock = fasteners.ReaderWriterLock() activated = collections.deque() def chaotic_writer(blow_up): with lock.write_lock(): if blow_up: raise RuntimeError("Broken") else: activated.append(lock.owner) def happy_reader(): with lock.read_lock(): activated.append(lock.owner) with futures.ThreadPoolExecutor(max_workers=20) as e: for i in range(0, 20): if i % 2 == 0: e.submit(chaotic_writer, blow_up=bool(i % 4 == 0)) else: e.submit(happy_reader) writers = [a for a in activated if a == 'w'] readers = [a for a in activated if a == 'r'] self.assertEqual(5, len(writers)) self.assertEqual(10, len(readers))
def test_writer_reader_writer(self): lock = fasteners.ReaderWriterLock() with lock.write_lock(): self.assertTrue(lock.is_writer()) with lock.read_lock(): self.assertTrue(lock.is_reader()) with lock.write_lock(): self.assertTrue(lock.is_writer())
def test_writer_reader_writer(): lock = fasteners.ReaderWriterLock() with lock.write_lock(): assert lock.is_writer() with lock.read_lock(): assert lock.is_reader() with lock.write_lock(): assert lock.is_writer()
def save(self): lock = fasteners.ReaderWriterLock() while True: try: with lock.write_lock(): torch.save(deepcopy(self).cpu().state_dict(), self.path) return except: sleep(np.random.random() + 1)
def load(self): lock = fasteners.ReaderWriterLock() while True: try: with lock.read_lock(): state_dict = torch.load(self.path) self.load_state_dict(state_dict) return except: sleep(np.random.random() + 1)
def test_reader_abort(): lock = fasteners.ReaderWriterLock() assert lock.owner is None with pytest.raises(RuntimeError): with lock.read_lock(): assert lock.owner == lock.READER raise RuntimeError("Broken") assert lock.owner is None
def test_double_reader(self): lock = fasteners.ReaderWriterLock() with lock.read_lock(): self.assertTrue(lock.is_reader()) self.assertFalse(lock.is_writer()) with lock.read_lock(): self.assertTrue(lock.is_reader()) self.assertTrue(lock.is_reader()) self.assertFalse(lock.is_reader()) self.assertFalse(lock.is_writer())
def test_reader_abort(self): lock = fasteners.ReaderWriterLock() self.assertFalse(lock.owner) def blow_up(): with lock.read_lock(): self.assertEqual(lock.READER, lock.owner) raise RuntimeError("Broken") self.assertRaises(RuntimeError, blow_up) self.assertFalse(lock.owner)
def test_reader_to_writer(): lock = fasteners.ReaderWriterLock() with lock.read_lock(): with pytest.raises(RuntimeError): with lock.write_lock(): pass assert lock.is_reader() assert not lock.is_writer() assert not lock.is_reader() assert not lock.is_writer()
def test_reader_to_writer(self): lock = fasteners.ReaderWriterLock() def writer_func(): with lock.write_lock(): pass with lock.read_lock(): self.assertRaises(RuntimeError, writer_func) self.assertFalse(lock.is_writer()) self.assertFalse(lock.is_reader()) self.assertFalse(lock.is_writer())
def test_single_reader_writer(): results = [] lock = fasteners.ReaderWriterLock() with lock.read_lock(): assert lock.is_reader() assert not results with lock.write_lock(): results.append(1) assert lock.is_writer() with lock.read_lock(): assert lock.is_reader() assert len(results) == 1 assert not lock.is_reader() assert not lock.is_writer()
def test_single_reader_writer(self): results = [] lock = fasteners.ReaderWriterLock() with lock.read_lock(): self.assertTrue(lock.is_reader()) self.assertEqual(0, len(results)) with lock.write_lock(): results.append(1) self.assertTrue(lock.is_writer()) with lock.read_lock(): self.assertTrue(lock.is_reader()) self.assertEqual(1, len(results)) self.assertFalse(lock.is_reader()) self.assertFalse(lock.is_writer())
def test_writer_to_reader(self): lock = fasteners.ReaderWriterLock() def reader_func(): with lock.read_lock(): self.assertTrue(lock.is_writer()) self.assertTrue(lock.is_reader()) with lock.write_lock(): self.assertIsNone(reader_func()) self.assertFalse(lock.is_reader()) self.assertFalse(lock.is_reader()) self.assertFalse(lock.is_writer())
def test_double_reader(): lock = fasteners.ReaderWriterLock() with lock.read_lock(): assert lock.is_reader() assert not lock.is_writer() with lock.read_lock(): assert lock.is_reader() assert lock.is_reader() assert not lock.is_reader() assert not lock.is_writer()
def __init__(self, conf): super(DirBackend, self).__init__(conf) max_cache_size = self._conf.get('max_cache_size') if max_cache_size is not None: max_cache_size = int(max_cache_size) if max_cache_size < 1: raise ValueError("Maximum cache size must be greater than" " or equal to one") self.file_cache = cachetools.LRUCache(max_cache_size) else: self.file_cache = {} self.encoding = self._conf.get('encoding', self.DEFAULT_FILE_ENCODING) if not self._path: raise ValueError("Empty path is disallowed") self._path = os.path.abspath(self._path) self.lock = fasteners.ReaderWriterLock()
def __init__(self, flow_detail, backend=None, scope_fetcher=None): self._result_mappings = {} self._reverse_mapping = {} if backend is None: # Err on the likely-hood that most people don't make there # objects able to be deepcopyable (resources, locks and such # can't be deepcopied)... backend = impl_memory.MemoryBackend({'deep_copy': False}) with contextlib.closing(backend.get_connection()) as conn: conn.update_flow_details(flow_detail, ignore_missing=True) self._backend = backend self._flowdetail = flow_detail self._transients = {} self._injected_args = {} self._lock = fasteners.ReaderWriterLock() self._ensure_matchers = [ ((task.BaseTask, ), (models.TaskDetail, 'Task')), ((retry.Retry, ), (models.RetryDetail, 'Retry')), ] if scope_fetcher is None: scope_fetcher = lambda atom_name: None self._scope_fetcher = scope_fetcher # NOTE(imelnikov): failure serialization looses information, # so we cache failures here, in atom name -> failure mapping. self._failures = {} for ad in self._flowdetail: fail_cache = {} if ad.failure is not None: fail_cache[states.EXECUTE] = ad.failure if ad.revert_failure is not None: fail_cache[states.REVERT] = ad.revert_failure self._failures[ad.name] = fail_cache self._atom_name_to_uuid = dict( (ad.name, ad.uuid) for ad in self._flowdetail) try: source, _clone = self._atomdetail_by_name( self.injector_name, expected_type=models.TaskDetail) except exceptions.NotFound: pass else: names_iter = six.iterkeys(source.results) self._set_result_mapping(source.name, dict((name, name) for name in names_iter))
def Store(self, obj, envKey): """Stores authorization token passed as an argument. param obj instance to be stored. """ DefaultStorageStrategy.cache_path = os.path.join( Configuration.TempPath, "cached-data." + envKey + ".py") if obj == None: return lock = fasteners.ReaderWriterLock() with lock.write_lock(): fp = open(DefaultStorageStrategy.cache_path, 'w') os.chmod(DefaultStorageStrategy.cache_path, stat.S_IRUSR | stat.S_IWUSR) # Write it to the result to the file as a json serializedObj = "#" + json.dumps(obj.__dict__) # add hash to prevent download token file via http when path is invalid fp.write(serializedObj) fp.close()
def Get(self, envKey): """Gets the currently stored objects as dictionary. return stored Token dictionary or null. """ DefaultStorageStrategy.cache_path = os.path.join( Configuration.TempPath, "cached-data." + envKey + ".py") if not os.path.exists(DefaultStorageStrategy.cache_path): return None lock = fasteners.ReaderWriterLock() with lock.read_lock(): fp = open(DefaultStorageStrategy.cache_path, 'rb') serializedObj = fp.read().decode('UTF-8') try: cached = json.loads(serializedObj[1:]) except: cached = None fp.close() return OAuthToken(cached)
def _spawn_variation(readers, writers, max_workers=None): start_stops = collections.deque() lock = fasteners.ReaderWriterLock() def read_func(ident): with lock.read_lock(): # TODO(harlowja): sometime in the future use a monotonic clock here # to avoid problems that can be caused by ntpd resyncing the clock # while we are actively running. enter_time = _utils.now() time.sleep(WORK_TIMES[ident % len(WORK_TIMES)]) exit_time = _utils.now() start_stops.append((lock.READER, enter_time, exit_time)) time.sleep(NAPPY_TIME) def write_func(ident): with lock.write_lock(): enter_time = _utils.now() time.sleep(WORK_TIMES[ident % len(WORK_TIMES)]) exit_time = _utils.now() start_stops.append((lock.WRITER, enter_time, exit_time)) time.sleep(NAPPY_TIME) if max_workers is None: max_workers = max(0, readers) + max(0, writers) if max_workers > 0: with futures.ThreadPoolExecutor(max_workers=max_workers) as e: count = 0 for _i in range(0, readers): e.submit(read_func, count) count += 1 for _i in range(0, writers): e.submit(write_func, count) count += 1 writer_times = [] reader_times = [] for (lock_type, start, stop) in list(start_stops): if lock_type == lock.WRITER: writer_times.append((start, stop)) else: reader_times.append((start, stop)) return (writer_times, reader_times)
def test_no_concurrent_readers_writers(self): lock = fasteners.ReaderWriterLock() watch = _utils.StopWatch(duration=5) watch.start() dups = collections.deque() active = collections.deque() def acquire_check(me, reader): if reader: lock_func = lock.read_lock else: lock_func = lock.write_lock with lock_func(): if not reader: # There should be no-one else currently active, if there # is ensure we capture them so that we can later blow-up # the test. if len(active) >= 1: dups.append(me) dups.extend(active) active.append(me) try: time.sleep(random.random() / 100) finally: active.remove(me) def run(): me = threading.current_thread() while not watch.expired(): acquire_check(me, random.choice([True, False])) threads = [] for i in range(0, self.THREAD_COUNT): t = _daemon_thread(run) threads.append(t) t.start() while threads: t = threads.pop() t.join() self.assertEqual([], list(dups)) self.assertEqual([], list(active))
def test_double_reader_abort(self): lock = fasteners.ReaderWriterLock() activated = collections.deque() def double_bad_reader(): with lock.read_lock(): with lock.read_lock(): raise RuntimeError("Broken") def happy_writer(): with lock.write_lock(): activated.append(lock.owner) with futures.ThreadPoolExecutor(max_workers=20) as e: for i in range(0, 20): if i % 2 == 0: e.submit(double_bad_reader) else: e.submit(happy_writer) self.assertEqual(10, len([a for a in activated if a == 'w']))
def _spawn_variation(readers, writers, max_workers=None): start_stops = collections.deque() lock = fasteners.ReaderWriterLock() def read_func(ident): with lock.read_lock(): enter_time = time.monotonic() time.sleep(WORK_TIMES[ident % len(WORK_TIMES)]) exit_time = time.monotonic() start_stops.append((lock.READER, enter_time, exit_time)) time.sleep(NAPPY_TIME) def write_func(ident): with lock.write_lock(): enter_time = time.monotonic() time.sleep(WORK_TIMES[ident % len(WORK_TIMES)]) exit_time = time.monotonic() start_stops.append((lock.WRITER, enter_time, exit_time)) time.sleep(NAPPY_TIME) if max_workers is None: max_workers = max(0, readers) + max(0, writers) if max_workers > 0: with futures.ThreadPoolExecutor(max_workers=max_workers) as e: count = 0 for _i in range(0, readers): e.submit(read_func, count) count += 1 for _i in range(0, writers): e.submit(write_func, count) count += 1 writer_times = [] reader_times = [] for (lock_type, start, stop) in list(start_stops): if lock_type == lock.WRITER: writer_times.append((start, stop)) else: reader_times.append((start, stop)) return writer_times, reader_times
def test_reader_chaotic(): lock = fasteners.ReaderWriterLock() activated = collections.deque() def chaotic_reader(blow_up): with lock.read_lock(): if blow_up: raise RuntimeError("Broken") else: activated.append(lock.owner) def happy_writer(): with lock.write_lock(): activated.append(lock.owner) with futures.ThreadPoolExecutor(max_workers=20) as e: for i in range(0, 20): if i % 2 == 0: e.submit(chaotic_reader, blow_up=bool(i % 4 == 0)) else: e.submit(happy_writer) assert sum(a == 'w' for a in activated) == 10 assert sum(a == 'r' for a in activated) == 5
def __init__(self, conf=None): super(MemoryBackend, self).__init__(conf) self.memory = FakeFilesystem( deep_copy=self._conf.get('deep_copy', True)) self.lock = fasteners.ReaderWriterLock()
import fasteners def read_something(ident, rw_lock): with rw_lock.read_lock(): print("Thread %s is reading something" % ident) time.sleep(1) def write_something(ident, rw_lock): with rw_lock.write_lock(): print("Thread %s is writing something" % ident) time.sleep(2) rw_lock = fasteners.ReaderWriterLock() threads = [] for i in range(0, 10): is_writer = random.choice([True, False]) if is_writer: threads.append( threading.Thread(target=write_something, args=(i, rw_lock))) else: threads.append( threading.Thread(target=read_something, args=(i, rw_lock))) try: for t in threads: t.start() finally: while threads:
def __init__(self): self._lock = fasteners.ReaderWriterLock()
class Project(Versioned): """ Represent the current Meltano project from a file-system perspective. """ __version__ = 1 _activate_lock = threading.Lock() _find_lock = threading.Lock() _meltano_rw_lock = fasteners.ReaderWriterLock() _default = None def __init__(self, root: Union[Path, str] = None): self.root = Path(root or os.getcwd()).resolve() self._meltano_ip_lock = fasteners.InterProcessLock( self.run_dir("meltano.yml.lock")) @classmethod @fasteners.locked(lock="_activate_lock") def activate(cls, project: "Project"): project.ensure_compatible() # helpful to refer to the current absolute project path os.environ["MELTANO_PROJECT_ROOT"] = str(project.root) # create a symlink to our current binary try: executable = Path(os.path.dirname(sys.executable), "meltano") if executable.is_file(): project.run_dir().joinpath("bin").symlink_to(executable) except FileExistsError: pass load_dotenv(dotenv_path=project.root.joinpath(".env")) logging.debug(f"Activated project at {project.root}") # set the default project cls._default = project @property def backend_version(self): return int(self.meltano.get("version", 1)) @classmethod @fasteners.locked(lock="_find_lock") def find(cls, from_dir: Union[Path, str] = None, activate=True): if cls._default: return cls._default project = Project(from_dir) if not project.meltanofile.exists(): raise ProjectNotFound() # if we activate a project using `find()`, it should # be set as the default project for future `find()` if activate: cls.activate(project) return project @property def meltano(self) -> Dict: """Return a copy of the current meltano config""" # fmt: off with self._meltano_rw_lock.read_lock(), \ self.meltanofile.open() as meltanofile: return yaml.safe_load(meltanofile) # fmt: on @contextmanager def meltano_update(self): """ Yield the current meltano configuration and update the meltanofile if the context ends gracefully. """ # fmt: off with self._meltano_rw_lock.write_lock(), \ self._meltano_ip_lock: with self.meltanofile.open() as meltanofile: # read the latest version meltano_update = yaml.safe_load(meltanofile) yield meltano_update try: with atomic_write(self.meltanofile, overwrite=True) as meltanofile: # update if everything is fine yaml.dump(meltano_update, meltanofile, default_flow_style=False) except Exception as err: logging.critical(f"Could not update meltano.yml: {err}") raise # fmt: on def root_dir(self, *joinpaths): return self.root.joinpath(*joinpaths) @property def meltanofile(self): return self.root.joinpath("meltano.yml") @makedirs def meltano_dir(self, *joinpaths): return self.root.joinpath(".meltano", *joinpaths) @makedirs def analyze_dir(self, *joinpaths): return self.root_dir("analyze", *joinpaths) @makedirs def venvs_dir(self, *prefixes): return self.meltano_dir(*prefixes, "venv") @makedirs def run_dir(self, *joinpaths): return self.meltano_dir("run", *joinpaths) @makedirs def job_dir(self, job_id, *joinpaths): return self.run_dir("elt", slugify(job_id), *joinpaths) @makedirs def model_dir(self, *joinpaths): return self.meltano_dir("models", *joinpaths) @makedirs def plugin_dir(self, plugin: "PluginRef", *joinpaths): return self.meltano_dir(plugin.type, plugin.name, *joinpaths) def __eq__(self, other): return self.root == other.root def __hash__(self): return self.root.__hash__()