def __init__(self): self.out_of_memory_restart = False self.total_locker = Lock() self.total_files_requested = 0 self.total_tuids_mapped = 0 self.threads_locker = Lock() self.waiting = 0 self.threads_waiting = 0 self.requests_locker = Lock() self.requests_total = 0 self.requests_complete = 0 self.requests_incomplete = 0 self.requests_passed = 0 self.requests_failed = 0 self.prev_mem = 0 self.curr_mem = 0 self.initial_growth = {} Thread.run("pc-daemon", self.run_pc_daemon) Thread.run("threads-daemon", self.run_threads_daemon) Thread.run("memory-daemon", self.run_memory_daemon) Thread.run("requests-daemon", self.run_requests_daemon)
def __init__(self, conn=None, tuid_service=None, kwargs=None): try: self.config = kwargs self.conn = conn if conn else sql.Sql(self.config.database.name) self.hg_cache = HgMozillaOrg( kwargs=self.config.hg_cache, use_cache=True) if self.config.hg_cache else Null self.tuid_service = tuid_service if tuid_service else tuid.service.TUIDService( database=None, hg=None, kwargs=self.config, conn=self.conn, clogger=self) self.rev_locker = Lock() self.working_locker = Lock() self.init_db() self.next_revnum = coalesce( self.conn.get_one("SELECT max(revnum)+1 FROM csetLog")[0], 1) self.csets_todo_backwards = Queue( name="Clogger.csets_todo_backwards") self.deletions_todo = Queue(name="Clogger.deletions_todo") self.maintenance_signal = Signal(name="Clogger.maintenance_signal") self.config = self.config.tuid self.disable_backfilling = False self.disable_tipfilling = False self.disable_deletion = False self.disable_maintenance = False # Make sure we are filled before allowing queries numrevs = self.conn.get_one("SELECT count(revnum) FROM csetLog")[0] if numrevs < MINIMUM_PERMANENT_CSETS: Log.note("Filling in csets to hold {{minim}} csets.", minim=MINIMUM_PERMANENT_CSETS) oldest_rev = 'tip' with self.conn.transaction() as t: tmp = t.query( "SELECT min(revnum), revision FROM csetLog").data[0][1] if tmp: oldest_rev = tmp self._fill_in_range(MINIMUM_PERMANENT_CSETS - numrevs, oldest_rev, timestamp=False) Log.note( "Table is filled with atleast {{minim}} entries. Starting workers...", minim=MINIMUM_PERMANENT_CSETS) Thread.run('clogger-tip', self.fill_forward_continuous) Thread.run('clogger-backfill', self.fill_backward_with_list) Thread.run('clogger-maintenance', self.csetLog_maintenance) Thread.run('clogger-deleter', self.csetLog_deleter) Log.note("Started clogger workers.") except Exception as e: Log.warning("Cannot setup clogger: {{cause}}", cause=str(e))
def __init__(self, name): self.name = name self.lock = Lock("rate locker") self.request_rate = 0.0 self.last_request = Date.now() Thread.run("rate logger", self._daemon)
def __init__(self, rollover_field, rollover_interval, rollover_max, queue_size=10000, batch_size=5000, kwargs=None): """ :param rollover_field: the FIELD with a timestamp to use for determining which index to push to :param rollover_interval: duration between roll-over to new index :param rollover_max: remove old indexes, do not add old records :param queue_size: number of documents to queue in memory :param batch_size: number of documents to push at once :param kwargs: plus additional ES settings :return: """ self.settings = kwargs self.locker = Lock("lock for rollover_index") self.rollover_field = jx.get(rollover_field) self.rollover_interval = self.settings.rollover_interval = Duration( kwargs.rollover_interval) self.rollover_max = self.settings.rollover_max = Duration( kwargs.rollover_max) self.known_queues = {} # MAP DATE TO INDEX self.cluster = elasticsearch.Cluster(self.settings)
def __init__(self, rate=None, amortization_period=None, source=None, database=None, kwargs=None): self.amortization_period = coalesce(amortization_period, AMORTIZATION_PERIOD) self.rate = coalesce(rate, HG_REQUEST_PER_SECOND) self.cache_locker = Lock() self.cache = {} # MAP FROM url TO (ready, headers, response, timestamp) PAIR self.no_cache = {} # VERY SHORT TERM CACHE self.workers = [] self.todo = Queue(APP_NAME+" todo") self.requests = Queue(APP_NAME + " requests", max=int(self.rate * self.amortization_period.seconds)) self.url = URL(source.url) self.db = Sqlite(database) self.inbound_rate = RateLogger("Inbound") self.outbound_rate = RateLogger("hg.mo") if not self.db.query("SELECT name FROM sqlite_master WHERE type='table'").data: with self.db.transaction() as t: t.execute( "CREATE TABLE cache (" " path TEXT PRIMARY KEY, " " headers TEXT, " " response TEXT, " " timestamp REAL " ")" ) self.threads = [ Thread.run(APP_NAME+" worker" + text_type(i), self._worker) for i in range(CONCURRENCY) ] self.limiter = Thread.run(APP_NAME+" limiter", self._rate_limiter) self.cleaner = Thread.run(APP_NAME+" cleaner", self._cache_cleaner)
def _find_revision(self, revision): please_stop = False locker = Lock() output = [] queue = Queue("branches", max=2000) queue.extend(b for b in self.branches if b.locale == DEFAULT_LOCALE and b.name in ["try", "mozilla-inbound", "autoland"]) queue.add(THREAD_STOP) problems = [] def _find(please_stop): for b in queue: if please_stop: return try: url = b.url + "json-info?node=" + revision rev = self.get_revision(Revision(branch=b, changeset={"id": revision})) with locker: output.append(rev) Log.note("Revision found at {{url}}", url=url) except Exception as f: problems.append(f) threads = [] for i in range(3): threads.append(Thread.run("find changeset " + text_type(i), _find, please_stop=please_stop)) for t in threads: with assert_no_exception: t.join() return output
def __init__(self, max): """ :param max: Maximum number of concurrent threads, all others will block """ self.lock = Lock() self.max = max self.remaining = max
def test_thread_wait(self): NUM = 100 locker = Lock("test") phase1 = [] phase2 = [] def work(value, please_stop): with locker: phase1.append(value) locker.wait() phase2.append(value) with locker: threads = [Thread.run(unicode(i), work, i) for i in range(NUM)] # CONTINUE TO USE THE locker SO WAITS GET TRIGGERED while len(phase2) < NUM: with locker: pass for t in threads: t.join() self.assertEqual(len(phase1), NUM, "expecting "+unicode(NUM)+" items") self.assertEqual(len(phase2), NUM, "expecting "+unicode(NUM)+" items") for i in range(NUM): self.assertTrue(i in phase1, "expecting "+unicode(i)) self.assertTrue(i in phase2, "expecting "+unicode(i)) Log.note("done")
def test_lock_and_till(self): locker = Lock("prime lock") got_lock = Signal() a_is_ready = Signal("a lock") b_is_ready = Signal("b lock") def loop(is_ready, please_stop): with locker: while not got_lock: # Log.note("{{thread}} is waiting", thread=Thread.current().name) locker.wait(till=Till(seconds=0)) is_ready.go() locker.wait() Log.note("thread is expected to get here") thread_a = Thread.run("a", loop, a_is_ready) thread_b = Thread.run("b", loop, b_is_ready) a_is_ready.wait() b_is_ready.wait() with locker: got_lock.go() Till(seconds=0.1).wait() # MUST WAIT FOR a AND b TO PERFORM locker.wait() Log.note("leaving") pass with locker: Log.note("leaving again") pass Till(seconds=1).wait() self.assertTrue(bool(thread_a.stopped), "Thread should be done by now") self.assertTrue(bool(thread_b.stopped), "Thread should be done by now")
def __init__( self, rollover_field, # the FIELD with a timestamp to use for determining which index to push to rollover_interval, # duration between roll-over to new index rollover_max, # remove old indexes, do not add old records schema, # es schema queue_size=10000, # number of documents to queue in memory batch_size=5000, # number of documents to push at once typed=None, # indicate if we are expected typed json kwargs=None # plus additional ES settings ): if kwargs.tjson != None: Log.error("not expected") if typed == None: Log.error("not expected") schema.settings.index.max_result_window = 100000 # REQUIRED FOR ACTIVEDATA NESTED QUERIES schema.settings.index.max_inner_result_window = 100000 # REQUIRED FOR ACTIVEDATA NESTED QUERIES self.settings = kwargs self.locker = Lock("lock for rollover_index") self.rollover_field = jx.get(rollover_field) self.rollover_interval = self.settings.rollover_interval = Duration( rollover_interval) self.rollover_max = self.settings.rollover_max = Duration(rollover_max) self.known_queues = {} # MAP DATE TO INDEX self.cluster = elasticsearch.Cluster(self.settings)
def __init__(self, duration=DAY, lock=False, ignore=None): self.timeout = duration self.ignore = ignore if lock: self.locker = Lock() else: self.locker = _FakeLock()
def __init__(self, name, target, *args, **kwargs): self.id = -1 self.name = name self.target = target self.end_of_thread = None self.synch_lock = Lock("response synch lock") self.args = args # ENSURE THERE IS A SHARED please_stop SIGNAL self.kwargs = copy(kwargs) self.kwargs["please_stop"] = self.kwargs.get( "please_stop", Signal("please_stop for " + self.name)) self.please_stop = self.kwargs["please_stop"] self.thread = None self.stopped = Signal("stopped signal for " + self.name) self.cprofiler = None self.children = [] if "parent_thread" in kwargs: del self.kwargs["parent_thread"] self.parent = kwargs["parent_thread"] else: self.parent = Thread.current() self.parent.add_child(self)
def __init__(self, from_address, to_address, subject, region, aws_access_key_id=None, aws_secret_access_key=None, cc=None, log_type="ses", average_interval=HOUR, kwargs=None): """ SEND WARNINGS AND ERRORS VIA EMAIL settings = { "log_type": "ses", "from_address": "*****@*****.**", "to_address": "*****@*****.**", "cc":[ {"to_address":"*****@*****.**", "where":{"eq":{"template":"gr"}}} ], "subject": "[ALERT][STAGING] Problem in ETL", "aws_access_key_id": "userkey" "aws_secret_access_key": "secret" "region":"us-west-2" } """ assert kwargs.log_type == "ses", "Expecing settings to be of type 'ses'" self.settings = kwargs self.accumulation = [] self.cc = listwrap(cc) self.next_send = Date.now() + MINUTE self.locker = Lock() self.settings.average_interval = Duration(kwargs.average_interval)
def __init__(self, name, config): config = to_data(config) if config.debug.logs: Log.error("not allowed to configure logging on other process") Log.note("begin process") # WINDOWS REQUIRED shell, WHILE LINUX NOT shell = "windows" in platform.system().lower() self.process = Process( name, [PYTHON, "-u", "mo_threads" + os.sep + "python_worker.py"], debug=False, cwd=os.getcwd(), shell=shell) self.process.stdin.add( value2json(set_default({}, config, {"debug": { "trace": True }}))) status = self.process.stdout.pop() if status != '{"out":"ok"}': Log.error("could not start python\n{{error|indent}}", error=self.process.stderr.pop_all() + [status] + self.process.stdin.pop_all()) self.lock = Lock("wait for response from " + name) self.current_task = DONE self.current_response = None self.current_error = None self.daemon = Thread.run("", self._daemon) self.errors = Thread.run("", self._stderr)
def __init__(self, db, parent, thread): self.db = db self.locker = Lock("transaction " + text(id(self)) + " todo lock") self.todo = [] self.complete = 0 self.end_of_life = False self.exception = None self.parent = parent self.thread = thread
def __copy__(self): output = object.__new__(ColumnList) Table.__init__(output, "meta.columns") output.data = { t: {c: list(cs) for c, cs in dd.items()} for t, dd in self.data.items() } output.locker = Lock() output._schema = None return output
def __init__(self, db, parent=None): self.db = db self.locker = Lock("transaction " + text_type(id(self)) + " todo lock") self.todo = [] self.complete = 0 self.end_of_life = False self.exception = None self.parent = parent self.thread = parent.thread if parent else Thread.current()
def __init__( self, work_queue, # SETTINGS FOR AWS QUEUE connect, # SETTINGS FOR Fabric `env` TO CONNECT TO INSTANCE minimum_utility, kwargs=None): InstanceManager.__init__(self, kwargs) self.locker = Lock() self.settings = kwargs
def __init__(self, host, user, password, database=None, port=5439, kwargs=None): self.settings = kwargs self.locker = Lock() self.connection = None
def test_lock_wait_timeout(self): locker = Lock("test") def take_lock(please_stop): with locker: locker.wait(Till(seconds=1)) locker.wait(Till(seconds=1)) locker.wait(Till(till=(Date.now()+SECOND).unix)) t = Thread.run("take lock", take_lock) t.join()
def __init__(self, name, data, schema=None): #TODO: STORE THIS LIKE A CUBE FOR FASTER ACCESS AND TRANSFORMATION data = list(unwrap(data)) Container.__init__(self, data, schema) if schema == None: self._schema = get_schema_from_list(name, data) else: self._schema = schema self.name = name self.data = data self.locker = Lock() # JUST IN CASE YOU WANT TO DO MORE THAN ONE THING
def __init__(self, _file): """ file - USES FILE FOR PERSISTENCE """ self.file = File.new_instance(_file) self.lock = Lock("lock for persistent queue using file " + self.file.name) self.please_stop = Signal() self.db = Data() self.pending = [] if self.file.exists: for line in self.file: with suppress_exception: delta = mo_json.json2value(line) apply_delta(self.db, delta) if self.db.status.start == None: # HAPPENS WHEN ONLY ADDED TO QUEUE, THEN CRASH self.db.status.start = 0 self.start = self.db.status.start # SCRUB LOST VALUES lost = 0 for k in self.db.keys(): with suppress_exception: if k != "status" and int(k) < self.start: self.db[k] = None lost += 1 # HAPPENS FOR self.db.status, BUT MAYBE OTHER PROPERTIES TOO if lost: Log.warning("queue file had {{num}} items lost", num=lost) if DEBUG: Log.note("Persistent queue {{name}} found with {{num}} items", name=self.file.abspath, num=len(self)) else: self.db.status = Data(start=0, end=0) self.start = self.db.status.start if DEBUG: Log.note("New persistent queue {{name}}", name=self.file.abspath)
def work_loop(self, please_stop): this = self connected = False work_list = {} work_list_lock = Lock() def process_task(mail, please_stop=None): try: if not this.call: this.call = this.celery.Task.__call__ this.dummy = this.celery.Task() name = mail.sender.name args = (mail,) if mail.sender.bind else tuple() this._status_update(mail, states.STARTED, {"response": {"start_time": Date.now().format()}}) fun = this.celery._tasks[name] mail.result = this.call(this.dummy, fun, *args, **unwrap(mail.message)) mail.status = states.SUCCESS except Exception as e: mail.result = Except.wrap(e) mail.status = states.FAILURE # mail = wrap({"request": {"id": mail.request.id}, "sender": {"name": "mail.sender.name"}}) Log.warning("worker failed to process {{mail}}", mail=mail, cause=e) mail.response.end_time = Date.now().format() if isinstance(mail.result, Exception): mail.result = Except.wrap(mail.result) mail.receiver.thread = None Log.note("Add {{id}} ({{name}}) to response queue\n{{result}}", id=mail.request.id, name=mail.sender.name, result=mail) this.response_queue.add(value2json(mail)) with work_list_lock: del work_list[mail.request.id] while not please_stop: try: _mail = json2value(self.request_queue.pop(till=please_stop)) except Exception as e: Log.warning("Could not pop work of request queue", cause=e) continue # MUST WAIT BEFORE TRYING TO CALL THE worker_process_inits if not connected: connected = True for r in worker_process_init.registered: r() Log.note("Got {{id}} ({{name}}) from request queue", id=_mail.request.id, name=_mail.sender.name) with work_list_lock: work_list[_mail.request.id] = _mail # _mail.receiver.thread = Thread.run(_mail.sender.name, process_task, _mail) process_task(_mail)
def __init__(self, db): Table.__init__(self, META_COLUMNS_NAME) self.data = {} # MAP FROM fact_name TO (abs_column_name to COLUMNS) self.locker = Lock() self._schema = None self.dirty = False self.db = db self.es_index = None self.last_load = Null self.todo = Queue( "update columns to es" ) # HOLD (action, column) PAIR, WHERE action in ['insert', 'update'] self._snowflakes = Data() self._load_from_database()
def __init__(self, name, max=None, silent=False, unique=False, allow_add_after_close=False): """ max - LIMIT THE NUMBER IN THE QUEUE, IF TOO MANY add() AND extend() WILL BLOCK silent - COMPLAIN IF THE READERS ARE TOO SLOW unique - SET True IF YOU WANT ONLY ONE INSTANCE IN THE QUEUE AT A TIME """ if not _Log: _late_import() self.name = name self.max = coalesce(max, 2**10) self.silent = silent self.allow_add_after_close = allow_add_after_close self.unique = unique self.please_stop = Signal("stop signal for " + name) self.lock = Lock("lock for queue " + name) self.queue = deque() self.next_warning = time() # FOR DEBUGGING
def test_and_signals(self): acc = [] locker = Lock() def worker(please_stop): with locker: acc.append("worker") a = Thread.run("a", worker) b = Thread.run("b", worker) c = Thread.run("c", worker) (a.stopped & b.stopped & c.stopped).wait() acc.append("done") self.assertEqual(acc, ["worker", "worker", "worker", "done"])
def __init__(self, filename=None, db=None, get_trace=None, upgrade=True, load_functions=False, kwargs=None): """ :param filename: FILE TO USE FOR DATABASE :param db: AN EXISTING sqlite3 DB YOU WOULD LIKE TO USE (INSTEAD OF USING filename) :param get_trace: GET THE STACK TRACE AND THREAD FOR EVERY DB COMMAND (GOOD FOR DEBUGGING) :param upgrade: REPLACE PYTHON sqlite3 DLL WITH MORE RECENT ONE, WITH MORE FUNCTIONS (NOT WORKING) :param load_functions: LOAD EXTENDED MATH FUNCTIONS (MAY REQUIRE upgrade) :param kwargs: """ if upgrade and not _upgraded: _upgrade() self.settings = kwargs self.filename = File(filename).abspath if known_databases.get(self.filename): Log.error("Not allowed to create more than one Sqlite instance for {{file}}", file=self.filename) # SETUP DATABASE DEBUG and Log.note("Sqlite version {{version}}", version=sqlite3.sqlite_version) try: if db == None: self.db = sqlite3.connect( database=coalesce(self.filename, ":memory:"), check_same_thread=False, isolation_level=None ) else: self.db = db except Exception as e: Log.error("could not open file {{filename}}", filename=self.filename, cause=e) load_functions and self._load_functions() self.locker = Lock() self.available_transactions = [] # LIST OF ALL THE TRANSACTIONS BEING MANAGED self.queue = Queue("sql commands") # HOLD (command, result, signal, stacktrace) TUPLES self.get_trace = coalesce(get_trace, TRACE) self.upgrade = upgrade self.closed = False # WORKER VARIABLES self.transaction_stack = [] # THE TRANSACTION OBJECT WE HAVE PARTIALLY RUN self.last_command_item = None # USE THIS TO HELP BLAME current_transaction FOR HANGING ON TOO LONG self.too_long = None self.delayed_queries = [] self.delayed_transactions = [] self.worker = Thread.run("sqlite db thread", self._worker) DEBUG and Log.note("Sqlite version {{version}}", version=self.query("select sqlite_version()").data[0][0])
def __init__(self, name): Table.__init__(self, "meta.columns") self.db_file = File("metadata." + name + ".sqlite") self.data = {} # MAP FROM ES_INDEX TO (abs_column_name to COLUMNS) self.locker = Lock() self._schema = None self.db = sqlite3.connect( database=self.db_file.abspath, check_same_thread=False, isolation_level=None ) self.last_load = Null self.todo = Queue( "update columns to db" ) # HOLD (action, column) PAIR, WHERE action in ['insert', 'update'] self._db_load() Thread.run("update " + name, self._db_worker)
def __init__(self, name, config): config = wrap(config) if config.debug.logs: Log.error("not allowed to configure logging on other process") self.process = Process(name, [PYTHON, "mo_threads" + os.sep + "python_worker.py"], shell=True) self.process.stdin.add(value2json(set_default({"debug": {"trace": True}}, config))) self.lock = Lock("wait for response from "+name) self.current_task = None self.current_response = None self.current_error = None self.daemon = Thread.run("", self._daemon) self.errors = Thread.run("", self._stderr)
def __init__(self, instance_manager, disable_prices=False, kwargs=None): self.settings = kwargs self.instance_manager = instance_manager aws_args = dict(region_name=kwargs.aws.region, aws_access_key_id=unwrap(kwargs.aws.aws_access_key_id), aws_secret_access_key=unwrap( kwargs.aws.aws_secret_access_key)) self.ec2_conn = boto.ec2.connect_to_region(**aws_args) self.vpc_conn = boto.vpc.connect_to_region(**aws_args) self.price_locker = Lock() self.prices = None self.price_lookup = None self.no_capacity = {} self.no_capacity_file = File( kwargs.price_file).parent / "no capacity.json" self.done_making_new_spot_requests = Signal() self.net_new_locker = Lock() self.net_new_spot_requests = UniqueIndex( ("id", )) # SPOT REQUESTS FOR THIS SESSION self.watcher = None self.active = None self.settings.uptime.bid_percentile = coalesce( self.settings.uptime.bid_percentile, self.settings.bid_percentile) self.settings.uptime.history = coalesce( Date(self.settings.uptime.history), DAY) self.settings.uptime.duration = coalesce( Duration(self.settings.uptime.duration), Date("5minute")) self.settings.max_percent_per_type = coalesce( self.settings.max_percent_per_type, 1) if ENABLE_SIDE_EFFECTS and instance_manager and instance_manager.setup_required( ): self._start_life_cycle_watcher() if not disable_prices: self.pricing()
def __init__(self, _file): """ file - USES FILE FOR PERSISTENCE """ self.file = File.new_instance(_file) self.lock = Lock("lock for persistent queue using file " + self.file.name) self.please_stop = Signal() self.db = Data() self.pending = [] if self.file.exists: for line in self.file: with suppress_exception: delta = mo_json.json2value(line) apply_delta(self.db, delta) if self.db.status.start == None: # HAPPENS WHEN ONLY ADDED TO QUEUE, THEN CRASH self.db.status.start = 0 self.start = self.db.status.start # SCRUB LOST VALUES lost = 0 for k in self.db.keys(): with suppress_exception: if k!="status" and int(k) < self.start: self.db[k] = None lost += 1 # HAPPENS FOR self.db.status, BUT MAYBE OTHER PROPERTIES TOO if lost: Log.warning("queue file had {{num}} items lost", num= lost) DEBUG and Log.note("Persistent queue {{name}} found with {{num}} items", name=self.file.abspath, num=len(self)) else: self.db.status = Data( start=0, end=0 ) self.start = self.db.status.start DEBUG and Log.note("New persistent queue {{name}}", name=self.file.abspath)
class PersistentQueue(object): """ THREAD-SAFE, PERSISTENT QUEUE CAN HANDLE MANY PRODUCERS, BUT THE pop(), commit() IDIOM CAN HANDLE ONLY ONE CONSUMER. IT IS IMPORTANT YOU commit() or close(), OTHERWISE NOTHING COMES OFF THE QUEUE """ def __init__(self, _file): """ file - USES FILE FOR PERSISTENCE """ self.file = File.new_instance(_file) self.lock = Lock("lock for persistent queue using file " + self.file.name) self.please_stop = Signal() self.db = Data() self.pending = [] if self.file.exists: for line in self.file: with suppress_exception: delta = mo_json.json2value(line) apply_delta(self.db, delta) if self.db.status.start == None: # HAPPENS WHEN ONLY ADDED TO QUEUE, THEN CRASH self.db.status.start = 0 self.start = self.db.status.start # SCRUB LOST VALUES lost = 0 for k in self.db.keys(): with suppress_exception: if k!="status" and int(k) < self.start: self.db[k] = None lost += 1 # HAPPENS FOR self.db.status, BUT MAYBE OTHER PROPERTIES TOO if lost: Log.warning("queue file had {{num}} items lost", num= lost) DEBUG and Log.note("Persistent queue {{name}} found with {{num}} items", name=self.file.abspath, num=len(self)) else: self.db.status = Data( start=0, end=0 ) self.start = self.db.status.start DEBUG and Log.note("New persistent queue {{name}}", name=self.file.abspath) def _add_pending(self, delta): delta = wrap(delta) self.pending.append(delta) def _apply_pending(self): for delta in self.pending: apply_delta(self.db, delta) self.pending = [] def __iter__(self): """ BLOCKING ITERATOR """ while not self.please_stop: try: value = self.pop() if value is not THREAD_STOP: yield value except Exception as e: Log.warning("Tell me about what happened here", cause=e) def add(self, value): with self.lock: if self.closed: Log.error("Queue is closed") if value is THREAD_STOP: DEBUG and Log.note("Stop is seen in persistent queue") self.please_stop.go() return self._add_pending({"add": {str(self.db.status.end): value}}) self.db.status.end += 1 self._add_pending({"add": {"status.end": self.db.status.end}}) self._commit() return self def __len__(self): with self.lock: return self.db.status.end - self.start def __getitem__(self, item): return self.db[str(item + self.start)] def pop(self, timeout=None): """ :param timeout: OPTIONAL DURATION :return: None, IF timeout PASSES """ with self.lock: while not self.please_stop: if self.db.status.end > self.start: value = self.db[str(self.start)] self.start += 1 return value if timeout is not None: with suppress_exception: self.lock.wait(timeout=timeout) if self.db.status.end <= self.start: return None else: self.lock.wait() DEBUG and Log.note("persistent queue already stopped") return THREAD_STOP def pop_all(self): """ NON-BLOCKING POP ALL IN QUEUE, IF ANY """ with self.lock: if self.please_stop: return [THREAD_STOP] if self.db.status.end == self.start: return [] output = [] for i in range(self.start, self.db.status.end): output.append(self.db[str(i)]) self.start = self.db.status.end return output def rollback(self): with self.lock: if self.closed: return self.start = self.db.status.start self.pending = [] def commit(self): with self.lock: if self.closed: Log.error("Queue is closed, commit not allowed") try: self._add_pending({"add": {"status.start": self.start}}) for i in range(self.db.status.start, self.start): self._add_pending({"remove": str(i)}) if self.db.status.end - self.start < 10 or Random.range(0, 1000) == 0: # FORCE RE-WRITE TO LIMIT FILE SIZE # SIMPLY RE-WRITE FILE if DEBUG: Log.note("Re-write {{num_keys}} keys to persistent queue", num_keys=self.db.status.end - self.start) for k in self.db.keys(): if k == "status" or int(k) >= self.db.status.start: continue Log.error("Not expecting {{key}}", key=k) self._commit() self.file.write(mo_json.value2json({"add": self.db}) + "\n") else: self._commit() except Exception as e: raise e def _commit(self): self.file.append("\n".join(mo_json.value2json(p) for p in self.pending)) self._apply_pending() def close(self): self.please_stop.go() with self.lock: if self.db is None: return self.add(THREAD_STOP) if self.db.status.end == self.start: DEBUG and Log.note("persistent queue clear and closed") self.file.delete() else: DEBUG and Log.note("persistent queue closed with {{num}} items left", num=len(self)) try: self._add_pending({"add": {"status.start": self.start}}) for i in range(self.db.status.start, self.start): self._add_pending({"remove": str(i)}) self.file.write(mo_json.value2json({"add": self.db}) + "\n" + ("\n".join(mo_json.value2json(p) for p in self.pending)) + "\n") self._apply_pending() except Exception as e: raise e self.db = None @property def closed(self): with self.lock: return self.db is None