def __init__(self):
self.out_of_memory_restart = False
self.total_locker = Lock()
self.total_files_requested = 0
self.total_tuids_mapped = 0
self.threads_locker = Lock()
self.waiting = 0
self.threads_waiting = 0
self.requests_locker = Lock()
self.requests_total = 0
self.requests_complete = 0
self.requests_incomplete = 0
self.requests_passed = 0
self.requests_failed = 0
self.prev_mem = 0
self.curr_mem = 0
self.initial_growth = {}
Thread.run("pc-daemon", self.run_pc_daemon)
Thread.run("threads-daemon", self.run_threads_daemon)
Thread.run("memory-daemon", self.run_memory_daemon)
Thread.run("requests-daemon", self.run_requests_daemon)
def __init__(self, conn=None, tuid_service=None, kwargs=None):
try:
self.config = kwargs
self.conn = conn if conn else sql.Sql(self.config.database.name)
self.hg_cache = HgMozillaOrg(
kwargs=self.config.hg_cache,
use_cache=True) if self.config.hg_cache else Null
self.tuid_service = tuid_service if tuid_service else tuid.service.TUIDService(
database=None,
hg=None,
kwargs=self.config,
conn=self.conn,
clogger=self)
self.rev_locker = Lock()
self.working_locker = Lock()
self.init_db()
self.next_revnum = coalesce(
self.conn.get_one("SELECT max(revnum)+1 FROM csetLog")[0], 1)
self.csets_todo_backwards = Queue(
name="Clogger.csets_todo_backwards")
self.deletions_todo = Queue(name="Clogger.deletions_todo")
self.maintenance_signal = Signal(name="Clogger.maintenance_signal")
self.config = self.config.tuid
self.disable_backfilling = False
self.disable_tipfilling = False
self.disable_deletion = False
self.disable_maintenance = False
# Make sure we are filled before allowing queries
numrevs = self.conn.get_one("SELECT count(revnum) FROM csetLog")[0]
if numrevs < MINIMUM_PERMANENT_CSETS:
Log.note("Filling in csets to hold {{minim}} csets.",
minim=MINIMUM_PERMANENT_CSETS)
oldest_rev = 'tip'
with self.conn.transaction() as t:
tmp = t.query(
"SELECT min(revnum), revision FROM csetLog").data[0][1]
if tmp:
oldest_rev = tmp
self._fill_in_range(MINIMUM_PERMANENT_CSETS - numrevs,
oldest_rev,
timestamp=False)
Log.note(
"Table is filled with atleast {{minim}} entries. Starting workers...",
minim=MINIMUM_PERMANENT_CSETS)
Thread.run('clogger-tip', self.fill_forward_continuous)
Thread.run('clogger-backfill', self.fill_backward_with_list)
Thread.run('clogger-maintenance', self.csetLog_maintenance)
Thread.run('clogger-deleter', self.csetLog_deleter)
Log.note("Started clogger workers.")
except Exception as e:
Log.warning("Cannot setup clogger: {{cause}}", cause=str(e))
def __init__(self, name):
self.name = name
self.lock = Lock("rate locker")
self.request_rate = 0.0
self.last_request = Date.now()
Thread.run("rate logger", self._daemon)
def __init__(self,
rollover_field,
rollover_interval,
rollover_max,
queue_size=10000,
batch_size=5000,
kwargs=None):
"""
:param rollover_field: the FIELD with a timestamp to use for determining which index to push to
:param rollover_interval: duration between roll-over to new index
:param rollover_max: remove old indexes, do not add old records
:param queue_size: number of documents to queue in memory
:param batch_size: number of documents to push at once
:param kwargs: plus additional ES settings
:return:
"""
self.settings = kwargs
self.locker = Lock("lock for rollover_index")
self.rollover_field = jx.get(rollover_field)
self.rollover_interval = self.settings.rollover_interval = Duration(
kwargs.rollover_interval)
self.rollover_max = self.settings.rollover_max = Duration(
kwargs.rollover_max)
self.known_queues = {} # MAP DATE TO INDEX
self.cluster = elasticsearch.Cluster(self.settings)
def __init__(self, rate=None, amortization_period=None, source=None, database=None, kwargs=None):
self.amortization_period = coalesce(amortization_period, AMORTIZATION_PERIOD)
self.rate = coalesce(rate, HG_REQUEST_PER_SECOND)
self.cache_locker = Lock()
self.cache = {} # MAP FROM url TO (ready, headers, response, timestamp) PAIR
self.no_cache = {} # VERY SHORT TERM CACHE
self.workers = []
self.todo = Queue(APP_NAME+" todo")
self.requests = Queue(APP_NAME + " requests", max=int(self.rate * self.amortization_period.seconds))
self.url = URL(source.url)
self.db = Sqlite(database)
self.inbound_rate = RateLogger("Inbound")
self.outbound_rate = RateLogger("hg.mo")
if not self.db.query("SELECT name FROM sqlite_master WHERE type='table'").data:
with self.db.transaction() as t:
t.execute(
"CREATE TABLE cache ("
" path TEXT PRIMARY KEY, "
" headers TEXT, "
" response TEXT, "
" timestamp REAL "
")"
)
self.threads = [
Thread.run(APP_NAME+" worker" + text_type(i), self._worker)
for i in range(CONCURRENCY)
]
self.limiter = Thread.run(APP_NAME+" limiter", self._rate_limiter)
self.cleaner = Thread.run(APP_NAME+" cleaner", self._cache_cleaner)
def _find_revision(self, revision):
please_stop = False
locker = Lock()
output = []
queue = Queue("branches", max=2000)
queue.extend(b for b in self.branches if b.locale == DEFAULT_LOCALE and b.name in ["try", "mozilla-inbound", "autoland"])
queue.add(THREAD_STOP)
problems = []
def _find(please_stop):
for b in queue:
if please_stop:
return
try:
url = b.url + "json-info?node=" + revision
rev = self.get_revision(Revision(branch=b, changeset={"id": revision}))
with locker:
output.append(rev)
Log.note("Revision found at {{url}}", url=url)
except Exception as f:
problems.append(f)
threads = []
for i in range(3):
threads.append(Thread.run("find changeset " + text_type(i), _find, please_stop=please_stop))
for t in threads:
with assert_no_exception:
t.join()
return output
def __init__(self, max):
"""
:param max: Maximum number of concurrent threads, all others will block
"""
self.lock = Lock()
self.max = max
self.remaining = max
def test_thread_wait(self):
NUM = 100
locker = Lock("test")
phase1 = []
phase2 = []
def work(value, please_stop):
with locker:
phase1.append(value)
locker.wait()
phase2.append(value)
with locker:
threads = [Thread.run(unicode(i), work, i) for i in range(NUM)]
# CONTINUE TO USE THE locker SO WAITS GET TRIGGERED
while len(phase2) < NUM:
with locker:
pass
for t in threads:
t.join()
self.assertEqual(len(phase1), NUM, "expecting "+unicode(NUM)+" items")
self.assertEqual(len(phase2), NUM, "expecting "+unicode(NUM)+" items")
for i in range(NUM):
self.assertTrue(i in phase1, "expecting "+unicode(i))
self.assertTrue(i in phase2, "expecting "+unicode(i))
Log.note("done")
def test_lock_and_till(self):
locker = Lock("prime lock")
got_lock = Signal()
a_is_ready = Signal("a lock")
b_is_ready = Signal("b lock")
def loop(is_ready, please_stop):
with locker:
while not got_lock:
# Log.note("{{thread}} is waiting", thread=Thread.current().name)
locker.wait(till=Till(seconds=0))
is_ready.go()
locker.wait()
Log.note("thread is expected to get here")
thread_a = Thread.run("a", loop, a_is_ready)
thread_b = Thread.run("b", loop, b_is_ready)
a_is_ready.wait()
b_is_ready.wait()
with locker:
got_lock.go()
Till(seconds=0.1).wait() # MUST WAIT FOR a AND b TO PERFORM locker.wait()
Log.note("leaving")
pass
with locker:
Log.note("leaving again")
pass
Till(seconds=1).wait()
self.assertTrue(bool(thread_a.stopped), "Thread should be done by now")
self.assertTrue(bool(thread_b.stopped), "Thread should be done by now")
def __init__(
self,
rollover_field, # the FIELD with a timestamp to use for determining which index to push to
rollover_interval, # duration between roll-over to new index
rollover_max, # remove old indexes, do not add old records
schema, # es schema
queue_size=10000, # number of documents to queue in memory
batch_size=5000, # number of documents to push at once
typed=None, # indicate if we are expected typed json
kwargs=None # plus additional ES settings
):
if kwargs.tjson != None:
Log.error("not expected")
if typed == None:
Log.error("not expected")
schema.settings.index.max_result_window = 100000 # REQUIRED FOR ACTIVEDATA NESTED QUERIES
schema.settings.index.max_inner_result_window = 100000 # REQUIRED FOR ACTIVEDATA NESTED QUERIES
self.settings = kwargs
self.locker = Lock("lock for rollover_index")
self.rollover_field = jx.get(rollover_field)
self.rollover_interval = self.settings.rollover_interval = Duration(
rollover_interval)
self.rollover_max = self.settings.rollover_max = Duration(rollover_max)
self.known_queues = {} # MAP DATE TO INDEX
self.cluster = elasticsearch.Cluster(self.settings)
def __init__(self, duration=DAY, lock=False, ignore=None):
self.timeout = duration
self.ignore = ignore
if lock:
self.locker = Lock()
else:
self.locker = _FakeLock()
def __init__(self, name, target, *args, **kwargs):
self.id = -1
self.name = name
self.target = target
self.end_of_thread = None
self.synch_lock = Lock("response synch lock")
self.args = args
# ENSURE THERE IS A SHARED please_stop SIGNAL
self.kwargs = copy(kwargs)
self.kwargs["please_stop"] = self.kwargs.get(
"please_stop", Signal("please_stop for " + self.name))
self.please_stop = self.kwargs["please_stop"]
self.thread = None
self.stopped = Signal("stopped signal for " + self.name)
self.cprofiler = None
self.children = []
if "parent_thread" in kwargs:
del self.kwargs["parent_thread"]
self.parent = kwargs["parent_thread"]
else:
self.parent = Thread.current()
self.parent.add_child(self)
def __init__(self,
from_address,
to_address,
subject,
region,
aws_access_key_id=None,
aws_secret_access_key=None,
cc=None,
log_type="ses",
average_interval=HOUR,
kwargs=None):
"""
SEND WARNINGS AND ERRORS VIA EMAIL
settings = {
"log_type": "ses",
"from_address": "*****@*****.**",
"to_address": "*****@*****.**",
"cc":[
{"to_address":"*****@*****.**", "where":{"eq":{"template":"gr"}}}
],
"subject": "[ALERT][STAGING] Problem in ETL",
"aws_access_key_id": "userkey"
"aws_secret_access_key": "secret"
"region":"us-west-2"
}
"""
assert kwargs.log_type == "ses", "Expecing settings to be of type 'ses'"
self.settings = kwargs
self.accumulation = []
self.cc = listwrap(cc)
self.next_send = Date.now() + MINUTE
self.locker = Lock()
self.settings.average_interval = Duration(kwargs.average_interval)
def __init__(self, name, config):
config = to_data(config)
if config.debug.logs:
Log.error("not allowed to configure logging on other process")
Log.note("begin process")
# WINDOWS REQUIRED shell, WHILE LINUX NOT
shell = "windows" in platform.system().lower()
self.process = Process(
name, [PYTHON, "-u", "mo_threads" + os.sep + "python_worker.py"],
debug=False,
cwd=os.getcwd(),
shell=shell)
self.process.stdin.add(
value2json(set_default({}, config, {"debug": {
"trace": True
}})))
status = self.process.stdout.pop()
if status != '{"out":"ok"}':
Log.error("could not start python\n{{error|indent}}",
error=self.process.stderr.pop_all() + [status] +
self.process.stdin.pop_all())
self.lock = Lock("wait for response from " + name)
self.current_task = DONE
self.current_response = None
self.current_error = None
self.daemon = Thread.run("", self._daemon)
self.errors = Thread.run("", self._stderr)
def __init__(self, db, parent, thread):
self.db = db
self.locker = Lock("transaction " + text(id(self)) + " todo lock")
self.todo = []
self.complete = 0
self.end_of_life = False
self.exception = None
self.parent = parent
self.thread = thread
def __copy__(self):
output = object.__new__(ColumnList)
Table.__init__(output, "meta.columns")
output.data = {
t: {c: list(cs) for c, cs in dd.items()} for t, dd in self.data.items()
}
output.locker = Lock()
output._schema = None
return output
def __init__(self, db, parent=None):
self.db = db
self.locker = Lock("transaction " + text_type(id(self)) + " todo lock")
self.todo = []
self.complete = 0
self.end_of_life = False
self.exception = None
self.parent = parent
self.thread = parent.thread if parent else Thread.current()
def __init__(
self,
work_queue, # SETTINGS FOR AWS QUEUE
connect, # SETTINGS FOR Fabric `env` TO CONNECT TO INSTANCE
minimum_utility,
kwargs=None):
InstanceManager.__init__(self, kwargs)
self.locker = Lock()
self.settings = kwargs
def __init__(self,
host,
user,
password,
database=None,
port=5439,
kwargs=None):
self.settings = kwargs
self.locker = Lock()
self.connection = None
def test_lock_wait_timeout(self):
locker = Lock("test")
def take_lock(please_stop):
with locker:
locker.wait(Till(seconds=1))
locker.wait(Till(seconds=1))
locker.wait(Till(till=(Date.now()+SECOND).unix))
t = Thread.run("take lock", take_lock)
t.join()
def __init__(self, name, data, schema=None):
#TODO: STORE THIS LIKE A CUBE FOR FASTER ACCESS AND TRANSFORMATION
data = list(unwrap(data))
Container.__init__(self, data, schema)
if schema == None:
self._schema = get_schema_from_list(name, data)
else:
self._schema = schema
self.name = name
self.data = data
self.locker = Lock() # JUST IN CASE YOU WANT TO DO MORE THAN ONE THING
def __init__(self, _file):
"""
file - USES FILE FOR PERSISTENCE
"""
self.file = File.new_instance(_file)
self.lock = Lock("lock for persistent queue using file " +
self.file.name)
self.please_stop = Signal()
self.db = Data()
self.pending = []
if self.file.exists:
for line in self.file:
with suppress_exception:
delta = mo_json.json2value(line)
apply_delta(self.db, delta)
if self.db.status.start == None: # HAPPENS WHEN ONLY ADDED TO QUEUE, THEN CRASH
self.db.status.start = 0
self.start = self.db.status.start
# SCRUB LOST VALUES
lost = 0
for k in self.db.keys():
with suppress_exception:
if k != "status" and int(k) < self.start:
self.db[k] = None
lost += 1
# HAPPENS FOR self.db.status, BUT MAYBE OTHER PROPERTIES TOO
if lost:
Log.warning("queue file had {{num}} items lost", num=lost)
if DEBUG:
Log.note("Persistent queue {{name}} found with {{num}} items",
name=self.file.abspath,
num=len(self))
else:
self.db.status = Data(start=0, end=0)
self.start = self.db.status.start
if DEBUG:
Log.note("New persistent queue {{name}}",
name=self.file.abspath)
def work_loop(self, please_stop):
this = self
connected = False
work_list = {}
work_list_lock = Lock()
def process_task(mail, please_stop=None):
try:
if not this.call:
this.call = this.celery.Task.__call__
this.dummy = this.celery.Task()
name = mail.sender.name
args = (mail,) if mail.sender.bind else tuple()
this._status_update(mail, states.STARTED, {"response": {"start_time": Date.now().format()}})
fun = this.celery._tasks[name]
mail.result = this.call(this.dummy, fun, *args, **unwrap(mail.message))
mail.status = states.SUCCESS
except Exception as e:
mail.result = Except.wrap(e)
mail.status = states.FAILURE
# mail = wrap({"request": {"id": mail.request.id}, "sender": {"name": "mail.sender.name"}})
Log.warning("worker failed to process {{mail}}", mail=mail, cause=e)
mail.response.end_time = Date.now().format()
if isinstance(mail.result, Exception):
mail.result = Except.wrap(mail.result)
mail.receiver.thread = None
Log.note("Add {{id}} ({{name}}) to response queue\n{{result}}", id=mail.request.id, name=mail.sender.name, result=mail)
this.response_queue.add(value2json(mail))
with work_list_lock:
del work_list[mail.request.id]
while not please_stop:
try:
_mail = json2value(self.request_queue.pop(till=please_stop))
except Exception as e:
Log.warning("Could not pop work of request queue", cause=e)
continue
# MUST WAIT BEFORE TRYING TO CALL THE worker_process_inits
if not connected:
connected = True
for r in worker_process_init.registered:
r()
Log.note("Got {{id}} ({{name}}) from request queue", id=_mail.request.id, name=_mail.sender.name)
with work_list_lock:
work_list[_mail.request.id] = _mail
# _mail.receiver.thread = Thread.run(_mail.sender.name, process_task, _mail)
process_task(_mail)
def __init__(self, db):
Table.__init__(self, META_COLUMNS_NAME)
self.data = {} # MAP FROM fact_name TO (abs_column_name to COLUMNS)
self.locker = Lock()
self._schema = None
self.dirty = False
self.db = db
self.es_index = None
self.last_load = Null
self.todo = Queue(
"update columns to es"
) # HOLD (action, column) PAIR, WHERE action in ['insert', 'update']
self._snowflakes = Data()
self._load_from_database()
def __init__(self,
name,
max=None,
silent=False,
unique=False,
allow_add_after_close=False):
"""
max - LIMIT THE NUMBER IN THE QUEUE, IF TOO MANY add() AND extend() WILL BLOCK
silent - COMPLAIN IF THE READERS ARE TOO SLOW
unique - SET True IF YOU WANT ONLY ONE INSTANCE IN THE QUEUE AT A TIME
"""
if not _Log:
_late_import()
self.name = name
self.max = coalesce(max, 2**10)
self.silent = silent
self.allow_add_after_close = allow_add_after_close
self.unique = unique
self.please_stop = Signal("stop signal for " + name)
self.lock = Lock("lock for queue " + name)
self.queue = deque()
self.next_warning = time() # FOR DEBUGGING
def test_and_signals(self):
acc = []
locker = Lock()
def worker(please_stop):
with locker:
acc.append("worker")
a = Thread.run("a", worker)
b = Thread.run("b", worker)
c = Thread.run("c", worker)
(a.stopped & b.stopped & c.stopped).wait()
acc.append("done")
self.assertEqual(acc, ["worker", "worker", "worker", "done"])
def __init__(self, filename=None, db=None, get_trace=None, upgrade=True, load_functions=False, kwargs=None):
"""
:param filename: FILE TO USE FOR DATABASE
:param db: AN EXISTING sqlite3 DB YOU WOULD LIKE TO USE (INSTEAD OF USING filename)
:param get_trace: GET THE STACK TRACE AND THREAD FOR EVERY DB COMMAND (GOOD FOR DEBUGGING)
:param upgrade: REPLACE PYTHON sqlite3 DLL WITH MORE RECENT ONE, WITH MORE FUNCTIONS (NOT WORKING)
:param load_functions: LOAD EXTENDED MATH FUNCTIONS (MAY REQUIRE upgrade)
:param kwargs:
"""
if upgrade and not _upgraded:
_upgrade()
self.settings = kwargs
self.filename = File(filename).abspath
if known_databases.get(self.filename):
Log.error("Not allowed to create more than one Sqlite instance for {{file}}", file=self.filename)
# SETUP DATABASE
DEBUG and Log.note("Sqlite version {{version}}", version=sqlite3.sqlite_version)
try:
if db == None:
self.db = sqlite3.connect(
database=coalesce(self.filename, ":memory:"),
check_same_thread=False,
isolation_level=None
)
else:
self.db = db
except Exception as e:
Log.error("could not open file {{filename}}", filename=self.filename, cause=e)
load_functions and self._load_functions()
self.locker = Lock()
self.available_transactions = [] # LIST OF ALL THE TRANSACTIONS BEING MANAGED
self.queue = Queue("sql commands") # HOLD (command, result, signal, stacktrace) TUPLES
self.get_trace = coalesce(get_trace, TRACE)
self.upgrade = upgrade
self.closed = False
# WORKER VARIABLES
self.transaction_stack = [] # THE TRANSACTION OBJECT WE HAVE PARTIALLY RUN
self.last_command_item = None # USE THIS TO HELP BLAME current_transaction FOR HANGING ON TOO LONG
self.too_long = None
self.delayed_queries = []
self.delayed_transactions = []
self.worker = Thread.run("sqlite db thread", self._worker)
DEBUG and Log.note("Sqlite version {{version}}", version=self.query("select sqlite_version()").data[0][0])
def __init__(self, name):
Table.__init__(self, "meta.columns")
self.db_file = File("metadata." + name + ".sqlite")
self.data = {} # MAP FROM ES_INDEX TO (abs_column_name to COLUMNS)
self.locker = Lock()
self._schema = None
self.db = sqlite3.connect(
database=self.db_file.abspath, check_same_thread=False, isolation_level=None
)
self.last_load = Null
self.todo = Queue(
"update columns to db"
) # HOLD (action, column) PAIR, WHERE action in ['insert', 'update']
self._db_load()
Thread.run("update " + name, self._db_worker)
def __init__(self, name, config):
config = wrap(config)
if config.debug.logs:
Log.error("not allowed to configure logging on other process")
self.process = Process(name, [PYTHON, "mo_threads" + os.sep + "python_worker.py"], shell=True)
self.process.stdin.add(value2json(set_default({"debug": {"trace": True}}, config)))
self.lock = Lock("wait for response from "+name)
self.current_task = None
self.current_response = None
self.current_error = None
self.daemon = Thread.run("", self._daemon)
self.errors = Thread.run("", self._stderr)
def __init__(self, instance_manager, disable_prices=False, kwargs=None):
self.settings = kwargs
self.instance_manager = instance_manager
aws_args = dict(region_name=kwargs.aws.region,
aws_access_key_id=unwrap(kwargs.aws.aws_access_key_id),
aws_secret_access_key=unwrap(
kwargs.aws.aws_secret_access_key))
self.ec2_conn = boto.ec2.connect_to_region(**aws_args)
self.vpc_conn = boto.vpc.connect_to_region(**aws_args)
self.price_locker = Lock()
self.prices = None
self.price_lookup = None
self.no_capacity = {}
self.no_capacity_file = File(
kwargs.price_file).parent / "no capacity.json"
self.done_making_new_spot_requests = Signal()
self.net_new_locker = Lock()
self.net_new_spot_requests = UniqueIndex(
("id", )) # SPOT REQUESTS FOR THIS SESSION
self.watcher = None
self.active = None
self.settings.uptime.bid_percentile = coalesce(
self.settings.uptime.bid_percentile, self.settings.bid_percentile)
self.settings.uptime.history = coalesce(
Date(self.settings.uptime.history), DAY)
self.settings.uptime.duration = coalesce(
Duration(self.settings.uptime.duration), Date("5minute"))
self.settings.max_percent_per_type = coalesce(
self.settings.max_percent_per_type, 1)
if ENABLE_SIDE_EFFECTS and instance_manager and instance_manager.setup_required(
):
self._start_life_cycle_watcher()
if not disable_prices:
self.pricing()
def __init__(self, _file):
"""
file - USES FILE FOR PERSISTENCE
"""
self.file = File.new_instance(_file)
self.lock = Lock("lock for persistent queue using file " + self.file.name)
self.please_stop = Signal()
self.db = Data()
self.pending = []
if self.file.exists:
for line in self.file:
with suppress_exception:
delta = mo_json.json2value(line)
apply_delta(self.db, delta)
if self.db.status.start == None: # HAPPENS WHEN ONLY ADDED TO QUEUE, THEN CRASH
self.db.status.start = 0
self.start = self.db.status.start
# SCRUB LOST VALUES
lost = 0
for k in self.db.keys():
with suppress_exception:
if k!="status" and int(k) < self.start:
self.db[k] = None
lost += 1
# HAPPENS FOR self.db.status, BUT MAYBE OTHER PROPERTIES TOO
if lost:
Log.warning("queue file had {{num}} items lost", num= lost)
DEBUG and Log.note("Persistent queue {{name}} found with {{num}} items", name=self.file.abspath, num=len(self))
else:
self.db.status = Data(
start=0,
end=0
)
self.start = self.db.status.start
DEBUG and Log.note("New persistent queue {{name}}", name=self.file.abspath)
class PersistentQueue(object):
"""
THREAD-SAFE, PERSISTENT QUEUE
CAN HANDLE MANY PRODUCERS, BUT THE pop(), commit() IDIOM CAN HANDLE ONLY
ONE CONSUMER.
IT IS IMPORTANT YOU commit() or close(), OTHERWISE NOTHING COMES OFF THE QUEUE
"""
def __init__(self, _file):
"""
file - USES FILE FOR PERSISTENCE
"""
self.file = File.new_instance(_file)
self.lock = Lock("lock for persistent queue using file " + self.file.name)
self.please_stop = Signal()
self.db = Data()
self.pending = []
if self.file.exists:
for line in self.file:
with suppress_exception:
delta = mo_json.json2value(line)
apply_delta(self.db, delta)
if self.db.status.start == None: # HAPPENS WHEN ONLY ADDED TO QUEUE, THEN CRASH
self.db.status.start = 0
self.start = self.db.status.start
# SCRUB LOST VALUES
lost = 0
for k in self.db.keys():
with suppress_exception:
if k!="status" and int(k) < self.start:
self.db[k] = None
lost += 1
# HAPPENS FOR self.db.status, BUT MAYBE OTHER PROPERTIES TOO
if lost:
Log.warning("queue file had {{num}} items lost", num= lost)
DEBUG and Log.note("Persistent queue {{name}} found with {{num}} items", name=self.file.abspath, num=len(self))
else:
self.db.status = Data(
start=0,
end=0
)
self.start = self.db.status.start
DEBUG and Log.note("New persistent queue {{name}}", name=self.file.abspath)
def _add_pending(self, delta):
delta = wrap(delta)
self.pending.append(delta)
def _apply_pending(self):
for delta in self.pending:
apply_delta(self.db, delta)
self.pending = []
def __iter__(self):
"""
BLOCKING ITERATOR
"""
while not self.please_stop:
try:
value = self.pop()
if value is not THREAD_STOP:
yield value
except Exception as e:
Log.warning("Tell me about what happened here", cause=e)
def add(self, value):
with self.lock:
if self.closed:
Log.error("Queue is closed")
if value is THREAD_STOP:
DEBUG and Log.note("Stop is seen in persistent queue")
self.please_stop.go()
return
self._add_pending({"add": {str(self.db.status.end): value}})
self.db.status.end += 1
self._add_pending({"add": {"status.end": self.db.status.end}})
self._commit()
return self
def __len__(self):
with self.lock:
return self.db.status.end - self.start
def __getitem__(self, item):
return self.db[str(item + self.start)]
def pop(self, timeout=None):
"""
:param timeout: OPTIONAL DURATION
:return: None, IF timeout PASSES
"""
with self.lock:
while not self.please_stop:
if self.db.status.end > self.start:
value = self.db[str(self.start)]
self.start += 1
return value
if timeout is not None:
with suppress_exception:
self.lock.wait(timeout=timeout)
if self.db.status.end <= self.start:
return None
else:
self.lock.wait()
DEBUG and Log.note("persistent queue already stopped")
return THREAD_STOP
def pop_all(self):
"""
NON-BLOCKING POP ALL IN QUEUE, IF ANY
"""
with self.lock:
if self.please_stop:
return [THREAD_STOP]
if self.db.status.end == self.start:
return []
output = []
for i in range(self.start, self.db.status.end):
output.append(self.db[str(i)])
self.start = self.db.status.end
return output
def rollback(self):
with self.lock:
if self.closed:
return
self.start = self.db.status.start
self.pending = []
def commit(self):
with self.lock:
if self.closed:
Log.error("Queue is closed, commit not allowed")
try:
self._add_pending({"add": {"status.start": self.start}})
for i in range(self.db.status.start, self.start):
self._add_pending({"remove": str(i)})
if self.db.status.end - self.start < 10 or Random.range(0, 1000) == 0: # FORCE RE-WRITE TO LIMIT FILE SIZE
# SIMPLY RE-WRITE FILE
if DEBUG:
Log.note("Re-write {{num_keys}} keys to persistent queue", num_keys=self.db.status.end - self.start)
for k in self.db.keys():
if k == "status" or int(k) >= self.db.status.start:
continue
Log.error("Not expecting {{key}}", key=k)
self._commit()
self.file.write(mo_json.value2json({"add": self.db}) + "\n")
else:
self._commit()
except Exception as e:
raise e
def _commit(self):
self.file.append("\n".join(mo_json.value2json(p) for p in self.pending))
self._apply_pending()
def close(self):
self.please_stop.go()
with self.lock:
if self.db is None:
return
self.add(THREAD_STOP)
if self.db.status.end == self.start:
DEBUG and Log.note("persistent queue clear and closed")
self.file.delete()
else:
DEBUG and Log.note("persistent queue closed with {{num}} items left", num=len(self))
try:
self._add_pending({"add": {"status.start": self.start}})
for i in range(self.db.status.start, self.start):
self._add_pending({"remove": str(i)})
self.file.write(mo_json.value2json({"add": self.db}) + "\n" + ("\n".join(mo_json.value2json(p) for p in self.pending)) + "\n")
self._apply_pending()
except Exception as e:
raise e
self.db = None
@property
def closed(self):
with self.lock:
return self.db is None
