def get_subscriber(self, name=None):
"""
GET SUBSCRIBER BY id, OR BY QUEUE name
"""
with self.db.transaction() as t:
result = t.query(
SQL(
f"""
SELECT
MIN(s.id) as id
FROM
{SUBSCRIBER} AS s
LEFT JOIN
{QUEUE} as q on q.id = s.queue
WHERE
q.name = {quote_value(name)}
GROUP BY
s.queue
"""
)
)
if not result:
Log.error("not expected")
queue = self.get_or_create_queue(name)
sub_info = t.query(
sql_query(
{"from": SUBSCRIBER, "where": {"eq": {"id": first_row(result).id}}}
)
)
return Subscription(queue=queue, kwargs=first_row(sub_info))
def get_or_create_queue(self, name, block_size_mb=8, kwargs=None):
for q in self.queues:
if q.name == name:
return q
# VERIFY QUEUE EXISTS
with self.db.transaction() as t:
result = t.query(
sql_query({"from": QUEUE, "where": {"eq": {"name": name}}})
)
if not result.data:
id = self.next_id()
t.execute(
sql_insert(
table=QUEUE,
records={
"id": id,
"name": name,
"next_serial": 1,
"block_size_mb": block_size_mb,
"block_start": 1,
"block_end": 1,
},
)
)
t.execute(
sql_insert(
table=SUBSCRIBER,
records={
"id": id,
"queue": id,
"confirm_delay_seconds": 60,
"look_ahead_serial": 1000,
"last_confirmed_serial": 0,
"next_emit_serial": 1,
"last_emit_timestamp": Date.now()
},
)
)
output = Queue(id=id, broker=self, kwargs=kwargs)
else:
kwargs = first_row(result)
output = Queue(broker=self, kwargs=kwargs)
self.queues.append(output)
return output
def _next_serial(self, t):
"""
EXPECTING OPEN TRANSACTION t
"""
# TEST IF THERE ARE MESSAGES TO EMIT
result = t.query(
SQL(f"""
SELECT
s.next_emit_serial
FROM
{SUBSCRIBER} AS s
LEFT JOIN
{QUEUE} AS q ON q.id = s.queue
WHERE
s.id = {quote_value(self.id)} AND
q.next_serial > s.next_emit_serial
"""))
if not result.data:
# NO NEW MESSAGES
return None
next_id = first_row(result).next_emit_serial
t.execute(
sql_update(
SUBSCRIBER,
{
"set": {
"next_emit_serial": next_id + 1
},
"where": {
"eq": {
"id": self.id
}
},
},
))
return next_id
def clean(self):
"""
REMOVE ANY RECORDS THAT ARE NOT NEEDED BY QUEUE OR SUBSCRIBERS
"""
now = Date.now()
# ANY BLOCKS TO FLUSH?
with self.db.transaction() as t:
result = t.query(
SQL(
f"""
SELECT id, block_size_mb, block_start
FROM {QUEUE} AS q
JOIN {BLOCKS} AS b ON b.queue=q.id AND b.serial=q.block_start
WHERE b.last_used < {quote_value(now-Duration(WRITE_INTERVAL))}
"""
)
)
for stale in rows(result):
queue = first(q for q in self.queues if q.id == stale.id)
queue._flush(**stale)
# REMOVE UNREACHABLE MESSAGES
conditions = []
for q in self.queues:
conditions.append(
SQL(f"(queue = {quote_value(q.id)} AND serial IN (")
+ SQL(
f"""
SELECT m.serial
FROM {MESSAGES} AS m
LEFT JOIN {UNCONFIRMED} as u ON u.serial = m.serial
LEFT JOIN {SUBSCRIBER} as s ON s.queue = m.queue and s.id = u.subscriber
LEFT JOIN {QUEUE} as q ON q.id = m.queue and m.serial >= q.block_start
LEFT JOIN {SUBSCRIBER} as la ON
la.queue = m.queue AND
la.last_confirmed_serial < m.serial AND
m.serial < la.next_emit_serial+la.look_ahead_serial
WHERE
m.queue = {q.id} AND
s.id IS NULL AND -- STILL UNCONFIRMED POP
q.id IS NULL AND -- NOT WRITTEN TO S3 YET
la.id IS NULL -- NOT IN LOOK-AHEAD FOR SUBSCRIBER
"""
)
+ SQL("))")
)
with self.db.transaction() as t:
if DEBUG:
result = t.query(
ConcatSQL(
SQL(f"SELECT count(1) AS `count` FROM {MESSAGES} WHERE "),
JoinSQL(SQL_OR, conditions),
)
)
Log.note(
"Delete {{num}} messages from database", num=first_row(result).count
)
t.execute(
ConcatSQL(
SQL(f"DELETE FROM {MESSAGES} WHERE "), JoinSQL(SQL_OR, conditions)
)
)
def pop_text(self):
with self.queue.broker.db.transaction() as t:
# CHECK IF SOME MESSAGES CAN BE RESENT
result = t.query(
SQL(f"""
SELECT
m.serial,
m.content
FROM
{UNCONFIRMED} AS u
LEFT JOIN
{MESSAGES} AS m ON m.serial=u.serial
WHERE
m.queue = {quote_value(self.queue.id)} AND
u.subscriber = {quote_value(self.id)} AND
u.deliver_time <= {quote_value(Date.now().unix - self.confirm_delay_seconds)}
ORDER BY
u.deliver_time
LIMIT
1
"""))
if result.data:
record = first_row(result)
# RECORD IT WAS SENT AGAIN
now = Date.now()
t.execute(
sql_update(
UNCONFIRMED,
{
"set": {
"deliver_time": now
},
"where": {
"eq": {
"subscriber": self.id,
"serial": record.serial
}
},
},
))
t.execute(
sql_update(SUBSCRIBER,
{"set": {
"last_emit_timestamp": now
}}))
return record.serial, record.content
# IS THERE A NEVER-SENT MESSAGE?
serial = self._next_serial(t)
if not serial:
return 0, None
result = t.query(
sql_query({
"select": "content",
"from": MESSAGES,
"where": {
"eq": {
"queue": self.queue.id,
"serial": serial
}
},
}))
if not result.data:
result = t.query(
SQL(f"""
SELECT
serial,
path
FROM
{BLOCKS}
WHERE
queue = {quote_value(self.queue.id)} AND
serial <= {quote_value(serial)}
ORDER BY
serial DESC
LIMIT 1
"""))
if not result.data:
Log.error("not expected")
row = first_row(result)
self.queue.load(path=row.path, start=row.serial)
# RETRY
result = t.query(
sql_query({
"select": "content",
"from": MESSAGES,
"where": {
"eq": {
"queue": self.queue.id,
"serial": serial
}
},
}))
if not result.data:
Log.error("not expected")
content = first_row(result).content
# RECORD IT WAS SENT
now = Date.now()
t.execute(
sql_insert(
UNCONFIRMED,
{
"subscriber": self.id,
"serial": serial,
"deliver_time": now
},
))
t.execute(
sql_update(SUBSCRIBER, {"set": {
"last_emit_timestamp": now
}}))
return serial, content