def setup(self):
if not self.db.about(ABOUT_TABLE):
with self.db.transaction() as t:
t.execute(
sql_create(ABOUT_TABLE, {"version": "TEXT", "next_id": "INTEGER"})
)
t.execute(sql_insert(ABOUT_TABLE, {"version": "1.0", "next_id": 1000}))
t.execute(sql_create(DIGITS_TABLE, {"value": "INTEGER"}))
t.execute(sql_insert(DIGITS_TABLE, [{"value": i} for i in range(10)]))
def get_or_create_queue(self, name, block_size_mb=8, kwargs=None):
for q in self.queues:
if q.name == name:
return q
# VERIFY QUEUE EXISTS
with self.db.transaction() as t:
result = t.query(
sql_query({"from": QUEUE, "where": {"eq": {"name": name}}})
)
if not result.data:
id = self.next_id()
t.execute(
sql_insert(
table=QUEUE,
records={
"id": id,
"name": name,
"next_serial": 1,
"block_size_mb": block_size_mb,
"block_start": 1,
"block_end": 1,
},
)
)
t.execute(
sql_insert(
table=SUBSCRIBER,
records={
"id": id,
"queue": id,
"confirm_delay_seconds": 60,
"look_ahead_serial": 1000,
"last_confirmed_serial": 0,
"next_emit_serial": 1,
"last_emit_timestamp": Date.now()
},
)
)
output = Queue(id=id, broker=self, kwargs=kwargs)
else:
kwargs = first_row(result)
output = Queue(broker=self, kwargs=kwargs)
self.queues.append(output)
return output
def replay(
self,
name,
confirm_delay_seconds=60,
next_emit_serial=1,
look_ahead_serial=1000
):
"""
A SUBSCRIBER FOR REVIEWING THE QUEUE CONTENTS, IN ORDER
"""
queue = self.get_or_create_queue(name)
id = self.next_id()
with self.db.transaction() as t:
t.execute(
sql_insert(
table=SUBSCRIBER,
records={
"id": id,
"queue": queue.id,
"last_emit_timestamp": Date.now(),
"confirm_delay_seconds": confirm_delay_seconds,
"last_confirmed_serial": next_emit_serial - 1,
"next_emit_serial": next_emit_serial,
"look_ahead_serial": look_ahead_serial,
},
)
)
return Subscription(
id=id, queue=queue, confirm_delay_seconds=confirm_delay_seconds
)
def device_register(self, path=None):
"""
EXPECTING A SIGNED REGISTRATION REQUEST
RETURN JSON WITH url FOR LOGIN
"""
now = Date.now()
expires = now + parse(self.device.register.session['max-age'])
request_body = request.get_data()
signed = json2value(request_body.decode("utf8"))
command = json2value(base642bytes(signed.data).decode("utf8"))
session.public_key = command.public_key
rsa_crypto.verify(signed, session.public_key)
self.session_manager.create_session(session)
session.expires = expires.unix
session.state = bytes2base64URL(crypto.bytes(32))
with self.device.db.transaction() as t:
t.execute(
sql_insert(
self.device.table,
{
"state": session.state,
"session_id": session.session_id
},
))
body = value2json(
Data(
session_id=session.session_id,
interval="5second",
expires=session.expires,
url=URL(
self.device.home,
path=self.device.endpoints.login,
query={"state": session.state},
),
))
response = Response(body,
headers={"Content-Type": mimetype.JSON},
status=200)
response.set_cookie(self.device.register.session.name,
session.session_id,
path=self.device.login.session.path,
domain=self.device.login.session.domain,
expires=expires.format(RFC1123),
secure=self.device.login.session.secure,
httponly=self.device.login.session.httponly)
return response
def get_tuids(self, branch, revision, files):
"""
GET TUIDS FROM ENDPOINT, AND STORE IN DB
:param branch: BRANCH TO FIND THE REVISION/FILE
:param revision: THE REVISION NUNMBER
:param files: THE FULL PATHS TO THE FILES
:return: MAP FROM FILENAME TO TUID LIST
"""
# SCRUB INPUTS
revision = revision[:12]
files = [file.lstrip('/') for file in files]
with Timer(
"ask tuid service for {{num}} files at {{revision|left(12)}}",
{
"num": len(files),
"revision": revision
},
silent=not DEBUG or not self.enabled):
response = self.db.query(
"SELECT file, tuids FROM tuid WHERE revision=" +
quote_value(revision) + " AND file IN " + quote_list(files))
found = {file: json2value(tuids) for file, tuids in response.data}
try:
remaining = set(files) - set(found.keys())
new_response = None
if remaining:
request = wrap({
"from": "files",
"where": {
"and": [{
"eq": {
"revision": revision
}
}, {
"in": {
"path": remaining
}
}, {
"eq": {
"branch": branch
}
}]
},
"branch": branch,
"meta": {
"format": "list",
"request_time": Date.now()
}
})
if self.push_queue is not None:
if DEBUG:
Log.note(
"record tuid request to SQS: {{timestamp}}",
timestamp=request.meta.request_time)
self.push_queue.add(request)
else:
if DEBUG:
Log.note("no recorded tuid request")
if not self.enabled:
return found
new_response = http.post_json(self.endpoint,
json=request,
timeout=self.timeout)
if new_response.data and any(r.tuids
for r in new_response.data):
try:
with self.db.transaction() as transaction:
command = sql_insert(
"tuid", [{
"revision": revision,
"file": r.path,
"tuids": value2json(r.tuids)
} for r in new_response.data
if r.tuids != None])
transaction.execute(command)
except Exception as e:
Log.error("can not insert {{data|json}}",
data=new_response.data,
cause=e)
self.num_bad_requests = 0
found.update(
{r.path: r.tuids
for r in new_response.data} if new_response else {})
return found
except Exception as e:
self.num_bad_requests += 1
if self.enabled:
if "502 Bad Gateway" in e:
self.enabled = False
Log.alert(
"TUID service has problems (502 Bad Gateway)",
cause=e)
elif self.num_bad_requests >= MAX_BAD_REQUESTS:
self.enabled = False
Log.alert(
"TUID service has problems (given up trying to use it)",
cause=e)
else:
Log.alert("TUID service has problems.", cause=e)
Till(seconds=SLEEP_ON_ERROR).wait()
return found
def pop_text(self):
with self.queue.broker.db.transaction() as t:
# CHECK IF SOME MESSAGES CAN BE RESENT
result = t.query(
SQL(f"""
SELECT
m.serial,
m.content
FROM
{UNCONFIRMED} AS u
LEFT JOIN
{MESSAGES} AS m ON m.serial=u.serial
WHERE
m.queue = {quote_value(self.queue.id)} AND
u.subscriber = {quote_value(self.id)} AND
u.deliver_time <= {quote_value(Date.now().unix - self.confirm_delay_seconds)}
ORDER BY
u.deliver_time
LIMIT
1
"""))
if result.data:
record = first_row(result)
# RECORD IT WAS SENT AGAIN
now = Date.now()
t.execute(
sql_update(
UNCONFIRMED,
{
"set": {
"deliver_time": now
},
"where": {
"eq": {
"subscriber": self.id,
"serial": record.serial
}
},
},
))
t.execute(
sql_update(SUBSCRIBER,
{"set": {
"last_emit_timestamp": now
}}))
return record.serial, record.content
# IS THERE A NEVER-SENT MESSAGE?
serial = self._next_serial(t)
if not serial:
return 0, None
result = t.query(
sql_query({
"select": "content",
"from": MESSAGES,
"where": {
"eq": {
"queue": self.queue.id,
"serial": serial
}
},
}))
if not result.data:
result = t.query(
SQL(f"""
SELECT
serial,
path
FROM
{BLOCKS}
WHERE
queue = {quote_value(self.queue.id)} AND
serial <= {quote_value(serial)}
ORDER BY
serial DESC
LIMIT 1
"""))
if not result.data:
Log.error("not expected")
row = first_row(result)
self.queue.load(path=row.path, start=row.serial)
# RETRY
result = t.query(
sql_query({
"select": "content",
"from": MESSAGES,
"where": {
"eq": {
"queue": self.queue.id,
"serial": serial
}
},
}))
if not result.data:
Log.error("not expected")
content = first_row(result).content
# RECORD IT WAS SENT
now = Date.now()
t.execute(
sql_insert(
UNCONFIRMED,
{
"subscriber": self.id,
"serial": serial,
"deliver_time": now
},
))
t.execute(
sql_update(SUBSCRIBER, {"set": {
"last_emit_timestamp": now
}}))
return serial, content