def responded(self, req_id):
self.clear_timeout(req_id)
self.req_ids.discard(req_id)
workers_by_id.pop(req_id, None)
self.start_time.pop(req_id, None)
if (self.cache_key is None or cached_workers.get(
self.cache_key,
(None, None))[1] is not self) and len(self.req_ids) == 0:
# Drain complete; goodbye!
if self.cache_key is not None:
print("Cache worker for %s drained (version %s)" %
(self.cache_key, self.app_version))
self.proc.terminate()
maybe_quit_if_draining_and_done()
def outbound_done():
self.outbound_ids.pop(id, None)
workers_by_id.pop(id, None)
maybe_quit_if_draining_and_done()
def read_loop(self):
try:
while True:
# marshal.load() takes the GIL, so only do it once we know there's something there to load.
dummy_char = self.proc.stdout.read(1)
if len(dummy_char) == 0:
break
msg = marshal.load(self.proc.stdout)
type = msg.get("type")
id = msg.get("id") or msg.get("requestId")
if type == "CALL" or type == "GET_APP":
self.outbound_ids[msg["id"]] = msg.get('originating-call')
workers_by_id[msg["id"]] = self
else:
if id is None:
if "output" in msg:
# Output from unknown thread? Broadcast it.
print(
"Broadcasting output from unknown thread: %s" %
msg)
for i in self.req_ids:
msg["id"] = i
send_with_header(msg)
else:
print("Discarding invalid message with no ID: %s" %
repr(msg))
continue
if id not in self.req_ids and id not in self.outbound_ids:
print("Discarding invalid message with bogus ID: %s" %
repr(msg))
if type == "CHUNK_HEADER":
print("Discarding binary data chunk")
marshal.load(self.proc.stdout)
continue
try:
if type == "CHUNK_HEADER":
x = marshal.load(self.proc.stdout)
send_with_header(msg, x)
if msg.get("lastChunk"):
self.transmitted_media(msg['requestId'],
msg['mediaId'])
else:
if "response" in msg and self.enable_profiling.get(id):
p = msg.get("profile", None)
msg["profile"] = {
"origin":
"Server (Python)",
"description":
"Downlink dispatch",
"start-time":
float(self.start_time.get(id, 0) * 1000),
"end-time":
float(time.time() * 1000)
}
if p is not None:
msg["profile"]["children"] = [p]
for o in msg.get("objects", []):
if o["path"][0] == "profile":
o["path"].insert(1, "children")
o["path"].insert(2, 0)
if "response" in msg and msg[
'id'] == 'pre-kill-task-state':
# Special case handling for a "clean" kill (where we manage to recover the state)
objects = msg.get('objects', [])
for o in objects:
if 'path' in o and o['path'][0] == 'response':
o['path'][0] = 'taskState'
if 'DataMedia' in o['type']:
msg['objects'] = []
msg['response'] = None
break
self.req_ids.discard('pre-kill-task-state')
send_with_header({
'type': 'NOTIFY_TASK_KILLED',
'id': list(self.req_ids)[0],
'taskState': msg['response'],
'objects': objects
})
self.proc.terminate()
else:
send_with_header(msg)
if "response" in msg or "error" in msg:
#if statsd and (id in self.start_time):
# statsd.timing('Downlink.WorkerLifetime', (time.time()*1000) - self.start_time.get(id, 0)*1000)
self.on_media_complete(msg, lambda: self.responded(id))
except UnicodeError:
send_with_header({
"id": id,
"error": {
"type":
"UnicodeError",
"message":
"This function returned a binary string (not text). If you want to return binary data, use a BlobMedia object instead."
}
})
self.responded(id)
except EOFError:
print(
"EOFError while reading worker stdout. This should not have happened."
)
pass
finally:
for i in self.req_ids:
workers_by_id.pop(i, None)
for i in self.outbound_ids.keys():
workers_by_id.pop(i, None)
rt = self.proc.poll()
if rt is None:
self.proc.terminate()
for _, t in self.timeouts.items():
t.cancel()
if self.cache_key is not None and cached_workers.get(
self.cache_key, (None, None))[1] is self:
cached_workers.pop(self.cache_key, None)
error_id = "".join(
[random.choice('0123456789abcdef') for x in range(10)])
for i in self.req_ids:
if self.timed_out:
message = 'Server code took too long'
type = "anvil.server.TimeoutError"
elif rt == -9:
message = 'Server code execution process was killed. It may have run out of memory: %s' % (
error_id)
type = "anvil.server.ExecutionTerminatedError"
sys.stderr.write(message + " (IDs %s)\n" % i)
sys.stderr.flush()
else:
message = 'Server code exited unexpectedly: %s' % (
error_id)
type = "anvil.server.ExecutionTerminatedError"
sys.stderr.write(message + " (IDs %s)\n" % i)
sys.stderr.flush()
send_with_header({
'id': i,
'error': {
'type': type,
'message': message
}
})
print("Worker terminated for IDs %s (return code %s)" %
(self.req_ids, rt))
maybe_quit_if_draining_and_done()