def send(self, msg, bindata=None):
id = msg.get("id")
if msg.get("type") in [
"CALL", "CALL_WITH_APP", "GET_TASK_STATE", "LAUNCH_REPL",
"REPL_COMMAND", "TERMINATE_REPL"
]:
# It's a new request! Start the timeout
#print ("Setting timeout and routing for new request ID %s" % id)
workers_by_id[id] = self
if msg.get("enable-profiling"):
self.enable_profiling[id] = True
self.start_time[id] = time.time()
self.req_ids.add(id)
if msg["type"] != "REPL_COMMAND":
self.set_timeout(id)
elif msg.get("type") == "REPL_KEEPALIVE":
self.clear_timeout(msg["repl"])
self.set_timeout(msg["repl"])
send_with_header({"id": id, "response": None})
return
self.to_worker.send(msg, bindata)
def outbound_done():
self.outbound_ids.pop(id, None)
workers_by_id.pop(id, None)
maybe_quit_if_draining_and_done()
if "response" in msg or "error" in msg:
self.on_media_complete(msg, outbound_done)
def _hard_kill_background_task(self):
try:
self.req_ids.discard('pre-kill-task-state')
send_with_header({
'type': 'NOTIFY_TASK_KILLED',
'id': list(self.req_ids)[0]
})
finally:
self.hard_timeout()
def _hard_kill_background_task(self):
print("TIMEOUT KILLING BACKGROUND TASK %s" % self.req_ids)
try:
self.req_ids.discard('pre-kill-task-state')
send_with_header({
'type': 'NOTIFY_TASK_KILLED',
'id': list(self.req_ids)[0]
})
finally:
self.hard_timeout()
def _hard_kill_background_task(self):
print("HARD KILL BACKGROUND TASK %s" % self.initial_req_id)
try:
self.req_ids.discard('pre-kill-task-state')
send_with_header({
'type': 'NOTIFY_TASK_KILLED',
'id': self.initial_req_id
})
finally:
self.hard_timeout()
def launch(data):
type = data["type"] if 'type' in data else None
id = data["id"] if 'id' in data else None
print("Calling function '%s' for app '%s' (ID %s)" % (data.get(
"command", "<LiveObjectCall>"), data.get("app-id", "<unknown>"), id))
persist_key = data.get("persist-key")
start_time = time.time()
is_background_task = (type == "LAUNCH_BACKGROUND_WITH_APP")
cache_key = None
worker = None
version = None
supplied_version = data.get("app-version")
if CAN_PERSIST and not is_background_task and persist_key is not None and "app-id" in data and supplied_version is not None:
cache_key = repr((data["app-id"], persist_key))
#print("Attempt persistence: %s" % cache_key)
version, worker = cached_workers.get(cache_key, (None, None))
#print("Version %s:\n%s\nvs\n%s" % (("MATCH" if version==supplied_version else "MISMATCH"), version, supplied_version))
if data['command'] == "anvil.private.pdf.get_component" and not is_background_task:
wid = data['args'][0][0]
worker = workers_by_id.get(wid)
if worker is None:
send_with_header({
'id': id,
'error': {
'message':
"No component worker found for print call '%s'" % wid
}
})
return
elif worker is None or version != supplied_version:
worker = Worker(id,
data.get("enable-profiling", False),
data.get("app-id", "<unknown>"),
cache_key=cache_key,
app_version=supplied_version,
set_timeout=not is_background_task)
worker.start_time[id] = start_time
worker.send(data)
def send(self, msg, bin=False):
if not bin:
id = msg.get("id")
if msg.get("type") in [
"CALL", "CALL_WITH_APP", "GET_TASK_STATE", "LAUNCH_REPL",
"REPL_COMMAND", "TERMINATE_REPL"
]:
# It's a new request! Start the timeout
#print ("Setting timeout and routing for new request ID %s" % id)
workers_by_id[id] = self
if msg.get("enable-profiling"):
self.enable_profiling[id] = True
self.start_time[id] = time.time()
self.req_ids.add(id)
if msg["type"] != "REPL_COMMAND":
self.set_timeout(id)
elif msg.get("type") == "REPL_KEEPALIVE":
self.clear_timeout(msg["repl"])
self.set_timeout(msg["repl"])
send_with_header({"id": id, "response": None})
return
# A horrid hack - a one-char "activation" that's not marshalled, because marshal holds the GIL and Windows doesn't support select() on pipes and urrrggghhh
d = b"X" + marshal.dumps(msg)
else:
d = marshal.dumps(msg)
self.proc.stdin.write(d)
self.proc.stdin.flush()
if not bin:
def outbound_done():
self.outbound_ids.pop(id, None)
workers_by_id.pop(id, None)
maybe_quit_if_draining_and_done()
if "response" in msg or "error" in msg:
self.on_media_complete(msg, outbound_done)
def launch(data):
type = data.get("type")
id = data.get("id")
persist_key = data.get("persist-key")
start_time = time.time()
is_background_task = (type in [
"LAUNCH_BACKGROUND", "LAUNCH_BACKGROUND_WITH_APP"
])
is_repl_launch = type == "LAUNCH_REPL"
cache_key = None
worker = None
version = None
supplied_version = data.get("app-version")
print("%s '%s' for app '%s' (ID %s)" %
("Launching REPL" if is_repl_launch else
"Launching BG task" if is_background_task else "Calling function",
data.get("command", "<no func>"), data.get("app-id",
"<unknown>"), id))
if CAN_PERSIST and not is_background_task and persist_key is not None and "app-id" in data and supplied_version is not None:
cache_key = repr((data["app-id"], persist_key))
#print("Attempt persistence: %s" % cache_key)
version, worker = cached_workers.get(cache_key, (None, None))
#print("Version %s:\n%s\nvs\n%s" % (("MATCH" if version==supplied_version else "MISMATCH"), version, supplied_version))
if data.get(
'command'
) == "anvil.private.pdf.get_component" and not is_background_task:
wid = data['args'][0][0]
worker = workers_by_id.get(wid)
if worker is None:
send_with_header({
'id': id,
'error': {
'message':
"No component worker found for print call '%s'" % wid
}
})
return
elif worker is None or version != supplied_version:
worker = Worker(
id,
data.get("enable-profiling", False),
data.get("app-id", "<unknown>"),
cache_key=cache_key,
app_version=supplied_version,
set_timeout=not is_background_task and not is_repl_launch,
task_info={
"app_id":
data["app-id"],
"type":
"repl"
if is_repl_launch else "background_task" if is_background_task
else "persistent_worker" if cache_key else "server_call",
"task":
data.get("command"),
"persist": {
"key": persist_key,
"version": supplied_version
} if cache_key else None,
})
worker.send(data)
def read_loop(self):
try:
while True:
# marshal.load() takes the GIL, so only do it once we know there's something there to load.
dummy_char = self.proc.stdout.read(1)
if len(dummy_char) == 0:
break
msg = marshal.load(self.proc.stdout)
type = msg.get("type")
id = msg.get("id") or msg.get("requestId")
if type == "CALL" or type == "GET_APP":
self.outbound_ids[msg["id"]] = msg.get('originating-call')
workers_by_id[msg["id"]] = self
else:
if id is None:
if "output" in msg:
# Output from unknown thread? Broadcast it.
print(
"Broadcasting output from unknown thread: %s" %
msg)
for i in self.req_ids:
msg["id"] = i
send_with_header(msg)
else:
print("Discarding invalid message with no ID: %s" %
repr(msg))
continue
if id not in self.req_ids and id not in self.outbound_ids:
print("Discarding invalid message with bogus ID: %s" %
repr(msg))
if type == "CHUNK_HEADER":
print("Discarding binary data chunk")
marshal.load(self.proc.stdout)
continue
try:
if type == "CHUNK_HEADER":
x = marshal.load(self.proc.stdout)
send_with_header(msg, x)
if msg.get("lastChunk"):
self.transmitted_media(msg['requestId'],
msg['mediaId'])
else:
if "response" in msg and self.enable_profiling.get(id):
p = msg.get("profile", None)
msg["profile"] = {
"origin":
"Server (Python)",
"description":
"Downlink dispatch",
"start-time":
float(self.start_time.get(id, 0) * 1000),
"end-time":
float(time.time() * 1000)
}
if p is not None:
msg["profile"]["children"] = [p]
for o in msg.get("objects", []):
if o["path"][0] == "profile":
o["path"].insert(1, "children")
o["path"].insert(2, 0)
if "response" in msg and msg[
'id'] == 'pre-kill-task-state':
# Special case handling for a "clean" kill (where we manage to recover the state)
objects = msg.get('objects', [])
for o in objects:
if 'path' in o and o['path'][0] == 'response':
o['path'][0] = 'taskState'
if 'DataMedia' in o['type']:
msg['objects'] = []
msg['response'] = None
break
self.req_ids.discard('pre-kill-task-state')
send_with_header({
'type': 'NOTIFY_TASK_KILLED',
'id': list(self.req_ids)[0],
'taskState': msg['response'],
'objects': objects
})
self.proc.terminate()
else:
send_with_header(msg)
if "response" in msg or "error" in msg:
#if statsd and (id in self.start_time):
# statsd.timing('Downlink.WorkerLifetime', (time.time()*1000) - self.start_time.get(id, 0)*1000)
self.on_media_complete(msg, lambda: self.responded(id))
except UnicodeError:
send_with_header({
"id": id,
"error": {
"type":
"UnicodeError",
"message":
"This function returned a binary string (not text). If you want to return binary data, use a BlobMedia object instead."
}
})
self.responded(id)
except EOFError:
print(
"EOFError while reading worker stdout. This should not have happened."
)
pass
finally:
for i in self.req_ids:
workers_by_id.pop(i, None)
for i in self.outbound_ids.keys():
workers_by_id.pop(i, None)
rt = self.proc.poll()
if rt is None:
self.proc.terminate()
for _, t in self.timeouts.items():
t.cancel()
if self.cache_key is not None and cached_workers.get(
self.cache_key, (None, None))[1] is self:
cached_workers.pop(self.cache_key, None)
error_id = "".join(
[random.choice('0123456789abcdef') for x in range(10)])
for i in self.req_ids:
if self.timed_out:
message = 'Server code took too long'
type = "anvil.server.TimeoutError"
elif rt == -9:
message = 'Server code execution process was killed. It may have run out of memory: %s' % (
error_id)
type = "anvil.server.ExecutionTerminatedError"
sys.stderr.write(message + " (IDs %s)\n" % i)
sys.stderr.flush()
else:
message = 'Server code exited unexpectedly: %s' % (
error_id)
type = "anvil.server.ExecutionTerminatedError"
sys.stderr.write(message + " (IDs %s)\n" % i)
sys.stderr.flush()
send_with_header({
'id': i,
'error': {
'type': type,
'message': message
}
})
print("Worker terminated for IDs %s (return code %s)" %
(self.req_ids, rt))
maybe_quit_if_draining_and_done()