def serviceDisplay(serviceObject,
instance=None,
objType=None,
queryArgs=None):
if not current_transaction().db().isSubscribedToType(Happy):
raise SubscribeAndRetry(lambda db: db.subscribeToType(Happy))
if instance:
return instance.display(queryArgs)
return Card(
Subscribed(lambda: Text(
"There are %s happy objects <this should not have lessthans>" %
len(Happy.lookupAll()))) +
Expands(
Text("Closed"),
Subscribed(lambda: HappyService.serviceDisplay(serviceObject)))
) + Button("go to google", "http://google.com/") + SubscribedSequence(
lambda: Happy.lookupAll(), lambda h: Button(
"go to the happy", serviceObject.urlForObject(h, x=10))
) + Subscribed(lambda: ButtonGroup([
Button(Octicon("list-unordered"),
lambda: None,
active=lambda: True),
Button(Octicon("terminal"), lambda: None, active=lambda: True),
Button(Octicon("graph"), lambda: None, active=lambda: True)
]).nowrap())
def test_subscribed_html_valid(self):
child = Text("Subscribed Text")
cell = Subscribed(child)
# TODO: does this makes sense?
cell.cells = self.cells
cell.recalculate()
html = cell.contents
self.assertHTMLNotEmpty(html)
self.assertHTMLValid(html)
def test_cells_ensure_subscribed(self):
schema = Schema("core.web.test2")
@schema.define
class Thing2:
k = Indexed(int)
x = int
computed = threading.Event()
def checkThing2s():
ensureSubscribedType(Thing2)
res = Sequence([
Span("Thing(k=%s).x = %s" % (thing.k, thing.x))
for thing in Thing2.lookupAll()
])
computed.set()
return res
self.cells.withRoot(Subscribed(checkThing2s))
self.cells.renderMessages()
self.assertTrue(computed.wait(timeout=5.0))
def test_cells_garbage_collection(self):
# create a cell that subscribes to a specific 'thing', but that
# creates new cells each time, and verify that we reduce our
# cell count, and that we send deletion messages
# subscribes to the set of cells with k=0 and displays something
self.cells.withRoot(
SubscribedSequence(
lambda: Thing.lookupAll(k=0),
lambda thing: Subscribed(lambda: Span("Thing(k=%s).x = %s" %
(thing.k, thing.x)))))
with self.db.transaction():
thing = Thing(x=1, k=0)
for i in range(100):
with self.db.transaction():
thing.k = 1
thing = Thing(x=i, k=0)
for anything in Thing.lookupAll():
anything.x = anything.x + 1
messages = self.cells.renderMessages()
self.assertTrue(
len(self.cells) < 20,
"Have %s cells at pass %s" % (len(self.cells), i))
self.assertTrue(
len(messages) < 20,
"Got %s messages at pass %s" % (len(messages), i))
def test_cells_memory_leak1(self):
cell = Subscribed(lambda: Sequence([
Span("Thing(k=%s).x = %s" % (thing.k, thing.x))
for thing in Thing.lookupAll(k=0)
]))
def workFn(db, cells, iterations=5000):
with db.view():
thing = Thing.lookupAny(k=0)
for counter in range(iterations):
with db.transaction():
thing.delete()
thing = Thing(k=0, x=counter)
cells.renderMessages()
def initFn(db, cells):
with db.transaction():
Thing(k=0, x=0)
cells.renderMessages()
workFn(db, cells, iterations=500)
self.helper_memory_leak(cell, initFn, workFn, 1)
def test_cells_memory_leak2(self):
cell = (
SubscribedSequence(
lambda: Thing.lookupAll(k=0),
lambda thing: Subscribed(lambda: Span("Thing(k=%s).x = %s" %
(thing.k, thing.x)))) +
SubscribedSequence(
lambda: Thing.lookupAll(k=1),
lambda thing: Subscribed(lambda: Span("Thing(k=%s).x = %s" %
(thing.k, thing.x)))))
def workFn(db, cells, iterations=5000):
with db.view():
thing = Thing.lookupAny(k=0)
self.assertTrue(thing)
self.assertTrue(Thing.lookupAny())
for counter in range(iterations):
with db.transaction():
if counter % 3 == 0:
thing.k = 1 - thing.k
thing.delete()
thing = Thing(x=counter, k=0)
self.assertTrue(Thing.lookupAny())
all_things = Thing.lookupAll()
self.assertEqual(len(all_things), 1)
for anything in all_things:
anything.x = anything.x + 1
cells.renderMessages()
def initFn(db, cells):
with db.transaction():
Thing(x=1, k=0)
cells.renderMessages()
workFn(db, cells, iterations=500)
self.helper_memory_leak(cell, initFn, workFn, 1)
def hostsTable():
hosts = Table(colFun=lambda: [
'Connection', 'IsMaster', 'Hostname', 'RAM ALLOCATION',
'CORE ALLOCATION', 'SERVICE COUNT', 'CPU USE', 'RAM USE'
],
rowFun=lambda: sorted(service_schema.ServiceHost.lookupAll(),
key=lambda s: s.hostname),
headerFun=lambda x: x,
rendererFun=lambda s, field: Subscribed(
lambda: hostsTableDataPrep(s, field)),
maxRowsPerPage=50)
return hosts
def serviceDisplay(serviceObject,
instance=None,
objType=None,
queryArgs=None):
ensureSubscribedType(GridValue)
return Grid(
colFun=lambda: list(range(COL_COUNT)),
rowFun=lambda: list(range(ROW_COUNT)),
headerFun=lambda x: x,
rowLabelFun=None,
rendererFun=lambda row, col: Subscribed(
lambda: GridValue.lookupAny(row_and_col=(row, col)).value))
def makeCells(self, queryArgs=None):
queryArgs = queryArgs or {}
rootCell = Subscribed(
lambda: ResearchFrontend.serviceDisplay(
self.serviceObject,
queryArgs=queryArgs
)
).withSerializationContext(self.ser_ctx)
cells = Cells(self.db).withRoot(rootCell)
cells.renderMessages()
return cells
def test_cells_reusable(self):
c1 = Card(Text("HI"))
c2 = Card(Text("HI2"))
slot = Slot(0)
self.cells.withRoot(Subscribed(lambda: c1 if slot.get() else c2))
self.cells.renderMessages()
slot.set(1)
self.cells.renderMessages()
slot.set(0)
self.cells.renderMessages()
self.assertFalse(self.cells.childrenWithExceptions())
def displayAndHeadersForPathAndQueryArgs(path, queryArgs):
if len(path) and path[0] == 'services':
if len(path) == 1:
return view(), []
serviceObj = service_schema.Service.lookupAny(name=path[1])
if serviceObj is None:
return Traceback("Unknown service %s" % path[1]), []
serviceType = serviceObj.instantiateServiceType()
serviceToggles = [
x.withSerializationContext(serviceObj.getSerializationContext())
for x in serviceType.serviceHeaderToggles(serviceObj)
]
if len(path) == 2:
return (Subscribed(lambda: serviceType.serviceDisplay(
serviceObj, queryArgs=queryArgs)).withSerializationContext(
serviceObj.getSerializationContext()), serviceToggles)
typename = path[2]
schemas = serviceObj.findModuleSchemas()
typeObj = None
for s in schemas:
typeObj = s.lookupFullyQualifiedTypeByName(typename)
if typeObj:
break
if typeObj is None:
return Traceback("Can't find fully-qualified type %s" %
typename), []
if len(path) == 3:
return (serviceType.serviceDisplay(
serviceObj, objType=typename,
queryArgs=queryArgs).withSerializationContext(
serviceObj.getSerializationContext()), serviceToggles)
instance = typeObj.fromIdentity(path[3])
return (serviceType.serviceDisplay(
serviceObj, instance=instance,
queryArgs=queryArgs).withSerializationContext(
serviceObj.getSerializationContext()), serviceToggles)
return Traceback("Invalid url path: %s" % path), []
def servicesTableDataPrep(s, field, serviceCounts):
"""Prep data for display in services table.
s : service object
service data dictionary
field : str
serviceCounts : list of ints
"""
if field == 'Service':
data = Clickable(s.name, "/services/" + s.name)
elif field == 'Codebase Status':
data = (
Clickable(
Sequence([Octicon('stop').color('red'),
Span('Unlocked')]), lambda: s.lock())
if s.isUnlocked else Clickable(
Sequence([Octicon('shield').color('green'),
Span('Locked')]), lambda: s.prepare())
if s.isLocked else Clickable(
Sequence([Octicon('shield').color('orange'),
Span('Prepared')]), lambda: s.unlock()))
elif field == 'Codebase':
data = (str(s.codebase) if s.codebase else "")
elif field == 'Module':
data = s.service_module_name
elif field == 'Class':
data = s.service_class_name
elif field == 'Placement':
data = s.placement
elif field == 'Active':
data = Subscribed(
lambda: len(service_schema.ServiceInstance.lookupAll(service=s)))
elif field == 'TargetCount':
data = Dropdown(s.target_count, [(str(ct), __serviceCountSetter(s, ct))
for ct in serviceCounts])
elif field == 'Cores':
data = str(s.coresUsed)
elif field == 'RAM':
data = str(s.gbRamUsed)
elif field == 'Boot Status':
data = (Popover(Octicon("alert"), "Failed",
Traceback(s.lastFailureReason or "<Unknown>"))
if s.isThrottled() else "")
else:
data = ""
return data
def twinned():
data = {
'PointsToShow': {
'timestamp': [1500000000 + x for x in range(1000)],
'y': [numpy.sin(x) for x in range(1000)]
}
}
slot = Slot(None)
p1 = Plot(lambda: data, xySlot=slot)
p2 = Plot(lambda: data, xySlot=slot)
def synchronize():
if slot.get() is not None:
p1.setXRange(*slot.get()[0])
p2.setXRange(*slot.get()[0])
return p1 + p2 + Subscribed(synchronize)
def servicesTable():
serviceCountsChain = chain(range(5), range(10, 100, 10),
range(100, 400, 25), range(400, 1001, 100))
serviceCounts = [val for val in serviceCountsChain]
table = Table(colFun=lambda: [
'Service', 'Codebase Status', 'Codebase', 'Module', 'Class',
'Placement', 'Active', 'TargetCount', 'Cores', 'RAM', 'Boot Status'
],
rowFun=lambda: sorted(service_schema.Service.lookupAll(),
key=lambda s: s.name),
headerFun=lambda x: x,
rendererFun=lambda s, field: Subscribed(
lambda: servicesTableDataPrep(s, field, serviceCounts)),
maxRowsPerPage=50)
return table
def test_cells_subscriptions(self):
self.cells.withRoot(
Subscribed(lambda: Sequence([
Span("Thing(k=%s).x = %s" % (thing.k, thing.x))
for thing in Thing.lookupAll()
])))
self.cells.renderMessages()
with self.db.transaction():
Thing(x=1, k=1)
Thing(x=2, k=2)
self.cells._recalculateCells()
with self.db.transaction():
Thing(x=3, k=3)
# three 'Span', three 'Text', the Sequence, the Subscribed, and a delete
self.assertEqual(len(self.cells.renderMessages()), 9)
def serviceDisplay(serviceObject,
instance=None,
objType=None,
queryArgs=None):
ensureSubscribedType(TextEditor)
toEval = Slot("{'x': [1,2,3,4,5], 'y': [1,5,1,5,1]}")
def onEnter(buffer, selection):
toEval.set(buffer)
TextEditor.lookupAny().code = buffer
def onTextChange(buffer, selection):
if TextEditor.lookupAny() is not None:
TextEditor.lookupAny().code = buffer
ed = CodeEditor(keybindings={'Enter': onEnter},
noScroll=True,
minLines=50,
onTextChange=onTextChange)
def makePlotData():
# data must be a list or dict here, but this is checked/asserted
# down the line in cells. Sending anything that is not a dict/list
# will break the entire plot.
try:
data = ast.literal_eval(toEval.get())
except (AttributeError, SyntaxError):
data = {}
return {'data': data}
def onCodeChange():
if TextEditor.lookupAny() is not None:
if ed.getContents() != TextEditor.lookupAny().code:
ed.setContents(TextEditor.lookupAny().code)
return Columns(ed, Card(
Plot(makePlotData).height("100%").width("100%"))) + Subscribed(
onCodeChange)
def mainBar(display, toggles, current_username, authorized_groups_text):
"""Main header bar.
Parameters:
-----------
display: serviceDisplay object
togges: list of servicesToggles
current_username : text
authorized_groups_text: text
"""
return (HeaderBar([
Subscribed(
lambda: Dropdown("Service", [("All", "/services")] + [
(s.name, "/services/" + s.name)
for s in sorted(service_schema.Service.lookupAll(),
key=lambda s: s.name)
]), )
], toggles, [
LargePendingDownloadDisplay(),
Octicon('person') + Span(current_username),
Span('Authorized Groups: {}'.format(authorized_groups_text)),
Button(Octicon('sign-out'), '/logout')
]) + Main(display))
def serviceDisplay(serviceObject,
instance=None,
objType=None,
queryArgs=None):
ensureSubscribedType(PointsToShow)
ensureSubscribedType(Feigenbaum)
depth = Slot(50)
def twinned():
data = {
'PointsToShow': {
'timestamp': [1500000000 + x for x in range(1000)],
'y': [numpy.sin(x) for x in range(1000)]
}
}
slot = Slot(None)
p1 = Plot(lambda: data, xySlot=slot)
p2 = Plot(lambda: data, xySlot=slot)
def synchronize():
if slot.get() is not None:
p1.setXRange(*slot.get()[0])
p2.setXRange(*slot.get()[0])
return p1 + p2 + Subscribed(synchronize)
return Tabs(
Overlay=Card(
Plot(
lambda: {
'single_array': [1, 2, 3, 1, 2, 3],
'xy': {
'x': [1, 2, 3, 1, 2, 3],
'y': [4, 5, 6, 7, 8, 9]
},
}).width(600).height(400) + Code("HI")),
AGrid=Grid(colFun=lambda: ['A', 'B', 'B'],
rowFun=lambda: ['1', '2', '2'],
headerFun=lambda x: x,
rowLabelFun=None,
rendererFun=lambda row, col: row + col),
ASheet=Sheet(
["A", "B", "C"], 1000000,
lambda rowIx: ["(%s) ts" % rowIx, rowIx, rowIx + 1, rowIx + 2]
).width('calc(100vw - 70px)').height('calc(100vh - 150px)'),
Timestamps=(
Button("Add a point!", GraphDisplayService.addAPoint) +
Card(Plot(
GraphDisplayService.chartData)).width(600).height(400)),
Twinned=Subscribed(twinned),
feigenbaum=(Dropdown("Depth", [
(val, depth.setter(val))
for val in [10, 50, 100, 250, 500, 750, 1000]
]) + Dropdown(
"Polynomial", [1.0, 1.5, 2.0], lambda polyVal: setattr(
Feigenbaum.lookupAny(), 'y', float(polyVal))) + Dropdown(
"Density", list(range(100, 10000, 100)),
lambda polyVal: setattr(Feigenbaum.lookupAny(
), 'density', float(polyVal))) + Card(
Plot(lambda graph: GraphDisplayService.feigenbaum(
graph, depth.get()))).width(600).height(400)))
def mainSocket(self, ws, path):
path = str(path).split("/")
queryArgs = dict(request.args.items())
sessionId = request.cookies.get("session")
# wait for the other socket to close if we were bounced
sleep(.25)
sessionState = self._getSessionState(sessionId)
self._logger.info("entering websocket with path %s", path)
reader = None
isFirstMessage = True
# set up message tracking
timestamps = []
lastDumpTimestamp = time.time()
lastDumpMessages = 0
lastDumpFrames = 0
lastDumpTimeSpentCalculating = 0.0
# set up cells
cells = Cells(self.db)
# reset the session state. There's only one per cells (which is why
# we keep a list of sessions.)
sessionState._reset(cells)
cells = cells.withRoot(
Subscribed(
lambda: self.displayForPathAndQueryArgs(path, queryArgs)),
serialization_context=self.db.serializationContext,
session_state=sessionState)
# large messages (more than frames_per_ack frames) send an ack
# after every frames_per_ackth message
largeMessageAck = gevent.queue.Queue()
reader = Greenlet.spawn(
functools.partial(readThread, ws, cells, largeMessageAck,
self._logger))
self._logger.info("Starting main websocket handler with %s", ws)
while not ws.closed:
t0 = time.time()
try:
# make sure user is authenticated
user = self.login_plugin.load_user(current_user.username)
if not user.is_authenticated:
ws.close()
return
messages = cells.renderMessages()
lastDumpTimeSpentCalculating += time.time() - t0
if isFirstMessage:
self._logger.info("Completed first rendering loop")
isFirstMessage = False
for message in messages:
gevent.socket.wait_write(ws.stream.handler.socket.fileno())
writeJsonMessage(message, ws, largeMessageAck,
self._logger)
lastDumpMessages += 1
lastDumpFrames += 1
# log slow messages
if time.time() - lastDumpTimestamp > 60.0:
self._logger.info(
"In the last %.2f seconds, spent %.2f seconds"
" calculating %s messages over %s frames",
time.time() - lastDumpTimestamp,
lastDumpTimeSpentCalculating, lastDumpMessages,
lastDumpFrames)
lastDumpFrames = 0
lastDumpMessages = 0
lastDumpTimeSpentCalculating = 0
lastDumpTimestamp = time.time()
# tell the browser to execute the postscripts that its built up
writeJsonMessage("postscripts", ws, largeMessageAck,
self._logger)
# request an ACK from the browser before sending any more data
# otherwise it can get overloaded and crash because it can't keep
# up with the data volume
writeJsonMessage("request_ack", ws, largeMessageAck,
self._logger)
ack = largeMessageAck.get()
if ack is StopIteration:
raise Exception("Websocket closed.")
cells.wait()
timestamps.append(time.time())
if len(timestamps) > MAX_FPS:
timestamps = timestamps[-MAX_FPS + 1:]
if (time.time() - timestamps[0]) < 1.0:
sleep(1.0 / MAX_FPS + .001)
except Exception:
self._logger.error("Websocket handler error: %s",
traceback.format_exc())
self.sessionStates[sessionId].append(sessionState)
self._logger.info(
"Returning session state to pool for %s. Have %s",
sessionId, len(self.sessionStates[sessionId]))
if reader:
reader.join()