def scraperentry(eoos, cclient): # local function
scrapereditors = { } # chatname -> (lasttouch, [clientnumbers])
running = False # we could make this an updated member of EditorsOnOneScraper like lasttouch
for usereditor in eoos.usereditormap.values():
cchatname = usereditor.userclients[0].cchatname
clientnumbers = [uclient.clientnumber for uclient in usereditor.userclients]
scrapereditors[cchatname] = (usereditor.userlasttouch, clientnumbers)
running = running or max([ bool(uclient.processrunning) for uclient in usereditor.userclients ])
for uclient in eoos.anonymouseditors:
scrapereditors[uclient.cchatname] = (uclient.clientlasttouch, [uclient.clientnumber])
# diff mode
if cclient:
scraperusercclient = {'chatname':cclient.cchatname, 'userlasttouch':jstime(cclient.clientlasttouch) }
if cclient.cchatname in scrapereditors:
scraperusercclient['present'] = True
scraperusercclient['uclients'] = scrapereditors[cclient.cchatname][1]
else:
scraperusercclient['present'] = False
scraperusers = [ scraperusercclient ]
else:
scraperusers = [ {'chatname':cchatname, 'userlasttouch':jstime(ultc[0]), 'uclients':ultc[1], 'present':True } for cchatname, ultc in scrapereditors.items() ]
return {'scrapername':eoos.scrapername, 'present':True, 'running':running, 'scraperusers':scraperusers, 'scraperlasttouch':jstime(eoos.scraperlasttouch) }
def notifyEditorClients(self, message):
editorstatusdata = { 'message_type':"editorstatus" }
editorstatusdata["nowtime"] = jstime(datetime.datetime.now())
editorstatusdata['earliesteditor'] = jstime(self.scrapersessionbegan)
editorstatusdata["scraperlasttouch"] = jstime(self.scraperlasttouch)
# order by who has first session (and not all draft mode) in order to determin who is the editor
usereditors = self.usereditormap.values()
usereditors.sort(key=lambda x: x.usersessionpriority)
editorstatusdata["loggedinusers"] = [ ]
editorstatusdata["loggedineditors"] = [ ]
editorstatusdata["countclients"] = len(self.anonymouseditors) # so we know what windows are connected to this editor (if any)
for usereditor in usereditors:
editorstatusdata["countclients"] += len(usereditor.userclients)
if usereditor.userclients[-1].savecode_authorized: # as recorded in last client for this user
editorstatusdata["loggedineditors"].append(usereditor.username)
else:
editorstatusdata["loggedinusers"].append(usereditor.username)
editorstatusdata["nanonymouseditors"] = len(self.anonymouseditors)
editorstatusdata["message"] = message
for client in self.anonymouseditors:
editorstatusdata["chatname"] = client.chatname
editorstatusdata["clientnumber"] = client.clientnumber
client.writejson(editorstatusdata);
for usereditor in self.usereditormap.values():
for client in usereditor.userclients:
editorstatusdata["chatname"] = client.chatname
editorstatusdata["clientnumber"] = client.clientnumber
client.writejson(editorstatusdata)
def clientConnectionRegistered(self, client):
# Can't remove from the list if it isn't in there so we need to check
try:
self.connectedclients.remove(client)
except:
logging.error('Failed to remove client %s as it is not in the connectedclient list' % client)
if client.username:
client.chatname = client.userrealname or client.username
elif client.clienttype == "editing":
client.chatname = "Anonymous%d" % self.anonymouscount
self.anonymouscount += 1
else:
client.chatname = "Anonymous"
client.cchatname = "%s|%s" % (client.username, client.chatname)
assert client.clienttype in [
"umlmonitoring", "editing", "rpcrunning",
"scheduledrun", "stimulate_run", "httpget"]
if client.clienttype == "umlmonitoring":
self.umlmonitoringclients.append(client)
elif client.clienttype == 'rpcrunning':
self.rpcrunningclients.append(client)
elif client.clienttype == 'httpget':
self.httpgetclients.append(client)
elif client.clienttype == 'stimulate_run':
self.stimulate_runclients.append(client)
elif client.guid:
if client.guid not in self.guidclientmap:
self.guidclientmap[client.guid] = EditorsOnOneScraper(client.guid, client.scrapername, client.scraperlanguage, client.originalrev)
if client.username in self.guidclientmap[client.guid].usereditormap:
message = "%s opens another window" % client.chatname
else:
message = "%s enters" % client.chatname
self.guidclientmap[client.guid].AddClient(client)
self.guidclientmap[client.guid].notifyEditorClients(message)
else: # draft scraper type (hardcode the output that would have gone with notifyEditorClients
editorstatusdata = {'message_type':"editorstatus", "loggedineditors":[], "loggedinusers":[], "nanonymouseditors":1, "countclients":1, "chatname":client.chatname, "message":"Draft scraper connection" }
editorstatusdata["nowtime"] = jstime(datetime.datetime.now())
editorstatusdata['earliesteditor'] = jstime(client.clientsessionbegan)
editorstatusdata["scraperlasttouch"] = jstime(client.clientlasttouch)
editorstatusdata["clientnumber"] = client.clientnumber
client.writejson(editorstatusdata);
self.draftscraperclients.append(client)
if client.clienttype in ["umlmonitoring", "editing", "rpcrunning"]:
self.notifyMonitoringClients(client)
def gotcontrollerconnectionprotocol(self, controllerconnection):
controllerconnection.srunner = self
self.controllerconnection = controllerconnection
# generate the header element that was generated by dispatcher,
# which should in future be made by the node-controller
msg = { 'message_type':'executionstatus', 'content':'startingrun', 'runID':self.jdata["runid"], 'uml':"Direct to controller",
'rev':self.jdata["rev"], 'chatname':self.client.chatname, 'nowtime':jstime(datetime.datetime.now())}
self.client.writeall(json.dumps(msg))
# send the data into the controller including all the code it should run
sdata = json.dumps(self.jdata)
logger.debug("sending: %s" % str([sdata[:1000]]))
controllerconnection.transport.write('POST /Execute HTTP/1.0\r\n')
controllerconnection.transport.write('Content-Length: %s\r\n' % len(sdata))
controllerconnection.transport.write('Content-Type: text/json\r\n') # text/json???
controllerconnection.transport.write('Connection: close\r\n')
controllerconnection.transport.write("\r\n")
controllerconnection.transport.write(sdata)
def gotcontrollerconnectionprotocol(self, controllerconnection):
controllerconnection.srunner = self
self.controllerconnection = controllerconnection
# generate the header element that was generated by dispatcher,
# which should in future be made by the node-controller
msg = { 'message_type':'executionstatus', 'content':'startingrun', 'runID':self.jdata["runid"], 'uml':"Direct to controller",
'rev':self.jdata["rev"], 'chatname':self.client.chatname, 'nowtime':jstime(datetime.datetime.now())}
self.client.writeall(json.dumps(msg))
# send the data into the controller including all the code it should run
sdata = json.dumps(self.jdata)
logger.debug("sending: %s" % str([sdata[:1000]]))
controllerconnection.transport.write('POST /Execute HTTP/1.0\r\n')
controllerconnection.transport.write('Content-Length: %s\r\n' % len(sdata))
controllerconnection.transport.write('Content-Type: text/json\r\n') # text/json???
controllerconnection.transport.write('Connection: close\r\n')
controllerconnection.transport.write("\r\n")
controllerconnection.transport.write(sdata)
def notifyMonitoringClientsSmallmessage(self, cclient, smallmessage):
if cclient.guid:
umlsavenotification = {'message_type':smallmessage, "scrapername":cclient.scrapername, "cchatname":cclient.cchatname, "nowtime":jstime(datetime.datetime.now()) }
for client in self.umlmonitoringclients:
client.writejson(umlsavenotification)
def notifyMonitoringClients(self, cclient): # cclient is the one whose state has changed (it can be normal editor or a umlmonitoring case)
Dtclients = len(self.connectedclients) + len(self.rpcrunningclients) + len(self.stimulate_runclients) + len(self.httpgetclients) + len(self.umlmonitoringclients) + len(self.draftscraperclients) + sum([eoos.Dcountclients() for eoos in self.guidclientmap.values()])
if len(self.clients) != Dtclients:
logger.error("Miscount of clients %d %d" % (len(self.clients), Dtclients))
# both of these are in the same format and read the same, but umlstatuschanges are shorter
umlstatuschanges = {'message_type':"umlchanges", "nowtime":jstime(datetime.datetime.now()) };
if self.notifiedmaxscheduledscrapers != self.maxscheduledscrapers:
self.notifiedmaxscheduledscrapers = self.maxscheduledscrapers
umlstatuschanges["maxscheduledscrapers"] = self.maxscheduledscrapers
if cclient and cclient.clienttype == "umlmonitoring":
umlstatusdata = {'message_type':"umlstatus", "nowtime":umlstatuschanges["nowtime"]}
umlstatusdata["maxscheduledscrapers"] = self.maxscheduledscrapers
else:
umlstatusdata = None
# the cchatnames are username|chatname, so the javascript has something to handle for cases of "|Anonymous5" vs "username|username"
# handle updates and changes in the set of clients that have the monitoring window open
umlmonitoringusers = { }
for client in self.umlmonitoringclients:
if client.cchatname in umlmonitoringusers:
umlmonitoringusers[client.cchatname] = max(client.clientlasttouch, umlmonitoringusers[client.cchatname])
else:
umlmonitoringusers[client.cchatname] = client.clientlasttouch
#umlmonitoringusers = set([ client.cchatname for client in self.umlmonitoringclients ])
if umlstatusdata:
umlstatusdata["umlmonitoringusers"] = [ {"chatname":chatname, "present":True, "lasttouch":jstime(chatnamelasttouch) } for chatname, chatnamelasttouch in umlmonitoringusers.items() ]
if cclient and cclient.clienttype == "umlmonitoring":
umlstatuschanges["umlmonitoringusers"] = [ {"chatname":cclient.cchatname, "present":(cclient.cchatname in umlmonitoringusers), "lasttouch":jstime(cclient.clientlasttouch) } ]
# rpcrunningclients
if umlstatusdata:
umlstatusdata["rpcrunningclients"] = [ {"clientnumber":client.clientnumber, "present":True, "chatname":client.chatname, "scrapername":client.scrapername, "lasttouch":jstime(client.clientlasttouch)} for client in self.rpcrunningclients ]
if cclient and cclient.clienttype == "rpcrunning":
umlstatuschanges["rpcrunningclients"] = [ {"clientnumber":cclient.clientnumber, "present":(cclient in self.rpcrunningclients), "chatname":cclient.cchatname, "scrapername":cclient.scrapername, "lasttouch":jstime(cclient.clientlasttouch) } ]
# handle draft scraper users and the run states (one for each user, though there may be multiple draft scrapers for them)
draftscraperusers = { } # chatname -> running state
for client in self.draftscraperclients:
draftscraperusers[client.cchatname] = bool(client.processrunning) or draftscraperusers.get(client.cchatname, False)
if umlstatusdata:
umlstatusdata["draftscraperusers"] = [ {"chatname":chatname, "present":True, "running":crunning } for chatname, crunning in draftscraperusers.items() ]
if cclient and not cclient.clienttype == "umlmonitoring" and not cclient.guid:
umlstatuschanges["draftscraperusers"] = [ { "chatname":cclient.cchatname, "present":(cclient.cchatname in draftscraperusers), "running":draftscraperusers.get(cclient.cchatname, False) } ]
# the complexity here reflects the complexity of the structure. the running flag could be set on any one of the clients
def scraperentry(eoos, cclient): # local function
scrapereditors = { } # chatname -> (lasttouch, [clientnumbers])
running = False # we could make this an updated member of EditorsOnOneScraper like lasttouch
for usereditor in eoos.usereditormap.values():
cchatname = usereditor.userclients[0].cchatname
clientnumbers = [uclient.clientnumber for uclient in usereditor.userclients]
scrapereditors[cchatname] = (usereditor.userlasttouch, clientnumbers)
running = running or max([ bool(uclient.processrunning) for uclient in usereditor.userclients ])
for uclient in eoos.anonymouseditors:
scrapereditors[uclient.cchatname] = (uclient.clientlasttouch, [uclient.clientnumber])
# diff mode
if cclient:
scraperusercclient = {'chatname':cclient.cchatname, 'userlasttouch':jstime(cclient.clientlasttouch) }
if cclient.cchatname in scrapereditors:
scraperusercclient['present'] = True
scraperusercclient['uclients'] = scrapereditors[cclient.cchatname][1]
else:
scraperusercclient['present'] = False
scraperusers = [ scraperusercclient ]
else:
scraperusers = [ {'chatname':cchatname, 'userlasttouch':jstime(ultc[0]), 'uclients':ultc[1], 'present':True } for cchatname, ultc in scrapereditors.items() ]
return {'scrapername':eoos.scrapername, 'present':True, 'running':running, 'scraperusers':scraperusers, 'scraperlasttouch':jstime(eoos.scraperlasttouch) }
if umlstatusdata:
umlstatusdata["scraperentries"] = [ ]
for eoos in self.guidclientmap.values():
umlstatusdata["scraperentries"].append(scraperentry(eoos, None))
if cclient and cclient.clienttype in ["editing", "scheduledrun"] and cclient.guid:
if cclient.guid in self.guidclientmap:
umlstatuschanges["scraperentries"] = [ scraperentry(self.guidclientmap[cclient.guid], cclient) ]
else:
umlstatuschanges["scraperentries"] = [ { 'scrapername':cclient.scrapername, 'present':False, 'running':False, 'scraperusers':[ ] } ]
# send the status to the target and updates to everyone else who is monitoring
#print "\numlstatus", umlstatusdata
# new monitoring client
if cclient and cclient.clienttype == "umlmonitoring":
cclient.writejson(umlstatusdata)
# send only updates to current clients
for client in self.umlmonitoringclients:
if client != cclient:
client.writejson(umlstatuschanges)
def clientcommand(self, command, parsed_data):
if command != 'typing':
logger.debug("command %s client# %d" % (command, self.clientnumber))
# update the lasttouch values on associated aggregations
if command != 'automode' and self.clienttype == "editing":
self.clientlasttouch = datetime.datetime.now()
if self.guid and self.username:
assert self.username in self.guidclienteditors.usereditormap
self.guidclienteditors.usereditormap[self.username].userlasttouch = self.clientlasttouch
self.guidclienteditors.scraperlasttouch = self.clientlasttouch
# data uploaded when a new connection is made from the editor
if command == 'connection_open':
self.lconnectionopen(parsed_data)
# finds the corresponding client and presses the run button on it
# receives a single record through the pipeline
elif command == 'stimulate_run':
self.clienttype = "stimulate_run"
self.factory.clientConnectionRegistered(self)
scrapername = parsed_data["scrapername"]
guid = parsed_data["guid"]
assert guid
username = parsed_data["username"]
clientnumber = parsed_data["clientnumber"]
client = None
eoos = self.factory.guidclientmap.get(guid)
if eoos:
usereditor = eoos.usereditormap.get(username)
if usereditor:
for lclient in usereditor.userclients:
if lclient.clientnumber == clientnumber:
client = lclient
if parsed_data.get('django_key') != djangokey:
logger.error("djangokey_mismatch")
self.writejson({'status':'twister djangokey mismatch'})
if client:
client.writejson({"message_type":"console", "content":"twister djangokey mismatch"})
client.writejson({'message_type':'executionstatus', 'content':'runfinished'})
client = None
if client:
logger.info("stimulate on : %s %s client# %d" % (client.cchatname, client.scrapername, client.clientnumber))
assert client.clienttype == "editing" and client.guid
if not client.processrunning:
client.runcode(parsed_data)
self.writejson({"status":"run started"})
else:
client.writejson({"message_type":"console", "content":"client already running"})
self.writejson({"status":"client already running"})
else:
parsed_data.pop("code", None) # shorten the log message
logger.warning("client not found %s" % parsed_data)
self.writejson({"status":"client not found"})
self.transport.loseConnection()
elif command == 'rpcrun':
self.username = parsed_data.get('username', '')
self.userrealname = parsed_data.get('userrealname', self.username)
self.scrapername = parsed_data.get('scrapername', '')
self.scraperlanguage = parsed_data.get('language', '')
self.guid = parsed_data.get("guid", '')
if parsed_data.get('django_key') == djangokey:
self.clienttype = "rpcrunning"
logger.info("connection open %s: %s %s client# %d" % (self.clienttype, self.username, self.scrapername, self.clientnumber))
self.factory.clientConnectionRegistered(self)
self.runcode(parsed_data)
# termination is by the calling function when it receives an executionstatus runfinished message
else:
logger.error("djangokey_mismatch")
self.writejson({'status':'twister djangokey mismatch'})
self.transport.loseConnection()
elif command == 'saved':
line = json.dumps({'message_type' : "saved", 'chatname' : self.chatname})
otherline = json.dumps({'message_type' : "othersaved", 'chatname' : self.chatname})
self.guidclienteditors.rev = parsed_data["rev"]
self.guidclienteditors.chainpatchnumber = 0
self.writeall(line, otherline)
self.factory.notifyMonitoringClientsSmallmessage(self, "savenote")
# should record the rev and chainpatchnumber so when we join to this scraper we know
elif command == 'typing':
logger.debug("command %s client# %d insertlinenumber %s" % (command, self.clientnumber, parsed_data.get("insertlinenumber")))
jline = {'message_type' : "typing", 'content' : "%s typing" % self.chatname}
jotherline = parsed_data.copy()
jotherline.pop("command")
jotherline["message_type"] = "othertyping"
jotherline["content"] = jline["content"]
self.guidclienteditors.chainpatchnumber = parsed_data.get("chainpatchnumber")
self.writeall(json.dumps(jline), json.dumps(jotherline))
self.factory.notifyMonitoringClientsSmallmessage(self, "typingnote")
# this one only applies to draft scrapers when you click run
elif command == 'run':
if self.processrunning:
self.writejson({'content':"Already running! (shouldn't happen)", 'message_type':'console'});
return
if self.username:
if self.automode == 'autoload':
self.writejson({'content':"Not supposed to run! "+self.automode, 'message_type':'console'});
return
if parsed_data.get('guid'):
self.writejson({'content':"scraper run can only be done through stimulate_run method", 'message_type':'console'});
return
logger.info("about to run code %s" % str(parsed_data)[:100])
self.runcode(parsed_data)
elif command == "umlcontrol":
# allows monitoring client to remotely kill processes
if self.clienttype != "umlmonitoring":
logger.error("umlcontrol called by non-monitoring client")
return
logger.info("umlcontrol %s" % ([parsed_data]))
subcommand = parsed_data.get("subcommand")
if subcommand == "killscraper":
scrapername = parsed_data["scrapername"]
for eoos in self.factory.guidclientmap.values(): # would be better if it was by scrapername instead of guid
if eoos.scrapername == scrapername:
for usereditor in eoos.usereditormap.values():
for uclient in usereditor.userclients:
if uclient.processrunning:
logger.info("umlcontrol killing run on client# %d %s" % (uclient.clientnumber, scrapername))
uclient.kill_run()
if subcommand == "killallscheduled":
for client in self.factory.scheduledrunners.values():
if client.processrunning:
logger.info("umlcontrol killing run on client# %d %s" % (client.clientnumber, client.scrapername))
client.kill_run()
else:
logger.info("umlcontrol client# %d %s wasn't running" % (client.clientnumber, client.scrapername))
if "maxscheduledscrapers" in parsed_data:
self.factory.maxscheduledscrapers = parsed_data["maxscheduledscrapers"]
self.factory.notifyMonitoringClients(None)
elif command == "kill":
if self.processrunning:
self.kill_run()
# allows the killing of a process in another open window by same user
elif self.username and self.guid:
usereditor = self.guidclienteditors.usereditormap[self.username]
for client in usereditor.userclients:
if client.processrunning:
client.kill_run()
elif command == 'chat':
line = json.dumps({'message_type':'chat', 'chatname':self.chatname, 'message':parsed_data.get('text'), 'nowtime':jstime(datetime.datetime.now()) })
self.writeall(line)
elif command == 'requesteditcontrol':
for usereditor in self.guidclienteditors.usereditormap.values():
for client in usereditor.userclients:
if client.automode == 'autosave':
client.writejson({'message_type':'requestededitcontrol', "username":self.username})
elif command == 'giveselrange':
self.writeall(None, json.dumps({'message_type':'giveselrange', 'selrange':parsed_data.get('selrange'), 'chatname':self.chatname }))
elif command == 'automode':
automode = parsed_data.get('automode')
if automode == self.automode:
return
if not self.username:
self.automode = automode
self.factory.notifyMonitoringClients(self)
return
usereditor = self.guidclienteditors.usereditormap[self.username]
# self-demote to autoload mode while choosing to promote a particular person to editing mode
if automode == 'autoload':
selectednexteditor = parsed_data.get('selectednexteditor')
if selectednexteditor and selectednexteditor in self.guidclienteditors.usereditormap:
assert self.guidclienteditors.usereditormap[selectednexteditor].usersessionpriority >= usereditor.usersessionpriority
self.guidclienteditors.usereditormap[selectednexteditor].usersessionpriority = usereditor.usersessionpriority
usereditor.usersessionpriority = self.guidclienteditors.usersessionprioritynext
self.guidclienteditors.usersessionprioritynext += 1
self.automode = automode
self.guidclienteditors.notifyEditorClients("")
self.factory.notifyMonitoringClients(self)
# this message helps kill it better and killing it from the browser end
elif command == 'loseconnection':
# Suspect it is possible in some cases that the client sends this command, and before
# we have had a chance to close the connection from here, the client has already gone.
# To cover this case let's handle the exception here and log that loseConnection failed
try:
self.transport.loseConnection()
except:
logger.debug('Closing connection on already closed connection failed')