def outReceived(self, data):
logger.debug("spawnrunner received for client# %d %s" % (self.client.clientnumber, data[:180]))
# although the client can parse the records itself, it is necessary to split them up here correctly so that this code can insert its own records into the stream.
lines = [ ]
spldata = data.split("\n")
self.lbuffer.append(spldata.pop(0))
while spldata:
lines.append("".join(self.lbuffer))
self.lbuffer = [ spldata.pop(0) ] # next one in
for line in lines:
# strip out the httpheaders that come back at the start of a node connection
if not self.httpheadersdone:
if re.match("HTTP/", line):
assert not self.httpheaders
continue
if line == "\r":
self.httpheadersdone = True
continue
mheader = re.match("(.*?):\s*(.*)\r", line)
if not mheader:
logger.error("Bad header: "+str([line]))
else:
self.httpheaders.append((mheader.group(1), mheader.group(2)))
continue
logger.info("Received and will write: "+str([line]))
self.client.writeall(line)
if self.runobjectmaker:
self.runobjectmaker.receiveline(line)
def outReceived(self, data):
logger.debug("spawnrunner received for client# %d %s" % (self.client.clientnumber, data[:180]))
# although the client can parse the records itself, it is necessary to split them up here correctly so that this code can insert its own records into the stream.
lines = [ ]
spldata = data.split("\n")
self.lbuffer.append(spldata.pop(0))
while spldata:
lines.append("".join(self.lbuffer))
self.lbuffer = [ spldata.pop(0) ] # next one in
for line in lines:
# strip out the httpheaders that come back at the start of a node connection
if not self.httpheadersdone:
if re.match("HTTP/", line):
assert not self.httpheaders
continue
if line == "\r":
self.httpheadersdone = True
continue
mheader = re.match("(.*?):\s*(.*)\r", line)
if not mheader:
logger.error("Bad header: "+str([line]))
else:
self.httpheaders.append((mheader.group(1), mheader.group(2)))
continue
logger.debug("Received and will write: "+str([line]))
self.client.writeall(line)
if self.runobjectmaker:
self.runobjectmaker.receiveline(line)
def notifyMonitoringClients(self, cclient): # cclient is the one whose state has changed (it can be normal editor or a umlmonitoring case)
Dtclients = len(self.connectedclients) + len(self.rpcrunningclients) + len(self.stimulate_runclients) + len(self.httpgetclients) + len(self.umlmonitoringclients) + len(self.draftscraperclients) + sum([eoos.Dcountclients() for eoos in self.guidclientmap.values()])
if len(self.clients) != Dtclients:
logger.error("Miscount of clients %d %d" % (len(self.clients), Dtclients))
# both of these are in the same format and read the same, but umlstatuschanges are shorter
umlstatuschanges = {'message_type':"umlchanges", "nowtime":jstime(datetime.datetime.now()) };
if self.notifiedmaxscheduledscrapers != self.maxscheduledscrapers:
self.notifiedmaxscheduledscrapers = self.maxscheduledscrapers
umlstatuschanges["maxscheduledscrapers"] = self.maxscheduledscrapers
if cclient and cclient.clienttype == "umlmonitoring":
umlstatusdata = {'message_type':"umlstatus", "nowtime":umlstatuschanges["nowtime"]}
umlstatusdata["maxscheduledscrapers"] = self.maxscheduledscrapers
else:
umlstatusdata = None
# the cchatnames are username|chatname, so the javascript has something to handle for cases of "|Anonymous5" vs "username|username"
# handle updates and changes in the set of clients that have the monitoring window open
umlmonitoringusers = { }
for client in self.umlmonitoringclients:
if client.cchatname in umlmonitoringusers:
umlmonitoringusers[client.cchatname] = max(client.clientlasttouch, umlmonitoringusers[client.cchatname])
else:
umlmonitoringusers[client.cchatname] = client.clientlasttouch
#umlmonitoringusers = set([ client.cchatname for client in self.umlmonitoringclients ])
if umlstatusdata:
umlstatusdata["umlmonitoringusers"] = [ {"chatname":chatname, "present":True, "lasttouch":jstime(chatnamelasttouch) } for chatname, chatnamelasttouch in umlmonitoringusers.items() ]
if cclient and cclient.clienttype == "umlmonitoring":
umlstatuschanges["umlmonitoringusers"] = [ {"chatname":cclient.cchatname, "present":(cclient.cchatname in umlmonitoringusers), "lasttouch":jstime(cclient.clientlasttouch) } ]
# rpcrunningclients
if umlstatusdata:
umlstatusdata["rpcrunningclients"] = [ {"clientnumber":client.clientnumber, "present":True, "chatname":client.chatname, "scrapername":client.scrapername, "lasttouch":jstime(client.clientlasttouch)} for client in self.rpcrunningclients ]
if cclient and cclient.clienttype == "rpcrunning":
umlstatuschanges["rpcrunningclients"] = [ {"clientnumber":cclient.clientnumber, "present":(cclient in self.rpcrunningclients), "chatname":cclient.cchatname, "scrapername":cclient.scrapername, "lasttouch":jstime(cclient.clientlasttouch) } ]
# handle draft scraper users and the run states (one for each user, though there may be multiple draft scrapers for them)
draftscraperusers = { } # chatname -> running state
for client in self.draftscraperclients:
draftscraperusers[client.cchatname] = bool(client.processrunning) or draftscraperusers.get(client.cchatname, False)
if umlstatusdata:
umlstatusdata["draftscraperusers"] = [ {"chatname":chatname, "present":True, "running":crunning } for chatname, crunning in draftscraperusers.items() ]
if cclient and not cclient.clienttype == "umlmonitoring" and not cclient.guid:
umlstatuschanges["draftscraperusers"] = [ { "chatname":cclient.cchatname, "present":(cclient.cchatname in draftscraperusers), "running":draftscraperusers.get(cclient.cchatname, False) } ]
# the complexity here reflects the complexity of the structure. the running flag could be set on any one of the clients
def scraperentry(eoos, cclient): # local function
scrapereditors = { } # chatname -> (lasttouch, [clientnumbers])
running = False # we could make this an updated member of EditorsOnOneScraper like lasttouch
for usereditor in eoos.usereditormap.values():
cchatname = usereditor.userclients[0].cchatname
clientnumbers = [uclient.clientnumber for uclient in usereditor.userclients]
scrapereditors[cchatname] = (usereditor.userlasttouch, clientnumbers)
running = running or max([ bool(uclient.processrunning) for uclient in usereditor.userclients ])
for uclient in eoos.anonymouseditors:
scrapereditors[uclient.cchatname] = (uclient.clientlasttouch, [uclient.clientnumber])
# diff mode
if cclient:
scraperusercclient = {'chatname':cclient.cchatname, 'userlasttouch':jstime(cclient.clientlasttouch) }
if cclient.cchatname in scrapereditors:
scraperusercclient['present'] = True
scraperusercclient['uclients'] = scrapereditors[cclient.cchatname][1]
else:
scraperusercclient['present'] = False
scraperusers = [ scraperusercclient ]
else:
scraperusers = [ {'chatname':cchatname, 'userlasttouch':jstime(ultc[0]), 'uclients':ultc[1], 'present':True } for cchatname, ultc in scrapereditors.items() ]
return {'scrapername':eoos.scrapername, 'present':True, 'running':running, 'scraperusers':scraperusers, 'scraperlasttouch':jstime(eoos.scraperlasttouch) }
if umlstatusdata:
umlstatusdata["scraperentries"] = [ ]
for eoos in self.guidclientmap.values():
umlstatusdata["scraperentries"].append(scraperentry(eoos, None))
if cclient and cclient.clienttype in ["editing", "scheduledrun"] and cclient.guid:
if cclient.guid in self.guidclientmap:
umlstatuschanges["scraperentries"] = [ scraperentry(self.guidclientmap[cclient.guid], cclient) ]
else:
umlstatuschanges["scraperentries"] = [ { 'scrapername':cclient.scrapername, 'present':False, 'running':False, 'scraperusers':[ ] } ]
# send the status to the target and updates to everyone else who is monitoring
#print "\numlstatus", umlstatusdata
# new monitoring client
if cclient and cclient.clienttype == "umlmonitoring":
cclient.writejson(umlstatusdata)
# send only updates to current clients
for client in self.umlmonitoringclients:
if client != cclient:
client.writejson(umlstatuschanges)
def clientConnectionLost(self, client):
if client in self.clients:
self.clients.remove(client) # main list
logger.debug("removing %s client# %d" % (client.clienttype, client.clientnumber))
# connection open but nothing else happened
if client.clienttype == None:
if client in self.connectedclients:
self.connectedclients.remove(client)
elif client.clienttype == "stimulate_run":
if client in self.stimulate_runclients:
self.stimulate_runclients.remove(client)
else:
logger.error("No place to remove stimulate_run client# %d" % client.clientnumber)
elif client.clienttype == "httpget":
if client in self.httpgetclients:
self.httpgetclients.remove(client)
else:
logger.error("No place to remove httpget client# %d" % client.clientnumber)
elif client.clienttype == "umlmonitoring":
if client in self.umlmonitoringclients:
self.umlmonitoringclients.remove(client)
else:
logger.error("No place to remove umlmonitoring client# %d" % client.clientnumber)
elif client.clienttype == "rpcrunning":
if client in self.rpcrunningclients:
self.rpcrunningclients.remove(client)
else:
logger.error("No place to remove rpcrunning client %d" % client.clientnumber)
elif not client.guid:
if client in self.draftscraperclients:
self.draftscraperclients.remove(client)
else:
logger.error("No place to remove draftscraper client %d" % client.clientnumber)
elif (client.guid in self.guidclientmap):
if not self.guidclientmap[client.guid].RemoveClient(client):
del self.guidclientmap[client.guid]
else:
if client.username in self.guidclientmap[client.guid].usereditormap:
message = "%s closes a window" % client.chatname
else:
message = "%s leaves" % client.chatname
self.guidclientmap[client.guid].notifyEditorClients(message)
else:
logger.error("No place to remove client %d" % client.clientnumber)
self.notifyMonitoringClients(client)
def clientcommand(self, command, parsed_data):
if command != 'typing':
logger.debug("command %s client# %d" % (command, self.clientnumber))
# update the lasttouch values on associated aggregations
if command != 'automode' and self.clienttype == "editing":
self.clientlasttouch = datetime.datetime.now()
if self.guid and self.username:
assert self.username in self.guidclienteditors.usereditormap
self.guidclienteditors.usereditormap[self.username].userlasttouch = self.clientlasttouch
self.guidclienteditors.scraperlasttouch = self.clientlasttouch
# data uploaded when a new connection is made from the editor
if command == 'connection_open':
self.lconnectionopen(parsed_data)
# finds the corresponding client and presses the run button on it
# receives a single record through the pipeline
elif command == 'stimulate_run':
self.clienttype = "stimulate_run"
self.factory.clientConnectionRegistered(self)
scrapername = parsed_data["scrapername"]
guid = parsed_data["guid"]
assert guid
username = parsed_data["username"]
clientnumber = parsed_data["clientnumber"]
client = None
eoos = self.factory.guidclientmap.get(guid)
if eoos:
usereditor = eoos.usereditormap.get(username)
if usereditor:
for lclient in usereditor.userclients:
if lclient.clientnumber == clientnumber:
client = lclient
if parsed_data.get('django_key') != djangokey:
logger.error("djangokey_mismatch")
self.writejson({'status':'twister djangokey mismatch'})
if client:
client.writejson({"message_type":"console", "content":"twister djangokey mismatch"})
client.writejson({'message_type':'executionstatus', 'content':'runfinished'})
client = None
if client:
logger.info("stimulate on : %s %s client# %d" % (client.cchatname, client.scrapername, client.clientnumber))
assert client.clienttype == "editing" and client.guid
if not client.processrunning:
client.runcode(parsed_data)
self.writejson({"status":"run started"})
else:
client.writejson({"message_type":"console", "content":"client already running"})
self.writejson({"status":"client already running"})
else:
parsed_data.pop("code", None) # shorten the log message
logger.warning("client not found %s" % parsed_data)
self.writejson({"status":"client not found"})
self.transport.loseConnection()
elif command == 'rpcrun':
self.username = parsed_data.get('username', '')
self.userrealname = parsed_data.get('userrealname', self.username)
self.scrapername = parsed_data.get('scrapername', '')
self.scraperlanguage = parsed_data.get('language', '')
self.guid = parsed_data.get("guid", '')
if parsed_data.get('django_key') == djangokey:
self.clienttype = "rpcrunning"
logger.info("connection open %s: %s %s client# %d" % (self.clienttype, self.username, self.scrapername, self.clientnumber))
self.factory.clientConnectionRegistered(self)
self.runcode(parsed_data)
# termination is by the calling function when it receives an executionstatus runfinished message
else:
logger.error("djangokey_mismatch")
self.writejson({'status':'twister djangokey mismatch'})
self.transport.loseConnection()
elif command == 'saved':
line = json.dumps({'message_type' : "saved", 'chatname' : self.chatname})
otherline = json.dumps({'message_type' : "othersaved", 'chatname' : self.chatname})
self.guidclienteditors.rev = parsed_data["rev"]
self.guidclienteditors.chainpatchnumber = 0
self.writeall(line, otherline)
self.factory.notifyMonitoringClientsSmallmessage(self, "savenote")
# should record the rev and chainpatchnumber so when we join to this scraper we know
elif command == 'typing':
logger.debug("command %s client# %d insertlinenumber %s" % (command, self.clientnumber, parsed_data.get("insertlinenumber")))
jline = {'message_type' : "typing", 'content' : "%s typing" % self.chatname}
jotherline = parsed_data.copy()
jotherline.pop("command")
jotherline["message_type"] = "othertyping"
jotherline["content"] = jline["content"]
self.guidclienteditors.chainpatchnumber = parsed_data.get("chainpatchnumber")
self.writeall(json.dumps(jline), json.dumps(jotherline))
self.factory.notifyMonitoringClientsSmallmessage(self, "typingnote")
# this one only applies to draft scrapers when you click run
elif command == 'run':
if self.processrunning:
self.writejson({'content':"Already running! (shouldn't happen)", 'message_type':'console'});
return
if self.username:
if self.automode == 'autoload':
self.writejson({'content':"Not supposed to run! "+self.automode, 'message_type':'console'});
return
if parsed_data.get('guid'):
self.writejson({'content':"scraper run can only be done through stimulate_run method", 'message_type':'console'});
return
logger.info("about to run code %s" % str(parsed_data)[:100])
self.runcode(parsed_data)
elif command == "umlcontrol":
# allows monitoring client to remotely kill processes
if self.clienttype != "umlmonitoring":
logger.error("umlcontrol called by non-monitoring client")
return
logger.info("umlcontrol %s" % ([parsed_data]))
subcommand = parsed_data.get("subcommand")
if subcommand == "killscraper":
scrapername = parsed_data["scrapername"]
for eoos in self.factory.guidclientmap.values(): # would be better if it was by scrapername instead of guid
if eoos.scrapername == scrapername:
for usereditor in eoos.usereditormap.values():
for uclient in usereditor.userclients:
if uclient.processrunning:
logger.info("umlcontrol killing run on client# %d %s" % (uclient.clientnumber, scrapername))
uclient.kill_run()
if subcommand == "killallscheduled":
for client in self.factory.scheduledrunners.values():
if client.processrunning:
logger.info("umlcontrol killing run on client# %d %s" % (client.clientnumber, client.scrapername))
client.kill_run()
else:
logger.info("umlcontrol client# %d %s wasn't running" % (client.clientnumber, client.scrapername))
if "maxscheduledscrapers" in parsed_data:
self.factory.maxscheduledscrapers = parsed_data["maxscheduledscrapers"]
self.factory.notifyMonitoringClients(None)
elif command == "kill":
if self.processrunning:
self.kill_run()
# allows the killing of a process in another open window by same user
elif self.username and self.guid:
usereditor = self.guidclienteditors.usereditormap[self.username]
for client in usereditor.userclients:
if client.processrunning:
client.kill_run()
elif command == 'chat':
line = json.dumps({'message_type':'chat', 'chatname':self.chatname, 'message':parsed_data.get('text'), 'nowtime':jstime(datetime.datetime.now()) })
self.writeall(line)
elif command == 'requesteditcontrol':
for usereditor in self.guidclienteditors.usereditormap.values():
for client in usereditor.userclients:
if client.automode == 'autosave':
client.writejson({'message_type':'requestededitcontrol', "username":self.username})
elif command == 'giveselrange':
self.writeall(None, json.dumps({'message_type':'giveselrange', 'selrange':parsed_data.get('selrange'), 'chatname':self.chatname }))
elif command == 'automode':
automode = parsed_data.get('automode')
if automode == self.automode:
return
if not self.username:
self.automode = automode
self.factory.notifyMonitoringClients(self)
return
usereditor = self.guidclienteditors.usereditormap[self.username]
# self-demote to autoload mode while choosing to promote a particular person to editing mode
if automode == 'autoload':
selectednexteditor = parsed_data.get('selectednexteditor')
if selectednexteditor and selectednexteditor in self.guidclienteditors.usereditormap:
assert self.guidclienteditors.usereditormap[selectednexteditor].usersessionpriority >= usereditor.usersessionpriority
self.guidclienteditors.usereditormap[selectednexteditor].usersessionpriority = usereditor.usersessionpriority
usereditor.usersessionpriority = self.guidclienteditors.usersessionprioritynext
self.guidclienteditors.usersessionprioritynext += 1
self.automode = automode
self.guidclienteditors.notifyEditorClients("")
self.factory.notifyMonitoringClients(self)
# this message helps kill it better and killing it from the browser end
elif command == 'loseconnection':
# Suspect it is possible in some cases that the client sends this command, and before
# we have had a chance to close the connection from here, the client has already gone.
# To cover this case let's handle the exception here and log that loseConnection failed
try:
self.transport.loseConnection()
except:
logger.debug('Closing connection on already closed connection failed')