def scraperentry(eoos, cclient):  # local function
     scrapereditors = { }   # chatname -> (lasttouch, [clientnumbers])
     running = False        # we could make this an updated member of EditorsOnOneScraper like lasttouch
     
     for usereditor in eoos.usereditormap.values():
         cchatname = usereditor.userclients[0].cchatname
         clientnumbers = [uclient.clientnumber  for uclient in usereditor.userclients]
         scrapereditors[cchatname] = (usereditor.userlasttouch, clientnumbers)
         running = running or max([ bool(uclient.processrunning)  for uclient in usereditor.userclients ])
     
     for uclient in eoos.anonymouseditors:
         scrapereditors[uclient.cchatname] = (uclient.clientlasttouch, [uclient.clientnumber])
     
         # diff mode
     if cclient:
         scraperusercclient = {'chatname':cclient.cchatname, 'userlasttouch':jstime(cclient.clientlasttouch) }
         if cclient.cchatname in scrapereditors:
             scraperusercclient['present'] = True
             scraperusercclient['uclients'] = scrapereditors[cclient.cchatname][1]
         else:
             scraperusercclient['present'] = False
         scraperusers = [ scraperusercclient ]
     else:
         scraperusers = [ {'chatname':cchatname, 'userlasttouch':jstime(ultc[0]), 'uclients':ultc[1], 'present':True }  for cchatname, ultc in scrapereditors.items() ]
     
     return {'scrapername':eoos.scrapername, 'present':True, 'running':running, 'scraperusers':scraperusers, 'scraperlasttouch':jstime(eoos.scraperlasttouch) }
 def notifyEditorClients(self, message):
     editorstatusdata = { 'message_type':"editorstatus" }
     
     editorstatusdata["nowtime"] = jstime(datetime.datetime.now())
     editorstatusdata['earliesteditor'] = jstime(self.scrapersessionbegan)
     editorstatusdata["scraperlasttouch"] = jstime(self.scraperlasttouch)
     
             # order by who has first session (and not all draft mode) in order to determin who is the editor
     usereditors = self.usereditormap.values()
     usereditors.sort(key=lambda x: x.usersessionpriority)
     editorstatusdata["loggedinusers"] = [ ]
     editorstatusdata["loggedineditors"] = [ ]
     editorstatusdata["countclients"] = len(self.anonymouseditors)  # so we know what windows are connected to this editor (if any)
     for usereditor in usereditors:
         editorstatusdata["countclients"] += len(usereditor.userclients)
         if usereditor.userclients[-1].savecode_authorized:   # as recorded in last client for this user
             editorstatusdata["loggedineditors"].append(usereditor.username)
         else:
             editorstatusdata["loggedinusers"].append(usereditor.username)
     
     editorstatusdata["nanonymouseditors"] = len(self.anonymouseditors)
     editorstatusdata["message"] = message
     for client in self.anonymouseditors:
         editorstatusdata["chatname"] = client.chatname
         editorstatusdata["clientnumber"] = client.clientnumber
         client.writejson(editorstatusdata); 
     
     for usereditor in self.usereditormap.values():
         for client in usereditor.userclients:
             editorstatusdata["chatname"] = client.chatname
             editorstatusdata["clientnumber"] = client.clientnumber
             client.writejson(editorstatusdata) 
    def clientConnectionRegistered(self, client):
        # Can't remove from the list if it isn't in there so we need to check
        try:
            self.connectedclients.remove(client)
        except:
            logging.error('Failed to remove client %s as it is not in the connectedclient list' % client)
            
        if client.username:
            client.chatname = client.userrealname or client.username
        elif client.clienttype == "editing":
            client.chatname = "Anonymous%d" % self.anonymouscount
            self.anonymouscount += 1
        else:
            client.chatname = "Anonymous"
        client.cchatname = "%s|%s" % (client.username, client.chatname)

        assert client.clienttype in [
          "umlmonitoring", "editing", "rpcrunning",
          "scheduledrun", "stimulate_run", "httpget"]
        
        if client.clienttype == "umlmonitoring":
            self.umlmonitoringclients.append(client)
        elif client.clienttype == 'rpcrunning':
            self.rpcrunningclients.append(client)
        elif client.clienttype == 'httpget':
            self.httpgetclients.append(client)
        elif client.clienttype == 'stimulate_run':
            self.stimulate_runclients.append(client)
        elif client.guid:
            if client.guid not in self.guidclientmap:
                self.guidclientmap[client.guid] = EditorsOnOneScraper(client.guid, client.scrapername, client.scraperlanguage, client.originalrev)
            
            if client.username in self.guidclientmap[client.guid].usereditormap:
                message = "%s opens another window" % client.chatname
            else:
                message = "%s enters" % client.chatname
            
            self.guidclientmap[client.guid].AddClient(client)
            self.guidclientmap[client.guid].notifyEditorClients(message)

        
        else:   # draft scraper type (hardcode the output that would have gone with notifyEditorClients
            editorstatusdata = {'message_type':"editorstatus", "loggedineditors":[], "loggedinusers":[], "nanonymouseditors":1, "countclients":1, "chatname":client.chatname, "message":"Draft scraper connection" }
            editorstatusdata["nowtime"] = jstime(datetime.datetime.now())
            editorstatusdata['earliesteditor'] = jstime(client.clientsessionbegan)
            editorstatusdata["scraperlasttouch"] = jstime(client.clientlasttouch)
            editorstatusdata["clientnumber"] = client.clientnumber
            
            client.writejson(editorstatusdata); 
            self.draftscraperclients.append(client)
        
        if client.clienttype in ["umlmonitoring", "editing", "rpcrunning"]:
            self.notifyMonitoringClients(client)
Example #4
0
    def gotcontrollerconnectionprotocol(self, controllerconnection):
        controllerconnection.srunner = self
        self.controllerconnection = controllerconnection

            # generate the header element that was generated by dispatcher, 
            # which should in future be made by the node-controller
        msg = { 'message_type':'executionstatus', 'content':'startingrun', 'runID':self.jdata["runid"], 'uml':"Direct to controller", 
                'rev':self.jdata["rev"], 'chatname':self.client.chatname, 'nowtime':jstime(datetime.datetime.now())}
        self.client.writeall(json.dumps(msg))
        
            # send the data into the controller including all the code it should run
        sdata = json.dumps(self.jdata)
        logger.debug("sending: %s" % str([sdata[:1000]]))
        controllerconnection.transport.write('POST /Execute HTTP/1.0\r\n')
        controllerconnection.transport.write('Content-Length: %s\r\n' % len(sdata))
        controllerconnection.transport.write('Content-Type: text/json\r\n') # text/json???
        controllerconnection.transport.write('Connection: close\r\n')
        controllerconnection.transport.write("\r\n")
        controllerconnection.transport.write(sdata)
Example #5
0
    def gotcontrollerconnectionprotocol(self, controllerconnection):
        controllerconnection.srunner = self
        self.controllerconnection = controllerconnection

            # generate the header element that was generated by dispatcher, 
            # which should in future be made by the node-controller
        msg = { 'message_type':'executionstatus', 'content':'startingrun', 'runID':self.jdata["runid"], 'uml':"Direct to controller", 
                'rev':self.jdata["rev"], 'chatname':self.client.chatname, 'nowtime':jstime(datetime.datetime.now())}
        self.client.writeall(json.dumps(msg))
        
            # send the data into the controller including all the code it should run
        sdata = json.dumps(self.jdata)
        logger.debug("sending: %s" % str([sdata[:1000]]))
        controllerconnection.transport.write('POST /Execute HTTP/1.0\r\n')
        controllerconnection.transport.write('Content-Length: %s\r\n' % len(sdata))
        controllerconnection.transport.write('Content-Type: text/json\r\n') # text/json???
        controllerconnection.transport.write('Connection: close\r\n')
        controllerconnection.transport.write("\r\n")
        controllerconnection.transport.write(sdata)
 def notifyMonitoringClientsSmallmessage(self, cclient, smallmessage):
     if cclient.guid:
         umlsavenotification = {'message_type':smallmessage, "scrapername":cclient.scrapername, "cchatname":cclient.cchatname, "nowtime":jstime(datetime.datetime.now()) }
         for client in self.umlmonitoringclients:
             client.writejson(umlsavenotification) 
    def notifyMonitoringClients(self, cclient):  # cclient is the one whose state has changed (it can be normal editor or a umlmonitoring case)
        Dtclients = len(self.connectedclients) + len(self.rpcrunningclients) + len(self.stimulate_runclients) + len(self.httpgetclients) + len(self.umlmonitoringclients) + len(self.draftscraperclients) + sum([eoos.Dcountclients()  for eoos in self.guidclientmap.values()])
        if len(self.clients) != Dtclients:
            logger.error("Miscount of clients %d %d" % (len(self.clients), Dtclients))
        
        # both of these are in the same format and read the same, but umlstatuschanges are shorter
        umlstatuschanges = {'message_type':"umlchanges", "nowtime":jstime(datetime.datetime.now()) }; 
        if self.notifiedmaxscheduledscrapers != self.maxscheduledscrapers:
            self.notifiedmaxscheduledscrapers = self.maxscheduledscrapers
            umlstatuschanges["maxscheduledscrapers"] = self.maxscheduledscrapers

        if cclient and cclient.clienttype == "umlmonitoring":
             umlstatusdata = {'message_type':"umlstatus", "nowtime":umlstatuschanges["nowtime"]}
             umlstatusdata["maxscheduledscrapers"] = self.maxscheduledscrapers
        else:
             umlstatusdata = None
        
        # the cchatnames are username|chatname, so the javascript has something to handle for cases of "|Anonymous5" vs "username|username"
        
        # handle updates and changes in the set of clients that have the monitoring window open
        umlmonitoringusers = { }
        for client in self.umlmonitoringclients:
            if client.cchatname in umlmonitoringusers:
                umlmonitoringusers[client.cchatname] = max(client.clientlasttouch, umlmonitoringusers[client.cchatname])
            else:
                umlmonitoringusers[client.cchatname] = client.clientlasttouch

        #umlmonitoringusers = set([ client.cchatname  for client in self.umlmonitoringclients ])
        if umlstatusdata:
            umlstatusdata["umlmonitoringusers"] = [ {"chatname":chatname, "present":True, "lasttouch":jstime(chatnamelasttouch) }  for chatname, chatnamelasttouch in umlmonitoringusers.items() ]
        if cclient and cclient.clienttype == "umlmonitoring":
            umlstatuschanges["umlmonitoringusers"] = [ {"chatname":cclient.cchatname, "present":(cclient.cchatname in umlmonitoringusers), "lasttouch":jstime(cclient.clientlasttouch) } ]

        # rpcrunningclients
        if umlstatusdata:
            umlstatusdata["rpcrunningclients"] = [ {"clientnumber":client.clientnumber, "present":True, "chatname":client.chatname, "scrapername":client.scrapername, "lasttouch":jstime(client.clientlasttouch)}  for client in self.rpcrunningclients ]
        if cclient and cclient.clienttype == "rpcrunning":
            umlstatuschanges["rpcrunningclients"] = [ {"clientnumber":cclient.clientnumber, "present":(cclient in self.rpcrunningclients), "chatname":cclient.cchatname, "scrapername":cclient.scrapername, "lasttouch":jstime(cclient.clientlasttouch) } ]

        # handle draft scraper users and the run states (one for each user, though there may be multiple draft scrapers for them)
        draftscraperusers = { }  # chatname -> running state
        for client in self.draftscraperclients:
            draftscraperusers[client.cchatname] = bool(client.processrunning) or draftscraperusers.get(client.cchatname, False)
        if umlstatusdata:
            umlstatusdata["draftscraperusers"] = [ {"chatname":chatname, "present":True, "running":crunning }  for chatname, crunning in draftscraperusers.items() ]
        if cclient and not cclient.clienttype == "umlmonitoring" and not cclient.guid:
            umlstatuschanges["draftscraperusers"] = [ { "chatname":cclient.cchatname, "present":(cclient.cchatname in draftscraperusers), "running":draftscraperusers.get(cclient.cchatname, False) } ]

        # the complexity here reflects the complexity of the structure.  the running flag could be set on any one of the clients
        def scraperentry(eoos, cclient):  # local function
            scrapereditors = { }   # chatname -> (lasttouch, [clientnumbers])
            running = False        # we could make this an updated member of EditorsOnOneScraper like lasttouch
            
            for usereditor in eoos.usereditormap.values():
                cchatname = usereditor.userclients[0].cchatname
                clientnumbers = [uclient.clientnumber  for uclient in usereditor.userclients]
                scrapereditors[cchatname] = (usereditor.userlasttouch, clientnumbers)
                running = running or max([ bool(uclient.processrunning)  for uclient in usereditor.userclients ])
            
            for uclient in eoos.anonymouseditors:
                scrapereditors[uclient.cchatname] = (uclient.clientlasttouch, [uclient.clientnumber])
            
                # diff mode
            if cclient:
                scraperusercclient = {'chatname':cclient.cchatname, 'userlasttouch':jstime(cclient.clientlasttouch) }
                if cclient.cchatname in scrapereditors:
                    scraperusercclient['present'] = True
                    scraperusercclient['uclients'] = scrapereditors[cclient.cchatname][1]
                else:
                    scraperusercclient['present'] = False
                scraperusers = [ scraperusercclient ]
            else:
                scraperusers = [ {'chatname':cchatname, 'userlasttouch':jstime(ultc[0]), 'uclients':ultc[1], 'present':True }  for cchatname, ultc in scrapereditors.items() ]
            
            return {'scrapername':eoos.scrapername, 'present':True, 'running':running, 'scraperusers':scraperusers, 'scraperlasttouch':jstime(eoos.scraperlasttouch) }
        
        
        if umlstatusdata:
            umlstatusdata["scraperentries"] = [ ]
            for eoos in self.guidclientmap.values():
                umlstatusdata["scraperentries"].append(scraperentry(eoos, None))
                
        if cclient and cclient.clienttype in ["editing", "scheduledrun"] and cclient.guid:
            if cclient.guid in self.guidclientmap:
                umlstatuschanges["scraperentries"] = [ scraperentry(self.guidclientmap[cclient.guid], cclient) ]
            else:
                umlstatuschanges["scraperentries"] = [ { 'scrapername':cclient.scrapername, 'present':False, 'running':False, 'scraperusers':[ ] } ]
        
        # send the status to the target and updates to everyone else who is monitoring
        #print "\numlstatus", umlstatusdata
        
        # new monitoring client
        if cclient and cclient.clienttype == "umlmonitoring":
            cclient.writejson(umlstatusdata) 
        
        # send only updates to current clients
        for client in self.umlmonitoringclients:
            if client != cclient:
                client.writejson(umlstatuschanges) 
    def clientcommand(self, command, parsed_data):
        if command != 'typing':
            logger.debug("command %s client# %d" % (command, self.clientnumber))
        
        # update the lasttouch values on associated aggregations
        if command != 'automode' and self.clienttype == "editing":
            self.clientlasttouch = datetime.datetime.now()
            if self.guid and self.username:
                assert self.username in self.guidclienteditors.usereditormap
                self.guidclienteditors.usereditormap[self.username].userlasttouch = self.clientlasttouch
                self.guidclienteditors.scraperlasttouch = self.clientlasttouch

        # data uploaded when a new connection is made from the editor
        if command == 'connection_open':
            self.lconnectionopen(parsed_data)

                # finds the corresponding client and presses the run button on it
                # receives a single record through the pipeline
        elif command == 'stimulate_run':
            self.clienttype = "stimulate_run"
            self.factory.clientConnectionRegistered(self)  

            scrapername = parsed_data["scrapername"]
            guid = parsed_data["guid"]
            assert guid
            username = parsed_data["username"]
            clientnumber = parsed_data["clientnumber"]

            client = None
            eoos = self.factory.guidclientmap.get(guid)
            if eoos:
                usereditor = eoos.usereditormap.get(username)
                if usereditor:
                    for lclient in usereditor.userclients:
                        if lclient.clientnumber == clientnumber:
                            client = lclient

            if parsed_data.get('django_key') != djangokey:
                logger.error("djangokey_mismatch")
                self.writejson({'status':'twister djangokey mismatch'})
                if client:
                    client.writejson({"message_type":"console", "content":"twister djangokey mismatch"})  
                    client.writejson({'message_type':'executionstatus', 'content':'runfinished'})
                    client = None
            
            if client:
                logger.info("stimulate on : %s %s client# %d" % (client.cchatname, client.scrapername, client.clientnumber))
                assert client.clienttype == "editing" and client.guid
                if not client.processrunning:
                    client.runcode(parsed_data)
                    self.writejson({"status":"run started"})  
                else:
                    client.writejson({"message_type":"console", "content":"client already running"})  
                    self.writejson({"status":"client already running"})  
            else:
                parsed_data.pop("code", None)   # shorten the log message
                logger.warning("client not found %s" % parsed_data)
                self.writejson({"status":"client not found"})  

            self.transport.loseConnection()

        elif command == 'rpcrun':
            self.username = parsed_data.get('username', '')
            self.userrealname = parsed_data.get('userrealname', self.username)
            self.scrapername = parsed_data.get('scrapername', '')
            self.scraperlanguage = parsed_data.get('language', '')
            self.guid = parsed_data.get("guid", '')
            if parsed_data.get('django_key') == djangokey:
                self.clienttype = "rpcrunning"
                logger.info("connection open %s: %s %s client# %d" % (self.clienttype, self.username, self.scrapername, self.clientnumber)) 
                self.factory.clientConnectionRegistered(self)  
                self.runcode(parsed_data)
                    # termination is by the calling function when it receives an executionstatus runfinished message
            else:
                logger.error("djangokey_mismatch")
                self.writejson({'status':'twister djangokey mismatch'})
                self.transport.loseConnection()


        elif command == 'saved':
            line = json.dumps({'message_type' : "saved", 'chatname' : self.chatname})
            otherline = json.dumps({'message_type' : "othersaved", 'chatname' : self.chatname})
            self.guidclienteditors.rev = parsed_data["rev"]
            self.guidclienteditors.chainpatchnumber = 0
            self.writeall(line, otherline)
            self.factory.notifyMonitoringClientsSmallmessage(self, "savenote")


    # should record the rev and chainpatchnumber so when we join to this scraper we know
        elif command == 'typing':
            logger.debug("command %s client# %d insertlinenumber %s" % (command, self.clientnumber, parsed_data.get("insertlinenumber")))
            jline = {'message_type' : "typing", 'content' : "%s typing" % self.chatname}
            jotherline = parsed_data.copy()
            jotherline.pop("command")
            jotherline["message_type"] = "othertyping"
            jotherline["content"] = jline["content"]
            self.guidclienteditors.chainpatchnumber = parsed_data.get("chainpatchnumber")
            self.writeall(json.dumps(jline), json.dumps(jotherline))
            self.factory.notifyMonitoringClientsSmallmessage(self, "typingnote")
            
            # this one only applies to draft scrapers when you click run
        elif command == 'run':
            if self.processrunning:
                self.writejson({'content':"Already running! (shouldn't happen)", 'message_type':'console'}); 
                return 
            if self.username:
                if self.automode == 'autoload':
                    self.writejson({'content':"Not supposed to run! "+self.automode, 'message_type':'console'}); 
                    return 
            
            if parsed_data.get('guid'):
                self.writejson({'content':"scraper run can only be done through stimulate_run method", 'message_type':'console'}); 
                return 

            logger.info("about to run code %s" % str(parsed_data)[:100])
            self.runcode(parsed_data)
        
        elif command == "umlcontrol":
                # allows monitoring client to remotely kill processes
            if self.clienttype != "umlmonitoring":
                logger.error("umlcontrol called by non-monitoring client")
                return
                
            logger.info("umlcontrol %s" % ([parsed_data]))

            subcommand = parsed_data.get("subcommand")
            if subcommand == "killscraper":
                scrapername = parsed_data["scrapername"]
                for eoos in self.factory.guidclientmap.values():   # would be better if it was by scrapername instead of guid
                    if eoos.scrapername == scrapername:
                        for usereditor in eoos.usereditormap.values():
                            for uclient in usereditor.userclients:
                                if uclient.processrunning:
                                    logger.info("umlcontrol killing run on client# %d %s" % (uclient.clientnumber, scrapername))
                                    uclient.kill_run()

            if subcommand == "killallscheduled":
                for client in self.factory.scheduledrunners.values():
                    if client.processrunning:
                       logger.info("umlcontrol killing run on client# %d %s" % (client.clientnumber, client.scrapername))
                       client.kill_run()
                    else:
                        logger.info("umlcontrol client# %d %s wasn't running" % (client.clientnumber, client.scrapername))
                       
            if "maxscheduledscrapers" in parsed_data:
                self.factory.maxscheduledscrapers = parsed_data["maxscheduledscrapers"]
                self.factory.notifyMonitoringClients(None)

        elif command == "kill":
            if self.processrunning:
                self.kill_run()
                
                # allows the killing of a process in another open window by same user
            elif self.username and self.guid:   
                usereditor = self.guidclienteditors.usereditormap[self.username]
                for client in usereditor.userclients:
                    if client.processrunning:
                        client.kill_run()

        elif command == 'chat':
            line = json.dumps({'message_type':'chat', 'chatname':self.chatname, 'message':parsed_data.get('text'), 'nowtime':jstime(datetime.datetime.now()) })
            self.writeall(line)
        
        elif command == 'requesteditcontrol':
            for usereditor in self.guidclienteditors.usereditormap.values():
                for client in usereditor.userclients:
                    if client.automode == 'autosave':
                        client.writejson({'message_type':'requestededitcontrol', "username":self.username})
        
        elif command == 'giveselrange':
            self.writeall(None, json.dumps({'message_type':'giveselrange', 'selrange':parsed_data.get('selrange'), 'chatname':self.chatname }))
            
        elif command == 'automode':
            automode = parsed_data.get('automode')
            if automode == self.automode:
                return

            if not self.username:
                self.automode = automode
                self.factory.notifyMonitoringClients(self)
                return

            usereditor = self.guidclienteditors.usereditormap[self.username]
 
                # self-demote to autoload mode while choosing to promote a particular person to editing mode
            if automode == 'autoload':
                selectednexteditor = parsed_data.get('selectednexteditor')
                if selectednexteditor and selectednexteditor in self.guidclienteditors.usereditormap:
                    assert self.guidclienteditors.usereditormap[selectednexteditor].usersessionpriority >= usereditor.usersessionpriority
                    self.guidclienteditors.usereditormap[selectednexteditor].usersessionpriority = usereditor.usersessionpriority
                usereditor.usersessionpriority = self.guidclienteditors.usersessionprioritynext
                self.guidclienteditors.usersessionprioritynext += 1
            
            self.automode = automode
            
            self.guidclienteditors.notifyEditorClients("")
            self.factory.notifyMonitoringClients(self)

        # this message helps kill it better and killing it from the browser end
        elif command == 'loseconnection':
            # Suspect it is possible in some cases that the client sends this command, and before
            # we have had a chance to close the connection from here, the client has already gone.
            # To cover this case let's handle the exception here and log that loseConnection failed
            try:
                self.transport.loseConnection()
            except: 
                logger.debug('Closing connection on already closed connection failed')