def forwardCommand(self,ip,port,cmd): """Forwards a RPC to another machine :param ip: IP of the server :param port: the port at which the server is listening :param cmd: the command that should be executed there :return: the result of the command """ result="" try: server=getServerProxy(ip,port) result=eval("server."+cmd) foamLogger("server").debug("Forwarding to "+ip+"the command\""+cmd+"\" Result:"+str(result)) except Fault as reason: result="Fault: "+str(reason) except socket.error as reason: result="socket.error: "+str(reason) except TypeError as reason: result="Type error: ",reason except SyntaxError as reason: result="Syntax Error in:"+cmd if result==None: result="" return result
def queryData(self): """Ask the server for additional data""" server=getServerProxy(self["ip"],self["port"],useSSL=self["ssl"]) for name in ["commandLine","cwd","foamVersion","isParallel","mpi","pyFoamVersion","scriptName","user","hostname"]: if server is not None: result=eval("server."+name+"()") else: result="no info" self[name]=result
def _registerServer(self,ip,pid,port,sync=True,external=False): """Registers a new server :param ip: IP of the server :param pid: Die PID at the server :param port: the port at which the server is listening :param external: was called via XMLRPC :param sync: (optional) if to sync with the webserver or not """ self.dataLock.acquire() serverID="%s:%d" % (ip,port) foamLogger("server").info("Registering: %s with PID: %d" % (serverID,pid)) insertServer=False try: if self.servers.has_key(serverID): # maybe it's another process server=getServerProxy(ip,port) gotPid=server.pid() if pid!=gotPid: self.servers.pop(serverID) foamLogger("server").warning("Server "+serverID+" changed PID from %d to %d" % (pid,gotPid)) insertServer=True else: foamLogger("server").warning("Server "+serverID+" already registered") else: insertServer=True if insertServer: new=ServerInfo(ip,pid,port) doIt=external if not doIt: doIt=new.checkValid() if doIt: new.queryData() # occasional errors with 'Connection refused' self.servers[serverID]=new foamLogger("server").debug("Inserted "+serverID) except: foamLogger("server").error("Registering Server "+serverID+" failed:"+str(sys.exc_info()[0])) foamLogger("server").error("Reason:"+str(sys.exc_info()[1])) foamLogger("server").error("Trace:"+str(extract_tb(sys.exc_info()[2]))) self.dataLock.release() if DO_WEBSYNC and insertServer and sync: foamLogger("server").info("Registering %s for webserver: %s" % (serverID,'new/%(ip)s/%(port)s/' % {'ip':ip, 'port':port})) try: self.webserver.new_process(ip, port) except: foamLogger("server").warning("Registering %s for webserver failed!" % (serverID)) return True
def forwardCommand(self,info,cmd): """Forwards a command :param info: dictionary with the information :param cmd: the command that will be forwarded """ result=0 try: if self.opts.zeroconf: server=getServerProxy(info["ip"],info["port"]) result=eval("server."+cmd) else: result=float(self.server.forwardCommand(info["ip"],info["port"],cmd)) except: pass return result
def forwardCommand(self, info, cmd): """Forwards a command :param info: dictionary with the information :param cmd: the command that will be forwarded """ result = 0 try: if self.opts.zeroconf: server = getServerProxy(info["ip"], info["port"]) result = eval("server." + cmd) else: result = float( self.server.forwardCommand(info["ip"], info["port"], cmd)) except: pass return result
def checkValid(self): """Check with server whether this data item is still valid""" result=False foamLogger("server").debug("Checking "+self["ip"]+"@"+str(self["port"])) try: server=getServerProxy(self["ip"],self["port"]) pid=server.pid() if pid==self["pid"]: result=True except socket.timeout as reason: foamLogger("server").info(self["ip"]+"@"+str(self["port"])+" seems to be dead") except: foamLogger("server").debug("Checking Valid "+self["ip"]+" failed:"+str(sys.exc_info()[0])) foamLogger("server").debug("Reason:"+str(sys.exc_info()[1])) foamLogger("server").debug("Trace:"+str(extract_tb(sys.exc_info()[2]))) foamLogger("server").debug("Result for "+self["ip"]+"@"+str(self["port"])+" = "+str(result)) return result
def run(self): self.parent.startupLock.acquire() foamLogger("server").info("Collector starting") if DO_WEBSYNC: foamLogger("server").info("Get Processes from Webserver") try: webserver = ServerProxy(WEBSERVER_RPCURL) for ip,port,pid in webserver.running_processes(): port = int(port) try: server=getServerProxy(ip,port) pid=server.pid() # occasional errors with 'Connection refused' self.parent._registerServer(ip,pid,port,sync=False) except: foamLogger("server").error("Unknown exception "+str(sys.exc_info()[0])+" while registering %s:%s" % (ip, port)) foamLogger("server").error("Reason:"+str(sys.exc_info()[1])) foamLogger("server").error("Trace:"+str(extract_tb(sys.exc_info()[2]))) except: foamLogger("server").warning("Unknown exception "+str(sys.exc_info()[0])+" while syncing with webserver %s" % (WEBSERVER_RPCURL)) port=config().getint("Network","startServerPort") length=config().getint("Network","nrServerPorts") machines=config().get("Metaserver","searchservers") addreses=machines.split(',') if self.additional!=None: addreses=self.additional.split(',')+addreses for a in addreses: foamLogger("server").info("Collecting in subnet "+a) for host in IP(a): try: name,alias,rest =socket.gethostbyaddr(str(host)) except socket.herror as reason: # no name for the host name="unknown" foamLogger("server").debug("Collector Checking:"+str(host)+" "+name) result=None try: result=checkFoamServers(str(host),port,length) except: foamLogger("server").error("Unknown exception "+str(sys.exc_info()[0])+" while checking for new servers"+str((str(host),port,length))) foamLogger("server").error("Reason:"+str(sys.exc_info()[1])) foamLogger("server").error("Trace:"+str(extract_tb(sys.exc_info()[2]))) if result!=None: foamLogger("server").debug("Collector Found "+str(result)+" for "+name) for p in result: try: server=getServerProxy(str(host),p) ip=server.ip() pid=server.pid() self.parent._registerServer(ip,pid,p) except: foamLogger("server").error("Unknown exception "+str(sys.exc_info()[0])+" while registering "+name) foamLogger("server").error("Reason:"+str(sys.exc_info()[1])) foamLogger("server").error("Trace:"+str(extract_tb(sys.exc_info()[2]))) else: foamLogger("server").debug("Collector Found "+str(result)+" for "+name) self.parent.startupLock.release() foamLogger("server").info("Collector finished")
def run(self): from PyFoam.Infrastructure.Authentication import ensureKeyPair ensureKeyPair() if self.opts.zeroconf: data = getServerList(self.opts.timeout, verbose=self.opts.debugZeroconf, progress=self.opts.progressZeroconf) else: self.warning( "Using the old method of querying the meta-server. This is deprecated" ) try: self.server = getServerProxy(self.parser.options.server, self.parser.options.port) data = self.server.list() except socket.error as reason: print_("Socket error while connecting:", reason) sys.exit(1) except ProtocolError as reason: print_("XMLRPC-problem", reason) sys.exit(1) if len(data) == 0: self.warning("No runs found") return hostString = "Hostname" maxhost = len(hostString) cmdString = "Command Line" maxcommandline = len(cmdString) for name, info in iteritems(data): if len(info["commandLine"]) > maxcommandline: maxcommandline = len(info["commandLine"]) if self.opts.ip: tmpHost = info["ip"] else: tmpHost = info["hostname"] if tmpHost == "no info": tmpHost = info["ip"] if len(tmpHost) > maxhost: maxhost = len(tmpHost) header = hostString + ( " " * (maxhost - len(hostString)) ) + " | " + " Port | User | " + cmdString + "\n" line = ("-" * (len(header))) header += line formatString = "%-" + str(maxhost) + "s | %6d | %10s | %s" print_(header) for name, info in iteritems(data): if self.opts.user: if self.opts.user != info["user"]: continue if self.opts.ip: tmpHost = info["ip"] else: tmpHost = info["hostname"] if tmpHost == "no info": tmpHost = info["ip"] print_(formatString % (tmpHost, info["port"], info["user"], info["commandLine"])) if self.parser.options.process: isParallel = self.forwardCommand(info, "isParallel()") if isParallel: pidString = "CPUs: %5d" % self.forwardCommand( info, "procNr()") else: pidString = "PID: %6d" % info["pid"] print_(" %s Working dir: %s" % (pidString, info["cwd"])) if self.parser.options.time: startTime = self.forwardCommand(info, "startTime()") endTime = self.forwardCommand(info, "endTime()") createTime = self.forwardCommand(info, "createTime()") nowTime = self.forwardCommand(info, "time()") try: progress = (nowTime - createTime) / (endTime - createTime) except ZeroDivisionError: progress = 0 try: progress2 = (nowTime - startTime) / (endTime - startTime) except ZeroDivisionError: progress2 = 0 print_( " Time: %g Timerange: [ %g , %g ] Mesh created: %g -> Progress: %.2f%% (Total: %.2f%%)" % (nowTime, startTime, endTime, createTime, progress * 100, progress2 * 100)) wallTime = self.forwardCommand(info, "wallTime()") now = time.time() start = now - wallTime startString = time.strftime("%Y-%b-%d %H:%M", time.localtime(start)) try: estimate = start + wallTime / progress estimateString = time.strftime("%Y-%b-%d %H:%M", time.localtime(estimate)) except ZeroDivisionError: estimate = start estimateString = " - NaN - " print_(" Started: %s Walltime: %8gs Estimated End: %s" % (startString, wallTime, estimateString)) if self.opts.resources: mem = self.forwardCommand(info, "usedMemory()") loads = self.forwardCommand(info, "loadAvg()") loadString = "" if len(loads) == 3: loadString = " Load 1m: %.1f - 5m: %.1f - 15m: %.1f" % tuple( loads) print((" Max memory: %f MB" % mem) + loadString) if self.parser.options.process or self.parser.options.time: print_(line) if self.parser.options.dump: print_(info) print_(line)
def run(self): from PyFoam.Infrastructure.Authentication import ensureKeyPair ensureKeyPair() host=self.parser.getArgs()[0] port=int(self.parser.getArgs()[1]) cmd=self.parser.options.command try: self.server=getServerProxy(host,port) methods=self.server.system.listMethods() if not cmd: print_("Connected to server",host,"on port",port) print_(len(methods),"available methods found") except socket.error as reason: print_("Socket error while connecting:",reason) sys.exit(1) except ProtocolError as reason: print_("XMLRPC-problem",reason) sys.exit(1) if cmd: result=self.executeCommand(cmd) if result!=None: print_(result) sys.exit(0) while 1: try: if PY3: line = input(self.prompt) else: line = raw_input(self.prompt) except (KeyboardInterrupt,EOFError): # Catch a ctrl-D print_() print_("Goodbye") sys.exit() line.strip() parts=line.split() if len(parts)==0: print_("For help type 'help'") continue if parts[0]=="help": if len(parts)==1: print_("For help on a method type 'help <method>'") print_("Available methods are:") for m in methods: print_("\t",m) elif len(parts)==2: name=parts[1] if name in methods: signature=self.executeCommand("system.methodSignature(\""+name+"\")") help=self.executeCommand("system.methodHelp(\""+name+"\")") print_("Method : ",name) print_("Signature : ",signature) print_(help) else: print_("Method",name,"does not exist") else: print_("Too many arguments") else: result=self.executeCommand(line) if result!=None: print_(result)
def run(self): if not self.opts.server and not self.opts.pickle: error("No mode selected") if self.opts.server and self.opts.pickle: error("Both modes selected") doPandas=self.opts.pandasData or self.opts.pandasSeries if self.opts.server: if len(self.parser.getArgs())!=2: error("Need a server and a port to be specified") host=self.parser.getArgs()[0] port=int(self.parser.getArgs()[1]) try: self.server=getServerProxy(host,port) methods=self.server.system.listMethods() except socket.error: reason = sys.exc_info()[1] # Needed because python 2.5 does not support 'as e' self.error("Socket error while connecting:",reason) except ProtocolError: reason = sys.exc_info()[1] # Needed because python 2.5 does not support 'as e' self.error("XMLRPC-problem",reason) plotInfo=self.executeCommand("getPlots()") lineInfo=self.executeCommand("getPlotData()") else: if len(self.parser.getArgs())!=1: warning("Only the first parameter is used") fName=self.parser.getArgs()[0] unpick=pickle.Unpickler(open(fName,"rb")) lineInfo=unpick.load() plotInfo=unpick.load() if not self.quiet: print_("Found",len(plotInfo),"plots and",len(lineInfo),"data sets") registry=TimeLinesRegistry() for nr,line in iteritems(lineInfo): if not self.quiet: print_("Adding line",nr) TimeLineCollection(preloadData=line,registry=registry) registry.resolveSlaves() if (self.opts.csvFiles or self.opts.excelFiles or doPandas or self.opts.numpyData) and self.opts.rawLines: rawData={} rawSeries={} rawNumpy={} for k,l in iteritems(registry.lines): name=str(k) if type(k)==int: name="Line%d" % k csvName=self.opts.filePrefix+name+".csv" if self.opts.csvFiles: if not self.quiet: print_("Writing",k,"to",csvName) l.getData().writeCSV(csvName) if self.opts.excelFiles: xlsName=self.opts.filePrefix+name+".xls" if not self.quiet: print_("Writing",k,"to",xlsName) l.getData().getData().to_excel(xlsName) if self.opts.pandasData: rawData[k]=l.getData().getData() if self.opts.numpyData: rawNumpy[k]=l.getData().data.copy() if self.opts.pandasSeries: rawSeries[k]=l.getData().getSeries() if self.opts.numpyData: self.setData({"rawNumpy":rawNumpy}) if self.opts.pandasData: self.setData({"rawData":rawData}) if self.opts.pandasSeries: self.setData({"rawSeries":rawSeries}) if self.opts.csvFiles or self.opts.excelFiles: return pRegistry=PlotLinesRegistry() plotNumpy={} plotData={} plotSeries={} for i,p in iteritems(plotInfo): theId=p["id"] if not self.quiet: print_("Plotting",i,":",theId,end=" ") spec=CustomPlotInfo(raw=p["spec"]) if len(registry.get(p["data"]).getTimes())>0 and len(registry.get(p["data"]).getValueNames())>0: if self.opts.csvFiles or self.opts.excelFiles or doPandas or self.opts.numpyData: dataSet=registry.get(p["data"]).getData() if self.opts.csvFiles: dataSet.writeCSV(self.opts.filePrefix+theId+".csv") if self.opts.excelFiles: dataSet.getData().to_excel(self.opts.filePrefix+theId+".xls") if self.opts.numpyData: plotNumpy[theId]=dataSet.data.copy() if self.opts.pandasData: plotData[theId]=dataSet.getData() if self.opts.pandasSeries: plotSeries[theId]=dataSet.getSeries() else: if self.opts.start or self.opts.end: # rewrite CustomPlotInfo one of these days if "start" in spec.getDict(): self.warning("Overriding plot start",spec["start"], "with",self.opts.start) spec.set("start",self.opts.start) if "end" in spec.getDict(): self.warning("Overriding plot end",spec["end"], "with",self.opts.end) spec.set("end",self.opts.end) mp=createPlotTimelines(registry.get(p["data"]), spec, implementation=self.opts.implementation, showWindow=self.opts.showWindow, registry=pRegistry) if self.opts.insertTitles: mp.actualSetTitle(p["spec"]["theTitle"]) if self.opts.writePictures: if mp.hasData(): mp.doHardcopy(self.opts.prefix+theId,"png") else: if not self.quiet: print_("has no data",end=" ") if not self.quiet: print_() else: if not self.quiet: print_("No data - skipping") if not(self.opts.csvFiles or doPandas): sleep(self.opts.sleepTime) # there seems to be a timing issue with Gnuplot if self.opts.numpyData: self.setData({"plotNumpy":plotNumpy}) if self.opts.pandasData: self.setData({"plotData":plotData}) if self.opts.pandasSeries: self.setData({"plotSeries":plotSeries})
def run(self): if not self.opts.server and not self.opts.pickle: error("No mode selected") if self.opts.server and self.opts.pickle: error("Both modes selected") doPandas = self.opts.pandasData or self.opts.pandasSeries if self.opts.server: if len(self.parser.getArgs()) != 2: error("Need a server and a port to be specified") host = self.parser.getArgs()[0] port = int(self.parser.getArgs()[1]) try: self.server = getServerProxy(host, port) methods = self.server.system.listMethods() except socket.error: reason = sys.exc_info()[ 1] # Needed because python 2.5 does not support 'as e' self.error("Socket error while connecting:", reason) except ProtocolError: reason = sys.exc_info()[ 1] # Needed because python 2.5 does not support 'as e' self.error("XMLRPC-problem", reason) plotInfo = self.executeCommand("getPlots()") lineInfo = self.executeCommand("getPlotData()") else: if len(self.parser.getArgs()) != 1: warning("Only the first parameter is used") fName = self.parser.getArgs()[0] unpick = pickle.Unpickler(open(fName, "rb")) lineInfo = unpick.load() plotInfo = unpick.load() if not self.quiet: print_("Found", len(plotInfo), "plots and", len(lineInfo), "data sets") registry = TimeLinesRegistry() for nr, line in iteritems(lineInfo): if not self.quiet: print_("Adding line", nr) TimeLineCollection(preloadData=line, registry=registry) registry.resolveSlaves() if (self.opts.csvFiles or self.opts.excelFiles or doPandas or self.opts.numpyData) and self.opts.rawLines: rawData = {} rawSeries = {} rawNumpy = {} for k, l in iteritems(registry.lines): name = str(k) if type(k) == int: name = "Line%d" % k csvName = self.opts.filePrefix + name + ".csv" if self.opts.csvFiles: if not self.quiet: print_("Writing", k, "to", csvName) l.getData().writeCSV(csvName) if self.opts.excelFiles: xlsName = self.opts.filePrefix + name + ".xls" if not self.quiet: print_("Writing", k, "to", xlsName) l.getData().getData().to_excel(xlsName) if self.opts.pandasData: rawData[k] = l.getData().getData() if self.opts.numpyData: rawNumpy[k] = l.getData().data.copy() if self.opts.pandasSeries: rawSeries[k] = l.getData().getSeries() if self.opts.numpyData: self.setData({"rawNumpy": rawNumpy}) if self.opts.pandasData: self.setData({"rawData": rawData}) if self.opts.pandasSeries: self.setData({"rawSeries": rawSeries}) if self.opts.csvFiles or self.opts.excelFiles: return pRegistry = PlotLinesRegistry() plotNumpy = {} plotData = {} plotSeries = {} for i, p in iteritems(plotInfo): theId = p["id"] if not self.quiet: print_("Plotting", i, ":", theId, end=" ") spec = CustomPlotInfo(raw=p["spec"]) if len(registry.get(p["data"]).getTimes()) > 0 and len( registry.get(p["data"]).getValueNames()) > 0: if self.opts.csvFiles or self.opts.excelFiles or doPandas or self.opts.numpyData: dataSet = registry.get(p["data"]).getData() if self.opts.csvFiles: dataSet.writeCSV(self.opts.filePrefix + theId + ".csv") if self.opts.excelFiles: dataSet.getData().to_excel(self.opts.filePrefix + theId + ".xls") if self.opts.numpyData: plotNumpy[theId] = dataSet.data.copy() if self.opts.pandasData: plotData[theId] = dataSet.getData() if self.opts.pandasSeries: plotSeries[theId] = dataSet.getSeries() else: if self.opts.start or self.opts.end: # rewrite CustomPlotInfo one of these days if "start" in spec.getDict(): self.warning("Overriding plot start", spec["start"], "with", self.opts.start) spec.set("start", self.opts.start) if "end" in spec.getDict(): self.warning("Overriding plot end", spec["end"], "with", self.opts.end) spec.set("end", self.opts.end) mp = createPlotTimelines( registry.get(p["data"]), spec, implementation=self.opts.implementation, showWindow=self.opts.showWindow, registry=pRegistry) if self.opts.insertTitles: mp.actualSetTitle(p["spec"]["theTitle"]) if self.opts.writePictures: if mp.hasData(): mp.doHardcopy(self.opts.prefix + theId, "png", termOpts=self.opts.terminalOptions) else: if not self.quiet: print_("has no data", end=" ") if not self.quiet: print_() else: if not self.quiet: print_("No data - skipping") if not (self.opts.csvFiles or doPandas): sleep(self.opts.sleepTime ) # there seems to be a timing issue with Gnuplot if self.opts.numpyData: self.setData({"plotNumpy": plotNumpy}) if self.opts.pandasData: self.setData({"plotData": plotData}) if self.opts.pandasSeries: self.setData({"plotSeries": plotSeries})
def run(self): from PyFoam.Infrastructure.Authentication import ensureKeyPair ensureKeyPair() if self.opts.zeroconf: data=getServerList(self.opts.timeout, verbose=self.opts.debugZeroconf, progress=self.opts.progressZeroconf) else: self.warning("Using the old method of querying the meta-server. This is deprecated") try: self.server=getServerProxy(self.parser.options.server,self.parser.options.port) data=self.server.list() except socket.error as reason: print_("Socket error while connecting:",reason) sys.exit(1) except ProtocolError as reason: print_("XMLRPC-problem",reason) sys.exit(1) if len(data)==0: self.warning("No runs found") return hostString="Hostname" maxhost=len(hostString) cmdString="Command Line" maxcommandline=len(cmdString) for name,info in iteritems(data): if len(info["commandLine"])>maxcommandline: maxcommandline=len(info["commandLine"]) if self.opts.ip: tmpHost=info["ip"] else: tmpHost=info["hostname"] if tmpHost=="no info": tmpHost=info["ip"] if len(tmpHost)>maxhost: maxhost=len(tmpHost) header=hostString+(" "*(maxhost-len(hostString)))+" | "+" Port | User | "+cmdString+"\n" line=("-"*(len(header))) header+=line formatString="%-"+str(maxhost)+"s | %6d | %10s | %s" print_(header) for name,info in iteritems(data): if self.opts.user: if self.opts.user!=info["user"]: continue if self.opts.ip: tmpHost=info["ip"] else: tmpHost=info["hostname"] if tmpHost=="no info": tmpHost=info["ip"] print_(formatString % (tmpHost,info["port"],info["user"],info["commandLine"])) if self.parser.options.process: isParallel=self.forwardCommand(info,"isParallel()") if isParallel: pidString="CPUs: %5d" % self.forwardCommand(info,"procNr()") else: pidString="PID: %6d" % info["pid"] print_(" %s Working dir: %s" % (pidString,info["cwd"])) if self.parser.options.time: startTime=self.forwardCommand(info,"startTime()") endTime=self.forwardCommand(info,"endTime()") createTime=self.forwardCommand(info,"createTime()") nowTime=self.forwardCommand(info,"time()") try: progress=(nowTime-createTime)/(endTime-createTime) except ZeroDivisionError: progress=0 try: progress2=(nowTime-startTime)/(endTime-startTime) except ZeroDivisionError: progress2=0 print_(" Time: %g Timerange: [ %g , %g ] Mesh created: %g -> Progress: %.2f%% (Total: %.2f%%)" % (nowTime,startTime,endTime,createTime,progress*100,progress2*100)) wallTime=self.forwardCommand(info,"wallTime()") now=time.time() start=now-wallTime startString=time.strftime("%Y-%b-%d %H:%M",time.localtime(start)) try: estimate=start+wallTime/progress estimateString=time.strftime("%Y-%b-%d %H:%M",time.localtime(estimate)) except ZeroDivisionError: estimate=start estimateString=" - NaN - " print_(" Started: %s Walltime: %8gs Estimated End: %s" % (startString,wallTime,estimateString)) if self.opts.resources: mem=self.forwardCommand(info,"usedMemory()") loads=self.forwardCommand(info,"loadAvg()") loadString="" try: if len(loads)==3: loadString=" Load 1m: %.1f - 5m: %.1f - 15m: %.1f" % tuple(loads) except TypeError: loadString=" Load: "+str(loads) print((" Max memory: %f MB" % mem)+loadString) if self.parser.options.process or self.parser.options.time: print_(line) if self.parser.options.dump: print_(info) print_(line)