Example #1
0
 def start(self):  
     """
     Start threads and directory observer.
     """
     #start threads
     
     for threadid in range(1,self.options.threads):
         print "start proc [",threadid,"]"
        
         worker=Process(target=funcworker, args=(self,threadid))
         worker.daemon=True
         self.pool.append(worker)
         worker.start() 
         #self.processimage(picture,options)
     self.starttime=time.time() 
     if self.options.watch:
         eventhandler=addtoqueue(self.picturequeue)
         self.observer = Observer()
         self.observer.schedule(eventhandler, self.args[0], recursive=True)
         self.observer.start()
     #We let the master process do some work because its useful for matplotlib.
     if not self.options.nowalk:
         self.fillqueuewithexistingfiles()
     if self.options.servermode:
          
          context = zmq.Context()
          socket = context.socket(zmq.REQ)
          tokenlist=  self.conf['Server'].split(":")
          
          server=":".join([tokenlist[0],tokenlist[1],self.options.serverport])
          print server
          socket.connect (server)
          from Leash import addauthentication
     try:
         while ( self.options.servermode or 
                 (not self.picturequeue.empty()) 
                 or (self.dirwalker and self.dirwalker.is_alive() )
                 or self.options.watch): 
                 try:
                     picture = self.picturequeue.get(timeout=1)
                 except Empty:
                     continue
                 lastfile, data =self.procimage(picture,0)
                 
                 if self.options.servermode:
                     request={"command":"putplotdata","argument":{"data":{
                             "result":"plot","data":{"filename":lastfile,"graphs":data,
                                                     "stat":{}}
                               }}}
                     socket.send_multipart([json.dumps(addauthentication( request,self.conf))])
                     socket.recv()
                 if np.mod(self.allp.value,500)==0:
                     self.timreport()
     except KeyboardInterrupt:
         pass
     if self.options.servermode:
         context.destroy()
     self.stop()
     self.timreport()
     return self.allp.value, time.time()-self.starttime
Example #2
0
def startservers(serverconfs):
    Servers = []
    for serverid, serverconf in enumerate(serverconfs):
        Servers.append(
            Process(target=saxsdogserver,
                    args=(serverconf, serverid, None, None)))
        Servers[-1].start()
Example #3
0
 def start_image_queue(self,object,attachment):
     """
     prepare new image queue
     start processing threads
     """
     self.lasttime=time.time()
     self.lastcount=0
     self.history=history()
     self.attachments=[]        
     try:
         self._checkdirectorycollision(object['argument']['calibration']['Directory'])
         for attachstr in attachment:
             self.attachments.append(json.loads(attachstr))        
         self.calibration=object['argument']['calibration']
         if object['argument']['calibration'].get("Threads")>0:
             self.threads=object['argument']['calibration'].get("Threads")
         else:
             self.threads=self.options.threads
         self.threads=max(self.threads,2)
         print "abort old queue"
         if self.imagequeue:
              self.queue_abort()
         print "aborted old queue"
         
         o=atrdict.AttrDict({"plotwindow":False,"threads":self.threads,
                 		"watch":self.options.watchdir,
                         "watchdir":os.sep.join(object['argument']['calibration'].get("Directory")),
                         "servermode":True,
                         "silent":True,"plotwindow":False,
                         "walkdirinthreads":True,
                 		"outdir":self.options.outdir,
                         "relpath":self.options.relpath,
                 		"writesvg":False,
                         "writepng":False,"resume":False,
                         "serverport":self.serverport,
                         "nowalk":True,
                         "GISAXSmode":self.calibration["GISAXSmode"],
                         "livefilelist":"xxx",
                         "OverwriteFiles":False
                          })
         cals=[]
         
         dir=os.path.normpath(
             os.path.join(
                           self.serverdir,
                          os.sep.join(object['argument']['calibration'].get('Directory')
                         )))
         if "Masks" in object['argument']['calibration']:
             for mnumber,mask in enumerate(object['argument']['calibration']["Masks"]):
                 cals.append(calibration.calibration(
                                             object['argument']['calibration'],
                                             mask,
                                             self.attachments[mnumber]))
         if "Slices" in   object['argument']['calibration']:
             for slice in object['argument']['calibration']["Slices"]:
                 cals.append(GISAXSSlices.slice(object['argument']['calibration'],slice,self.attachments))
         
         if self.calibration["OverwriteFiles"]:
             o["OverwriteFiles"]=True
         
         '''Create empty file for filelisting'''
         filelist_path="xxx"
         if self.calibration["Live-Filelisting"]:
             filelist_path = os.path.join(os.path.split(dir)[0],"results")
             filelist_name = "filelist_" + os.path.split(dir)[1]+".log"
             filelist_path = os.path.join(filelist_path,filelist_name)
             try:
                 open(filelist_path, "w+").close()
                 o["livefilelist"]=filelist_path
             except:
                 print "Couldn't open " + filelist_path
             
         self.imagequeue=imagequeuelib.imagequeue(cals,
                 o,dir,self.serverconf)
         print "startimgq"
         self.imagequeueprocess=Process(target=self.imagequeue.start)
         self.imagequeueprocess.start()
         print "listening to feeder"
         serverdir= self.serverdir
         self.feederproc=Process(target=subscribeToFileChanges,args=
                                 (self.imagequeue,
                                  self.feederurl,
                                 dir,
                                 serverdir
                                 )
                                 )
         print "directory to watch "+dir
     
         self.feederproc.start()
         
         self.queuestatrtime=time.time()
         self.plotresult={"result":"Empty","data":{"stat":self.stat()}}
        
         result={"result":"new queue","data":{"cal":object['argument']['calibration']}}
     except IOError as e: 
         result={"result":"IOError","data":{"Error": str(e).replace("\n"," ")}}
     except ValueError as e:
         result={"result":"ValueError","data":{"Error": str(e)}}
     except Exception as e:
         result={"result":"Error","data":{"Error": str(e)}}
         print e
     return result
Example #4
0
class Server():
    """
    class to manage a saxsdog server
    """
    def __init__(self,conf,serverid,stopflag=None,serverdir=None):
        self.files=None
        self.stopflag=stopflag
        self.serverid=serverid
        self.options, self.args=parsecommandline()
        if len(self.args)==0:
            self.args=["."]
        if   self.options.outdir!="" :
              print '"'+self.options.outdir+'"'+" directory does not exist"
              sys.exit()
        if self.options.feederurl=="":
            self.feederurl=conf["Feeder"]
        self.serverconf=conf
        self.comandosocket=None
        if serverdir:
            self.serverdir=serverdir
        else:
            self.serverdir=self.args[0]
        self.commandschema=json.load(open(os.path.abspath(os.path.dirname(__file__))+'/LeashRequestSchema.json'))
        self.imagequeue=None
        self.feederproc=None
        self.mergedataqueue=Queue()
        self.mergecount=0
        self.mergeresult={}
        self.mergeprocess=None
        self.mergestatus = ""
        self.mergestatusprotocoll = ""
        self.threads=self.options.threads
        self.plotdata=None
        
           
    def start(self):
        """
        start server loop
        """
        if self.options.port=="":
            self.serverport=self.serverconf['Server'].split(':')[-1]
        else:
            self.serverport=self.options.port
        self.secret=self.serverconf['Secret']
        context = zmq.Context()
        self.comandosocket = context.socket(zmq.REP)
        print "server listenes at tcp://*:%s" % self.serverport
        self.comandosocket.bind("tcp://*:%s" % self.serverport)
       
        while True:
            try:
                message=self.comandosocket.recv_multipart()
                 
                object=json.loads(message[0])
                validate(object,self.commandschema)
                self.authenticate(object)
                attachment=message[1:]
                result=self.commandhandler(object,attachment)
            except ValidationError as e:
                result={"result":"ValidationError in request","data":e.message}
            except ValueError as e:
                result={"result":"ValueError in request","data":{"Error":e.message}}
            except  AuthenticationError as e:
                 result={"result":"AuthenticationError","data":{"Error":e.message}}
            except KeyboardInterrupt:
                context.destroy()
                self.queue_abort()
            except Exception as e:
                result={"result":"ServerError","data":{"Error":e.message}}
                print e
            try:
                self.comandosocket.send(json.dumps(result))
            except Exception as e:
                result={"result":"ServerError","data":{"Error":e.message}}
                self.comandosocket.send(json.dumps(result))
           
            if self.stopflag and self.stopflag.value==1:
                print "#######STOP##########"
                break
            
    def authenticate(self,data):
        """
        check signature of request
        """
        sign=data['sign']
        data["sign"]=""
        m=hashlib.sha512()
        now=time.time() 
         
        m.update(json.dumps(data, sort_keys=True))
        m.update(self.secret)
        if not abs(data["time"]-now)<900:
            raise AuthenticationError("Untimely request.")
        if not sign==m.hexdigest():
            raise AuthenticationError("Wrong signature.")
        
                
            
    def listdir(self,request):
        print  self.serverdir
        dir=  os.path.join( self.serverdir, os.sep.join(request["argument"]['directory']))
        try:
            files=os.listdir(os.path.join(dir))
            print files
        except OSError as e:
            return {"result":"OSError","data":{"Error":str(e)}}
        content=[]
        for item in files:
            if os.path.isdir(os.path.join(dir,item)):
                content.append({"isdir":True,"path":item})
            else:
                content.append({"isdir":False,"path":item})
        return {"result":"listdir","data":{"dircontent":content,"directory":dir.split(os.sep)}}
    def commandhandler(self,object,attachment):
        """
        
        """
        command=object['command']
        #print "got: "+ command 
        if command=='new':
            result= self.start_image_queue(object,attachment)
            print str(datetime.datetime.now())+": new queue for:"
            if object['argument']['calibration'].get("Directory"):
                print "    '"+os.sep.join(object['argument']['calibration'].get("Directory"))+"'"
        elif command=='abort':
             result=self.queue_abort()
        elif command=='close':
             result=self.queue_close()
        elif command=="readdir":
             result=self.readdir(object)
        elif command=="listdir":
             result=self.listdir(object)
        elif command=="putplotdata":
             result=self.updateplot(object)
        elif command=="plot":
             if self.imagequeue:
                result=self.plot()
             else:
                result={"result":"no queue","data":{}}
        elif command=="stat":
             result={"result":"stat","data":{"stat":self.stat()}}
        elif command=="get":
            if self.imagequeue:
                 result={"result":"cal","data":{
                                                "cal":self.calibration,
                                               
                                                "attachments":self.attachments
                                                }}
            else:
                result={"result":"no queue","data":{}}
        elif command=="fileslist":
            result=self.getresultfileslists()
        elif command=="mergedata":
            result=self.mergedatacommand(object['argument']["mergeconf"],attachment)
        elif command=="getmergedata":
            result=self.mergeresult
        elif command=="mergestat":
            result={"result":"stat","data":{"mergeinfo":self.mergestatus}}
            self.mergestatusprotocoll+=self.mergestatus
            self.mergestatus = ""
        else:
            result={"result":"ErrorNotimplemented","data":{"Error":command+" not implemented"}}
       
         
        return result
    def start_image_queue(self,object,attachment):
        """
        prepare new image queue
        start processing threads
        """
        self.lasttime=time.time()
        self.lastcount=0
        self.history=history()
        self.attachments=[]        
        try:
            self._checkdirectorycollision(object['argument']['calibration']['Directory'])
            for attachstr in attachment:
                self.attachments.append(json.loads(attachstr))        
            self.calibration=object['argument']['calibration']
            if object['argument']['calibration'].get("Threads")>0:
                self.threads=object['argument']['calibration'].get("Threads")
            else:
                self.threads=self.options.threads
            self.threads=max(self.threads,2)
            print "abort old queue"
            if self.imagequeue:
                 self.queue_abort()
            print "aborted old queue"
            
            o=atrdict.AttrDict({"plotwindow":False,"threads":self.threads,
                    		"watch":self.options.watchdir,
                            "watchdir":os.sep.join(object['argument']['calibration'].get("Directory")),
                            "servermode":True,
                            "silent":True,"plotwindow":False,
                            "walkdirinthreads":True,
                    		"outdir":self.options.outdir,
                            "relpath":self.options.relpath,
                    		"writesvg":False,
                            "writepng":False,"resume":False,
                            "serverport":self.serverport,
                            "nowalk":True,
                            "GISAXSmode":self.calibration["GISAXSmode"],
                            "livefilelist":"xxx",
                            "OverwriteFiles":False
                             })
            cals=[]
            
            dir=os.path.normpath(
                os.path.join(
                              self.serverdir,
                             os.sep.join(object['argument']['calibration'].get('Directory')
                            )))
            if "Masks" in object['argument']['calibration']:
                for mnumber,mask in enumerate(object['argument']['calibration']["Masks"]):
                    cals.append(calibration.calibration(
                                                object['argument']['calibration'],
                                                mask,
                                                self.attachments[mnumber]))
            if "Slices" in   object['argument']['calibration']:
                for slice in object['argument']['calibration']["Slices"]:
                    cals.append(GISAXSSlices.slice(object['argument']['calibration'],slice,self.attachments))
            
            if self.calibration["OverwriteFiles"]:
                o["OverwriteFiles"]=True
            
            '''Create empty file for filelisting'''
            filelist_path="xxx"
            if self.calibration["Live-Filelisting"]:
                filelist_path = os.path.join(os.path.split(dir)[0],"results")
                filelist_name = "filelist_" + os.path.split(dir)[1]+".log"
                filelist_path = os.path.join(filelist_path,filelist_name)
                try:
                    open(filelist_path, "w+").close()
                    o["livefilelist"]=filelist_path
                except:
                    print "Couldn't open " + filelist_path
                
            self.imagequeue=imagequeuelib.imagequeue(cals,
                    o,dir,self.serverconf)
            print "startimgq"
            self.imagequeueprocess=Process(target=self.imagequeue.start)
            self.imagequeueprocess.start()
            print "listening to feeder"
            serverdir= self.serverdir
            self.feederproc=Process(target=subscribeToFileChanges,args=
                                    (self.imagequeue,
                                     self.feederurl,
                                    dir,
                                    serverdir
                                    )
                                    )
            print "directory to watch "+dir
        
            self.feederproc.start()
            
            self.queuestatrtime=time.time()
            self.plotresult={"result":"Empty","data":{"stat":self.stat()}}
           
            result={"result":"new queue","data":{"cal":object['argument']['calibration']}}
        except IOError as e: 
            result={"result":"IOError","data":{"Error": str(e).replace("\n"," ")}}
        except ValueError as e:
            result={"result":"ValueError","data":{"Error": str(e)}}
        except Exception as e:
            result={"result":"Error","data":{"Error": str(e)}}
            print e
        return result
    def queue_abort(self):
        if self.imagequeue:
            print "trystop"
            self.imagequeue.stop()
            if  os.sys.platform!="win32":
                print "terminate"
                self.imagequeueprocess.terminate()
            else:
                self.imagequeueprocess.join(1)
        self.imagequeue=None
        return {"result":"queue aborted","data":{"stat":self.stat()}}
    def queue_close(self):
        if self.feederproc:
            if  os.sys.platform!="win32":
                print "feeder terminate"
                self.feederproc.terminate()
            print "feeder join"
            self.feederproc.join(1)
            self.feederproc=None
        return {"result":"queue closed","data":{"stat":self.stat()}}
    def readdir(self,object):
        print "readdir"
        if self.imagequeue:
            self.imagequeue.clearqueue()
        self.history.hist=[]
        self.history.IntegralParameters.clear()
        self.history.filelist.clear()
        try:
            self.imagequeue.fillqueuewithexistingfiles()
            pass
        except AttributeError  :
            result={"result":"ValueError","data":{"Error":"Start Queue first"}}
            return result
        except Exception as e:
            result={"result":"Error","data":{"Error":str(e)}}
            return result
        return {"result":"queue restarted with all files","data":{"stat":self.stat()}}
    
    def plot(self):
        if self.plotdata:
            plotresult=self.plotdata
        else:
            plotresult={"result":"plotdata"}
        plotresult['data']["stat"]=self.stat()  
        plotresult['data']["history"]=self.history.hist
        plotresult['data']["IntegralParameters"]=self.history.IntegralParameters

        return  plotresult
   
    def stat(self):
        
        if self.imagequeue:
            self.lasttime=time.time()
            self.lastcount=self.imagequeue.allp.value
            self.history.update(self.imagequeue.histqueue)
            while True:
                try:
                    self.plotdata=self.imagequeue.plotdataqueue.get(False)['argument']["data"]
                except Empty:
                    break
            result= {"images processed":self.imagequeue.allp.value,
             "queue length":self.imagequeue.picturequeue.qsize(),
             "time":time.time(),
             "start time":self.queuestatrtime}
            try:
                self.mergeresult=self.mergedataqueue.get(False)
                self.mergecount+=1
                result["mergecount"]= self.mergecount
            except Empty:
                pass 
            return result
        else:
            return{}
    def getresultfileslists(self):
        filelists={}
        for basename in sorted(self.history.filelist.keys()):
            fileset= self.history.filelist[basename]
            for kind in fileset.keys():
                if kind in  filelists :
                    filelists[kind].append(fileset[kind])
                else:
                    filelists[kind]=  [fileset[kind]]
        return {"result":"resultfileslists","data":{"fileslist":filelists}}
    def mergedatacommand(self,conf,attachment):
        self.mergestatusprotocoll=""
        try:
            directory=os.path.normpath(
                        os.path.join(self.serverdir, os.sep.join(self.calibration["Directory"])))
            relativedirname=os.path.dirname(conf["OutputFileBaseName"])
            resultdir=os.path.join(directory,relativedirname)
            if not  os.path.isdir(resultdir):
                os.mkdir(resultdir)
            conf["OutputFileBaseName"]= directory.split(os.sep)[-1]+os.path.basename(conf["OutputFileBaseName"])
            self.mergestatus+="\nBase-directory of merge is: " + directory
            #print "\nBase-directory of merge is: " + directory
            for table in conf["LogDataTables"]:
                for file in table["Files"]:
                    if "RemotePath" in file:
                        file["RemotePath"].insert(0,self.serverdir)
            
            self.mergestatus+="\nMerging datalogger files: "
            print "\nMerging datalogger files: "
            logsTable,firstImage,zeroCorr,peakframe,logbasename=datamerge.mergelogs(self,conf,attachment=attachment,directory=resultdir)
           
            
            def mergeimages(logsTable,firstImage,peakframe,mergedataqueue,resultdir):
                try:
                    imd,filelists=datamerge.readallimages(self,directory)     
                    basename=os.path.normpath(os.sep.join([os.path.normpath(resultdir), "Imagedata"]))
                    imd.to_csv(basename+".csv")
                    mergestatus= "\nImagedata can be found in: " +  (basename+".csv")
                
                    self.mergestatus+="\nNow merging imagedata with logfiles.."
                    mergedTable,delta= datamerge.mergeimgdata(self,logbasename,directory,logsTable,imd,firstImage=firstImage,zeroCorr=zeroCorr)
                    peakframe.index = peakframe.index + delta
                    plotdata=datamerge.syncplot(peakframe,imd)
                    plotdata["CalculatedTimeshift"]=str(delta)
                    
                    self.mergestatus+="\nWriting output tables..."
                    datamerge.writeTable(self,conf,mergedTable,directory=resultdir)
                    datamerge.writeFileLists(self,conf,filelists,directory=resultdir,serverdir=self.serverdir)
                    if conf["OutputFormats"]["hdf"] and conf['HDFOptions']["IncludeTIF"]:
                        datamerge.imgtohdf(conf,directory,resultdir)
                    if conf["OutputFormats"]["hdf"] and conf['HDFOptions']["IncludeCHI"]:
                        datamerge.graphstohdf(conf,filelists,resultdir)
                    mergedataqueue.put({"result":
                                        "mergedata","data":{"syncplot":plotdata,"fileslist":filelists}})
                except Exception as e:
                    self.mergestatus+="\n------------ERROR------------\n"
                    self.mergestatus+=str(e)
                    self.mergestatus+="\n-----------------------------\n"
                    self.mergestatus+="The datamerger was stopped!"
                except ValueError:
                    self.mergestatus+="\n------------ERROR------------\n"
                    self.mergestatus+="A Value Error occurred..."
                    self.mergestatus+="\n-----------------------------\n"
                    self.mergestatus+="The datamerger was stopped!"
                
            if  not self.mergeprocess or not  self.mergeprocess.is_alive():
                self.mergeprocess=threading.Thread(target=mergeimages,
                                                   args=(logsTable,firstImage,peakframe,self.mergedataqueue,resultdir))
                self.mergeprocess.start()
            else:
                return {"result":"Error","data":{"Error": "Merge already started please wait"}}
                
        except Exception as e:
            return {"result":"Error","data":{"Error": str(e)}}
        return {"result":"merge started"  "mergedata","data":{}}
    
    def _checkdirectorycollision(self,pathlist):
         if not self.serverid=="Local":
             serverconfs=json.load(open(os.path.expanduser("~"+os.sep+".saxsdognetwork")))
             mydir=os.path.normpath(os.sep.join(pathlist))
             
             for i,conf in enumerate(serverconfs):
                if i!=self.serverid and self.serverid!="Local":
                    argu=["get"]
                    opt=atrdict.AttrDict({"serverno":i,"server":conf["Server"]})
                    result=json.loads(Leash.initcommand(opt,argu,conf))
                    if result['result']=="cal":
                        otherpath=os.path.normpath(os.sep.join(result["data"]["cal"]['Directory']))
                        
                    
                        if ((otherpath.startswith(mydir) or mydir.startswith(otherpath))
                            or (otherpath=="." or mydir==".")):
                            raise DirectoryCollisionException("Directory collides with: "+otherpath)
    
    def writeToMergeStatus(self,new_status):
        self.mergestatus+=new_status
    
    def getMergeStatusProtocoll(self):
        time.sleep(2)
        return self.mergestatusprotocoll
Example #5
0
    def start(self):
        """
        Start threads and directory observer.
        """
        #start threads

        for threadid in range(1, self.options.threads):
            print "start proc [", threadid, "]"

            worker = Process(target=funcworker, args=(self, threadid))
            worker.daemon = True
            self.pool.append(worker)
            worker.start()
            #self.processimage(picture,options)
        self.starttime = time.time()
        if self.options.watch:
            eventhandler = addtoqueue(self.picturequeue)
            self.observer = Observer()
            self.observer.schedule(eventhandler, self.args[0], recursive=True)
            self.observer.start()
        #We let the master process do some work because its useful for matplotlib.
        if not self.options.nowalk:
            self.fillqueuewithexistingfiles()
        if self.options.servermode:

            context = zmq.Context()
            socket = context.socket(zmq.REQ)
            tokenlist = self.conf['Server'].split(":")

            server = ":".join(
                [tokenlist[0], tokenlist[1], self.options.serverport])
            print server
            socket.connect(server)
            from Leash import addauthentication
        try:
            while (self.options.servermode or (not self.picturequeue.empty())
                   or (self.dirwalker and self.dirwalker.is_alive())
                   or self.options.watch):
                try:
                    picture = self.picturequeue.get(timeout=1)
                except Empty:
                    continue
                lastfile, data = self.procimage(picture, 0)

                if self.options.servermode:
                    request = {
                        "command": "putplotdata",
                        "argument": {
                            "data": {
                                "result": "plot",
                                "data": {
                                    "filename": lastfile,
                                    "graphs": data,
                                    "stat": {}
                                }
                            }
                        }
                    }
                    socket.send_multipart(
                        [json.dumps(addauthentication(request, self.conf))])
                    socket.recv()
                if np.mod(self.allp.value, 500) == 0:
                    self.timreport()
        except KeyboardInterrupt:
            pass
        if self.options.servermode:
            context.destroy()
        self.stop()
        self.timreport()
        return self.allp.value, time.time() - self.starttime
Example #6
0
    def start(self):
        """
        Start threads and directory observer.
        """
        #start threads

        for threadid in range(1, self.options.threads):
            print "start proc [", threadid, "]"

            worker = Process(target=funcworker, args=(self, threadid))
            worker.daemon = True
            self.pool.append(worker)
            worker.start()
            #self.processimage(picture,options)
        self.starttime = time.time()
        if self.options.watch:
            eventhandler = addtoqueue(self.picturequeue)
            self.observer = Observer()
            self.observer.schedule(eventhandler, self.args[0], recursive=True)
            self.observer.start()
        #We let the master process do some work because its useful for matplotlib.
        if not self.options.nowalk:
            self.fillqueuewithexistingfiles()
        if self.options.servermode:

            from Leash import addauthentication
        try:
            while (self.options.servermode or (not self.picturequeue.empty())
                   or (self.dirwalker and self.dirwalker.is_alive())
                   or self.options.watch):
                try:
                    picture = self.picturequeue.get(timeout=1)
                except Empty:
                    continue

#in Case something goes wrong
                try:
                    lastfile, data = self.procimage(picture, 0)
                except:
                    continue

                if self.options.servermode:
                    request = {
                        "command": "putplotdata",
                        "argument": {
                            "data": {
                                "result": "plot",
                                "data": {
                                    "filename": lastfile,
                                    "graphs": data,
                                    "stat": {}
                                }
                            }
                        }
                    }

                    self.plotdataqueue.put(request)
                if np.mod(self.allp.value, 500) == 0:
                    self.timreport()
        except KeyboardInterrupt:
            pass

        self.stop()
        self.timreport()
        return self.allp.value, time.time() - self.starttime