def startservers(serverconfs):
Servers = []
for serverid, serverconf in enumerate(serverconfs):
Servers.append(
Process(target=saxsdogserver,
args=(serverconf, serverid, None, None)))
Servers[-1].start()
def start_image_queue(self,object,attachment):
"""
prepare new image queue
start processing threads
"""
self.lasttime=time.time()
self.lastcount=0
self.history=history()
self.attachments=[]
try:
self._checkdirectorycollision(object['argument']['calibration']['Directory'])
for attachstr in attachment:
self.attachments.append(json.loads(attachstr))
self.calibration=object['argument']['calibration']
if object['argument']['calibration'].get("Threads")>0:
self.threads=object['argument']['calibration'].get("Threads")
else:
self.threads=self.options.threads
self.threads=max(self.threads,2)
print "abort old queue"
if self.imagequeue:
self.queue_abort()
print "aborted old queue"
o=atrdict.AttrDict({"plotwindow":False,"threads":self.threads,
"watch":self.options.watchdir,
"watchdir":os.sep.join(object['argument']['calibration'].get("Directory")),
"servermode":True,
"silent":True,"plotwindow":False,
"walkdirinthreads":True,
"outdir":self.options.outdir,
"relpath":self.options.relpath,
"writesvg":False,
"writepng":False,"resume":False,
"serverport":self.serverport,
"nowalk":True,
"GISAXSmode":self.calibration["GISAXSmode"],
"livefilelist":"xxx",
"OverwriteFiles":False
})
cals=[]
dir=os.path.normpath(
os.path.join(
self.serverdir,
os.sep.join(object['argument']['calibration'].get('Directory')
)))
if "Masks" in object['argument']['calibration']:
for mnumber,mask in enumerate(object['argument']['calibration']["Masks"]):
cals.append(calibration.calibration(
object['argument']['calibration'],
mask,
self.attachments[mnumber]))
if "Slices" in object['argument']['calibration']:
for slice in object['argument']['calibration']["Slices"]:
cals.append(GISAXSSlices.slice(object['argument']['calibration'],slice,self.attachments))
if self.calibration["OverwriteFiles"]:
o["OverwriteFiles"]=True
'''Create empty file for filelisting'''
filelist_path="xxx"
if self.calibration["Live-Filelisting"]:
filelist_path = os.path.join(os.path.split(dir)[0],"results")
filelist_name = "filelist_" + os.path.split(dir)[1]+".log"
filelist_path = os.path.join(filelist_path,filelist_name)
try:
open(filelist_path, "w+").close()
o["livefilelist"]=filelist_path
except:
print "Couldn't open " + filelist_path
self.imagequeue=imagequeuelib.imagequeue(cals,
o,dir,self.serverconf)
print "startimgq"
self.imagequeueprocess=Process(target=self.imagequeue.start)
self.imagequeueprocess.start()
print "listening to feeder"
serverdir= self.serverdir
self.feederproc=Process(target=subscribeToFileChanges,args=
(self.imagequeue,
self.feederurl,
dir,
serverdir
)
)
print "directory to watch "+dir
self.feederproc.start()
self.queuestatrtime=time.time()
self.plotresult={"result":"Empty","data":{"stat":self.stat()}}
result={"result":"new queue","data":{"cal":object['argument']['calibration']}}
except IOError as e:
result={"result":"IOError","data":{"Error": str(e).replace("\n"," ")}}
except ValueError as e:
result={"result":"ValueError","data":{"Error": str(e)}}
except Exception as e:
result={"result":"Error","data":{"Error": str(e)}}
print e
return result
def start(self):
"""
Start threads and directory observer.
"""
#start threads
for threadid in range(1, self.options.threads):
print "start proc [", threadid, "]"
worker = Process(target=funcworker, args=(self, threadid))
worker.daemon = True
self.pool.append(worker)
worker.start()
#self.processimage(picture,options)
self.starttime = time.time()
if self.options.watch:
eventhandler = addtoqueue(self.picturequeue)
self.observer = Observer()
self.observer.schedule(eventhandler, self.args[0], recursive=True)
self.observer.start()
#We let the master process do some work because its useful for matplotlib.
if not self.options.nowalk:
self.fillqueuewithexistingfiles()
if self.options.servermode:
context = zmq.Context()
socket = context.socket(zmq.REQ)
tokenlist = self.conf['Server'].split(":")
server = ":".join(
[tokenlist[0], tokenlist[1], self.options.serverport])
print server
socket.connect(server)
from Leash import addauthentication
try:
while (self.options.servermode or (not self.picturequeue.empty())
or (self.dirwalker and self.dirwalker.is_alive())
or self.options.watch):
try:
picture = self.picturequeue.get(timeout=1)
except Empty:
continue
lastfile, data = self.procimage(picture, 0)
if self.options.servermode:
request = {
"command": "putplotdata",
"argument": {
"data": {
"result": "plot",
"data": {
"filename": lastfile,
"graphs": data,
"stat": {}
}
}
}
}
socket.send_multipart(
[json.dumps(addauthentication(request, self.conf))])
socket.recv()
if np.mod(self.allp.value, 500) == 0:
self.timreport()
except KeyboardInterrupt:
pass
if self.options.servermode:
context.destroy()
self.stop()
self.timreport()
return self.allp.value, time.time() - self.starttime
def start(self):
"""
Start threads and directory observer.
"""
#start threads
for threadid in range(1, self.options.threads):
print "start proc [", threadid, "]"
worker = Process(target=funcworker, args=(self, threadid))
worker.daemon = True
self.pool.append(worker)
worker.start()
#self.processimage(picture,options)
self.starttime = time.time()
if self.options.watch:
eventhandler = addtoqueue(self.picturequeue)
self.observer = Observer()
self.observer.schedule(eventhandler, self.args[0], recursive=True)
self.observer.start()
#We let the master process do some work because its useful for matplotlib.
if not self.options.nowalk:
self.fillqueuewithexistingfiles()
if self.options.servermode:
from Leash import addauthentication
try:
while (self.options.servermode or (not self.picturequeue.empty())
or (self.dirwalker and self.dirwalker.is_alive())
or self.options.watch):
try:
picture = self.picturequeue.get(timeout=1)
except Empty:
continue
#in Case something goes wrong
try:
lastfile, data = self.procimage(picture, 0)
except:
continue
if self.options.servermode:
request = {
"command": "putplotdata",
"argument": {
"data": {
"result": "plot",
"data": {
"filename": lastfile,
"graphs": data,
"stat": {}
}
}
}
}
self.plotdataqueue.put(request)
if np.mod(self.allp.value, 500) == 0:
self.timreport()
except KeyboardInterrupt:
pass
self.stop()
self.timreport()
return self.allp.value, time.time() - self.starttime
