def imageSubtraction(self, datFile, aveBuffer):
subtractedDatIntensities = []
subtractedDatq = []
subtractedErrors = []
for i in range(len(datFile.intensities)):
#Intensities
value = datFile.intensities[i] - aveBuffer["intensities"][i]
subtractedDatIntensities.insert(i, value)
#Q Values
subtractedDatq.insert(i, datFile.q[i])
subtractedErrors.insert(i, datFile.errors[i])
self.subtractedDatIntensities = subtractedDatIntensities
self.subtractedDatq = subtractedDatq
self.subtractedErrors = subtractedErrors
fileName = "sub_" + str(datFile.getFileName())
self.dbPush.send("sub_static_image")
self.dbPush.send(fileName)
self.EMBLmolSizePush.send("subtracted_dat")
self.EMBLmolSizePush.send(fileName)
self.datWriter.writeFile(self.absoluteLocation + "/sub/raw_sub/" , fileName , { 'q' : self.subtractedDatq, 'i' : self.subtractedDatIntensities, 'errors' : self.subtractedErrors})
logger(self.name, "Static Image Written ->" + fileName)
def imageSubtraction(self):
subtractedDatIntensities = []
subtractedDatq = []
subtractedErrors = []
for i in range(len(self.datFile.intensities)):
#Intensities
value = self.datFile.intensities[i] - self.aveBuffer["intensities"][i]
subtractedDatIntensities.insert(i, value)
#Q Values
subtractedDatq.insert(i, self.datFile.q[i])
subtractedErrors.insert(i, self.datFile.errors[i])
self.subtractedDatIntensities = subtractedDatIntensities
self.subtractedDatq = subtractedDatq
self.subtractedErrors = subtractedErrors
fileName = "average_"+self.datFile.getBaseFileName()
if (self.newSample_sub):
self.dbPush.send("subtracted_average_image")
self.dbPush.send(fileName)
self.newSample_sub = False
self.datWriter.writeFile(self.absoluteLocation + "/sub/" , fileName , { 'q' : self.subtractedDatq, 'i' : self.subtractedDatIntensities, 'errors' : self.subtractedErrors})
logger(self.name, "Static Image Written ->" + fileName)
def setUser(self, char_value = False, **kw):
try:
self.killWatchImageThread()
except:
pass
#TODO remove this, need another way to pass user directly
if not (char_value): #needed for cli, though my kill engine, if user monitor doesnt return a valid value
char_value = raw_input("Enter User: "******"User Changed -> " + str(self.user))
self.sendCommand("update_user")
self.sendCommand(self.user)
self.generateDirectoryStructure()
self.absoluteLocation = self.rootDirectory + self.user
self.logLocation = self.absoluteLocation + self.relativeLogFileLocation
self.datFileLocation = self.absoluteLocation + "/raw_dat/"
self.watchForImage(self.logLocation)
#Clear out some variables to be ready for the new user
self.lineIndex = 0
self.log = ""
self.sendCommand("absolute_location")
self.sendCommand(self.absoluteLocation)
def process(self, test):
if (str(test) == "test"):
logger(self.name, "RECIEVED - 'test' message")
if (str(test) == "static_image"):
self.datFile = self.pull.recv_pyobj()
logger(self.name, "Static Image Received")
self.imageSubtraction(self.datFile, self.aveBuffer)
def process(self, test):
if test == "test":
logger(self.name, "RECIEVED - 'test' message")
if test == "buffer":
buffer = self.pull.recv_pyobj()
logger(self.name, "RECIEVED - Buffer")
self.average(buffer)
def clear(self):
#Clear all lists TODO dictionaries/variables
for i in range(len(self.dataList)):
self.dataList[i] = []
self.needBuffer = True
print self.dataList
self.sampleIndex = 0
logger(self.name, "Cleared")
def connect(self, pullPort, dbPushPort):
self.pull.bind("tcp://127.0.0.1:"+str(pullPort))
if (dbPushPort):
self.dbPush.connect("tcp://127.0.0.1:"+str(dbPushPort))
logger(self.name, "All Ports Connected -> pullPort: "+str(pullPort) + " -> dbPushPort: "+str(dbPushPort))
else:
logger(self.name, "All Ports Connected -> pullPort: "+str(pullPort))
self.run()
def sendData(self):
try:
while True:
test = self.reply.recv() # wait for request of buffer
if test == "test":
self.reply.send_pyobj("REQUESTED DATA")
if test == "request_buffer":
logger(self.name, "BufferRequested")
v = self.getAverageBuffer()
if v["intensities"]:
self.reply.send_pyobj(self.getAverageBuffer())
else:
self.reply.send_pyobj("no_buffer")
except KeyboardInterrupt:
pass
def connect(self, pullPort, dbPushPort, replyPort):
self.pull.connect("tcp://127.0.0.1:" + str(pullPort))
self.reply.bind("tcp://127.0.0.1:" + str(replyPort))
if dbPushPort:
self.dbPush.connect("tcp://127.0.0.1:" + str(dbPushPort))
logger(
self.name,
"All Ports Connected -> pullPort: "
+ str(pullPort)
+ "-> replyPort: "
+ str(replyPort)
+ " -> dbPushPort: "
+ str(dbPushPort),
)
else:
logger(self.name, "All Ports Connected -> pullPort: " + str(pullPort) + "-> replyPort: " + str(replyPort))
logger(self.name, "All Ports Connected -> replyPort: " + str(replyPort))
replyThread = Thread(target=self.sendData)
replyThread.setDaemon(True)
replyThread.start()
self.run()
def average(self, datBuffer):
self.allIntensities.append(datBuffer.intensities)
self.allErrors.append(datBuffer.errors)
self.allQ.append(datBuffer.q)
# averaging out
self.avIntensities = self.ave.average(self.allIntensities)
self.avErrors = datBuffer.errors
self.avQ = datBuffer.q
fileName = "buffer" + str(self.index) + "_avg_" + str(datBuffer.getBaseFileName())
if self.changeInBuffer:
self.dbPush.send("buffer_file")
self.dbPush.send(fileName)
self.changeInBuffer = False
self.datWriter.writeFile(
self.absoluteLocation + "/avg/", fileName, {"q": self.avQ, "i": self.avIntensities, "errors": self.avErrors}
)
logger(self.name, "Averaging Completed")
def average(self):
self.allIntensities.append(self.datFile.intensities)
self.allQ.append(self.datFile.q)
self.allErrors.append(self.datFile.errors)
#averaging out
self.aveIntensities = self.ave.average(self.allIntensities)
self.aveQ = self.datFile.q
self.aveErrors = self.datFile.errors
logger(self.name, "Averaging Completed")
fileName = "sample_"+self.datFile.getBaseFileName()
if (self.newSample):
self.dbPush.send("average_image")
self.dbPush.send(fileName)
self.newSample = False
self.datWriter.writeFile(self.absoluteLocation+"/avg/", fileName, { 'q': self.aveQ, 'i' : self.aveIntensities, 'errors':self.aveErrors})
def __init__(self, name):
self.name = name
self.newSample = True #For writting out to db
self.newSample_sub = True
#need this to be able to save data etc
self.user = ""
self.experiment = ""
self.rootDirectory = ""
self.absoluteLocation = ""
self.sampleIndex = 0
self.aveBuffer = []
#ZMQ stuff
self.context = zmq.Context()
self.pull = self.context.socket(zmq.PULL)
self.dbPush = self.context.socket(zmq.PUSH)
self.EMBLmolSizePush = self.context.socket(zmq.PUSH)
#DatFile writer
self.datWriter = DatFileWriter.DatFileWriter()
#Averager
self.ave = AverageList.AverageList()
self.dataList = []
logger(self.name, "Generated")
def forceDBCreation(self, user):
logger(self.name, "Forcing Database Creation")
try:
db = mysql.connect(user="******", host="localhost", passwd="a")
c = db.cursor()
cmd = "CREATE DATABASE IF NOT EXISTS " + str(user) + ";"
c.execute(cmd)
logger(self.name, "Database Created for user: "******"Database CREATION FAILED for user:" + str(user))
raise
def getImage(self, imageFileName):
start_time = time.time()
#Get jsut the image/dat nme without extension
imageName = os.path.splitext(imageFileName)[0]
#add the .dat to image name, so it knows what its looking for
imageName = imageName + ".dat"
time.sleep(0.1) #Give it a little break to write out the dat file
logger(self.name, "getImage called, with %s" % imageName)
while not os.path.isfile(self.datFileLocation + imageName):
logger(self.name, "Waiting for: %s" % imageName)
time.sleep(0.5)
if time.time()-start_time > 3.0:
logger(self.name, "Timeout waiting for: %s" % imageName)
return
logger(self.name, "Retrieved: %s" % imageName)
self.datFile = DatFile.DatFile(self.datFileLocation + imageName)
self.processDatFile(self.datFile, self.logLines[-1])
def connect(self, pullPort, dbPushPort = None, EMBLmolSizePushPort = None):
self.pull.connect("tcp://127.0.0.1:"+str(pullPort))
if (EMBLmolSizePushPort):
self.EMBLmolSizePush.connect("tcp://127.0.0.1:"+str(pullPort))
logger(self.name, "EMBL mol Size push connected")
if (dbPushPort):
self.dbPush.connect("tcp://127.0.0.1:"+str(dbPushPort))
logger(self.name, "All Ports Connected -> pullPort: "+str(pullPort) + " -> dbPushPort: "+str(dbPushPort))
else:
logger(self.name, "All Ports Connected -> pullPort: "+str(pullPort))
self.run()
def close(self):
"""Close all zmq sockets"""
#time.sleep(0.1)
self.pull.close()
logger(self.name, "Sockets Closed")
sys.exit()
def run(self):
#if (self.reqBuffer != False):
#replyThread = Thread(target=self.sendData)
#replyThread.start()
try:
while True:
test = self.pull.recv()
#Generic Worker Control
if (str(test) == "update_user"):
logger(self.name, "Received Command - updateUser")
self.user = self.pull.recv()
logger(self.name, "New User -> " + self.user)
if (str(test) == "absolute_location"):
logger(self.name, "Received Command - absolute_location")
self.absoluteLocation = self.pull.recv()
logger(self.name, "Absolute Location: " + str(self.absoluteLocation))
if (str(test) == "getUser"):
logger(self.name, "Current User : "******"rootDirectory"):
self.rootDirectory = self.pull.recv()
logger(self.name, "Root Experiment Directory -> " + self.rootDirectory)
if (str(test) == "returnDirectory"):
logger(self.name, "Current Root Directory : " + self.rootDirectory)
if (str(test) == "new_buffer"):
self.newBuffer()
if (str(test) == "average_buffer"):
self.aveBuffer = self.pull.recv_pyobj()
print self.aveBuffer
logger(self.name, "Received average buffer")
if (str(test) == 'clear'):
self.clear()
if (str(test) == "exit"):
self.close()
if (str(test) == "test"):
logger(self.name, "Received TEST")
#Test shit
if (str(test) == "testPush"):
testString = self.pull.recv();
logger(self.name, "Test Pull/Push - Completed - String Received : " + testString)
else:
self.process(test)
except KeyboardInterrupt:
pass
def test(self):
logger(self.name, "Test Method preformed")
def processDatFile(self, datFile, logLine):
"""
Sample Types:
6 - Water
0 - Buffer
1 - Static Sample
"""
logger(self.name, "SampleType - " + str(logLine.getValue('SampleType')))
try:
if (changeInRootName(os.path.basename(self.logLines[-1].getValue("ImageLocation")), os.path.basename(self.logLines[-2].getValue("ImageLocation")))):
logger(self.name, "Change in root names")
if (logLine.getValue("SampleType") == "0"):
self.sendCommand("new_buffer")
self.needBuffer = True
#logger(self.name, "Buffer Received")
self.sendBuffer(datFile)
if (logLine.getValue("SampleType") == "1"):
if (self.needBuffer):
logger(self.name, "Need A Buffer")
if (self.requestAverageBuffer()):
self.sendAverageBuffer(self.aveBuffer)
self.needBuffer = False
self.sendImage(datFile)
else:
logger(self.name, "Request for averaged buffer failed, subtraction unable to occurr")
else:
self.sendImage(datFile)
logger(self.name, "Just Image Sent")
else:
logger(self.name, "NO CHANGE in root names")
if (logLine.getValue("SampleType") == "0"):
self.sendBuffer(datFile)
logger(self.name, "BUFFER SENT")
if (logLine.getValue("SampleType") == "1"):
if (self.needBuffer):
logger(self.name, "Need A Buffer")
if (self.requestAverageBuffer()):
self.sendAverageBuffer(self.aveBuffer)
self.needBuffer = False
self.sendImage(datFile)
else:
logger(self.name, "Request for averaged buffer failed, can not subtract datfile")
else:
self.sendImage(datFile)
logger(self.name, "No buffer needed")
#send just the image
except(IndexError):
logger(self.name, "index error, must be first pass")
def run(self):
try:
while True:
test = self.pull.recv()
#Generic Worker Control
if (str(test) == "update_user"):
logger(self.name, "Received Command - updateUser")
self.user = self.pull.recv()
self.forceDBCreation(self.user)
self.buildTables()
if (str(test) == "getUser"):
logger(self.name, "Current User : "******"log_line"):
logger(self.name, "GOT CALLED LOG LINE")
try:
logLine = self.pull.recv_pyobj()
except:
logger(self.name, "UNPICKLING ERROR - LINE MISSED TO DB")
self.writeLogToDB(logLine)
if (str(test) == "sub_static_image"):
loc = self.pull.recv()
self.writeSubtractionLocation(loc)
if (str(test) == "buffer_file"):
loc = self.pull.recv()
self.writeBufferLocation(loc)
if (str(test) == "subtracted_average_image"):
loc = self.pull.recv()
self.writeAveragedSubtactedLocation(loc)
if (str(test) == "average_image"):
loc = self.pull.recv()
self.writeAveragedLocation(loc)
if (str(test) == 'clear'):
self.clear()
if (str(test) == "exit"):
self.close()
if (str(test) == "test"):
logger(self.name, "Received TEST")
#Test shit
if (str(test) == "testPush"):
testString = self.pull.recv();
logger(self.name, "Test Pull/Push - Completed - String Received : " + testString)
except KeyboardInterrupt:
pass
def __init__(self, configuration):
self.name = "Engine"
#Get Configuration settings
try:
stream = file(configuration, 'r')
except IOError:
logger(self.name, "Unable to find configuration file (config.yaml, in current directory), exiting.")
exit()
self.config = yaml.load(stream)
self.rootDirectory = self.config['RootDirectory']
self.imageTakenPV = self.config['ImageTakenPV']
self.userChangePV = self.config['UserChangePV']
self.relativeLogFileLocation = self.config['RelativeLogFileLocation']
self.experimentName = self.config['ExperimentName']
self.logReader = ""
self.lineIndex = 0
self.log = ""
self.logLines = []
self.lines = []
#Will hold the latest created dat file
self.datFile = ""
self.needBuffer = True #Switch for requesting a new buffer
self.aveBuffer = "" #For holding the latest averaged buffer
self.absoluteLocation = "" #Properly Created with setuser, it is a concatenation of rootDirectory & user
self.logLocation = "" #Properly set in setUser also
self.datFileLocation = "" #Properly set in setUser
logger(self.name, "Engine Started")
#ZeroMQ setup stuff
self.context = zmq.Context()
#Current User
self.user = ""
#Default workers
self.bufferAverage = WorkerBufferAverage.WorkerBufferAverage()
self.staticImage = WorkerStaticImage.WorkerStaticImage()
self.rollingAverageSubtraction = WorkerRollingAverageSubtraction.WorkerRollingAverageSubtraction()
self.dbWorker = WorkerDB.WorkerDB()
self.WorkerEMBLmolSize = WorkerEMBLmolSize.WorkerEMBLmolSize()
#Connect Up all Workers, and have them ready
self.bufferRequest = self.context.socket(zmq.REQ)
self.bufferRequest.connect("tcp://127.0.0.1:5000")
logger(self.name, "Connected -> BufferRequest")
self.bufferPush = self.context.socket(zmq.PUSH)
self.bufferPush.bind("tcp://127.0.0.1:5001")
logger(self.name, "Binded -> BufferPush")
self.staticPush = self.context.socket(zmq.PUSH)
self.staticPush.connect("tcp://127.0.0.1:5002")
logger(self.name, "Binded -> StaticPush")
self.dbPush = self.context.socket(zmq.PUSH)
self.dbPush.connect("tcp://127.0.0.1:5003")
logger(self.name, "Binded -> dbPush")
self.rollingPush = self.context.socket(zmq.PUSH)
self.rollingPush.bind("tcp://127.0.0.1:5004")
logger(self.name, "Binded -> RollingPush")
self.EMBLmolSizePush = self.context.socket(zmq.PUSH)
self.EMBLmolSizePush.connect("tcp://127.0.0.1:5005")
logger(self.name, "Binded -> EMBL1Push")
time.sleep(0.5)
time.sleep(0.1)
bufferThread = Thread(target=self.bufferAverage.connect, args=(5001, 5003, 5000,))
bufferThread.setDaemon(True)
bufferThread.start()
time.sleep(0.1)
EMBL1Thread = Thread(target=self.WorkerEMBLmolSize.connect, args=(5005, 5003, ))
EMBL1Thread.setDaemon(True)
EMBL1Thread.start()
staticImageThread = Thread(target=self.staticImage.connect, args=(5002, 5003, 5005, ))
staticImageThread.setDaemon(True)
staticImageThread.start()
time.sleep(0.1)
rollingAverageThread = Thread(target=self.rollingAverageSubtraction.connect, args=(5004, 5003,))
rollingAverageThread.setDaemon(True)
rollingAverageThread.start()
time.sleep(0.1)
dbThread = Thread(target=self.dbWorker.connect, args=(5003, ))
dbThread.setDaemon(True)
dbThread.start()
time.sleep(0.5)
self.setRootDirectory()
self.watchForUserChangeOver()
time.sleep(0.1)
self.logLocation = self.absoluteLocation + self.relativeLogFileLocation
self.datFileLocation = self.absoluteLocation + "/raw_dat/"
logger(self.name, "All Workers ready")
self.dbPush.send("test")
#Force setUser()
#TODO: REMOVE this, shouldnt really be set here
self.absoluteLocation = self.rootDirectory + self.user
self.logLocation = self.absoluteLocation + self.relativeLogFileLocation
self.datFileLocation = self.absoluteLocation + "/raw_dat/"
#Start this thread last
cliThread = Thread(target=self.cli())
cliThread.setDaemon(True)
cliThread.start()
def testRequest(self):
self.bufferRequest.send("test")
test = self.bufferRequest.recv_pyobj()
logger(self.name, "RESPONSE RECIEVED -> " + test)
def generateDirectoryStructure(self):
dirCreator = DirectoryCreator.DirectoryCreator(self.rootDirectory)
dirCreator.createFolderStructure(self.user, "experiment1")
logger(self.name, "Generated Directory Structure")
def process(self, test):
if (str(test) == "subtracted_dat"):
datFile = self.pull.recv()
logger(self.name, "About to process DatFile: " + str(datFile))
self.processDatFile(datFile)
def returnUser(self):
logger(self.name, "Current User : "******"getUser")
def exitEngine(self):
self.sendCommand("exit")
time.sleep(0.2)
logger(self.name, "Exiting")
sys.exit()
def close(self):
"""Close all zmq sockets"""
self.pull.close()
self.reply.close()
logger(self.name, "Sockets Closed")
sys.exit()