class QtStreamHandler(logging.Handler):
def __init__(self, parent, main):
logging.Handler.__init__(self)
self.parent = parent
self.main = main
self.textWidget = parent
self.formater = logging.Formatter("%(asctime)s - %(levelname)s -> %(message)s")
self.logString = ""
def getLoggingCompleteText(self):
return self.logString
def createLock(self):
self.mutex = QMutex()
def acquire(self):
self.mutex.lock()
def release(self):
self.mutex.unlock()
def emit(self, record):
# self.textWidget.appendPlainText(self.formater.format(record))
formattedText = self.formater.format(record)
#separate text from date
index = formattedText.find('->')
self.textWidget.setTextColor(QColor("green"))
self.textWidget.insertPlainText(formattedText[:index])
self.textWidget.setTextColor(QColor("blue"))
self.textWidget.insertPlainText(formattedText[index:] + "\n")
self.textWidget.moveCursor(QTextCursor.EndOfLine)
self.textWidget.ensureCursorVisible()
self.logString += formattedText + "\n";
class MainW(QtGui.QMainWindow):
"""
Mainw to host and run one single thread
"""
def __init__(self):
QtGui.QMainWindow.__init__(self)
self.mutex = QMutex()
self.thr = GuiThread(process=self.fun, period=0.1)
self.thr2 = GuiThread(process=self.fun2, period=0.1)
self.buff = CircIoBuffer(byte_size=15 * len('rafal miecznik'))
self.thr.start()
self.thr2.start()
time.sleep(2)
print(self.buff.read())
for d in dir(self.mutex):
print(d)
def fun(self):
print(self.mutex.tryLock())
self.buff.write(' rafal')
self.mutex.unlock()
#print self.thr.currentThreadId()
def fun2(self):
self.mutex.lock()
self.buff.write(' miecznik')
class Feed(QRunnable):
def __init__(self, label, parent=None):
QRunnable.__init__(self)
self.label = label
self.parent = parent
self.data = None
self.mutex = QMutex()
self.setAutoDelete(True)
def run(self):
self.check()
def check(self):
try:
self.fetch()
except (urllib2.URLError, urllib2.HTTPError) as err:
self.parent.error.emit(str(err))
else:
self.read()
if self.data:
self.parse()
def fetch(self):
auth_handler = urllib2.HTTPBasicAuthHandler()
auth_handler.add_password(REALM, HOST, self.parent.user,
self.parent.passwd)
opener = urllib2.build_opener(auth_handler)
urllib2.install_opener(opener)
url = "%s/%s" % (URL_ATOM, self.label) if self.label else URL_ATOM
self.conn = urllib2.urlopen(url)
def read(self):
code = self.conn.getcode()
if code != 200:
self.conn.close()
self.parent.error.emit(
"HTTP Error %d: %s" %
(code, BaseHTTPRequestHandler.responses[code]))
else:
self.data = self.conn.read()
self.conn.close()
def parse(self):
tree = et.fromstring(self.data)
for e in tree.findall("%sentry" % XMLNS):
entry = Entry()
entry.title = e.find("%stitle" % XMLNS).text
entry.summary = e.find("%ssummary" % XMLNS).text
entry.link = e.find("%slink" % XMLNS).attrib["href"]
entry.modified = e.find("%smodified" % XMLNS).text
entry.issued = e.find("%sissued" % XMLNS).text
entry.id = e.find("%sid" % XMLNS).text
for a in e.findall("%sauthor" % XMLNS):
entry.author_name = a.find("%sname" % XMLNS).text
entry.author_email = a.find("%semail" % XMLNS).text
self.mutex.lock()
self.parent.entries.append((self.label, entry))
self.mutex.unlock()
class Feed(QRunnable):
def __init__(self, label, parent=None):
QRunnable.__init__(self)
self.label = label
self.parent = parent
self.data = None
self.mutex = QMutex()
self.setAutoDelete(True)
def run(self):
self.check()
def check(self):
try:
self.fetch()
except (urllib2.URLError, urllib2.HTTPError) as err:
self.parent.error.emit(str(err))
else:
self.read()
if self.data:
self.parse()
def fetch(self):
auth_handler = urllib2.HTTPBasicAuthHandler()
auth_handler.add_password(REALM, HOST,
self.parent.user, self.parent.passwd)
opener = urllib2.build_opener(auth_handler)
urllib2.install_opener(opener)
url = "%s/%s" % (URL_ATOM, self.label) if self.label else URL_ATOM
self.conn = urllib2.urlopen(url)
def read(self):
code = self.conn.getcode()
if code != 200:
self.conn.close()
self.parent.error.emit("HTTP Error %d: %s" % (
code, BaseHTTPRequestHandler.responses[code]))
else:
self.data = self.conn.read()
self.conn.close()
def parse(self):
tree = et.fromstring(self.data)
for e in tree.findall("%sentry" % XMLNS):
entry = Entry()
entry.title = e.find("%stitle" % XMLNS).text
entry.summary = e.find("%ssummary" % XMLNS).text
entry.link = e.find("%slink" % XMLNS).attrib["href"]
entry.modified = e.find("%smodified" % XMLNS).text
entry.issued = e.find("%sissued" % XMLNS).text
entry.id = e.find("%sid" % XMLNS).text
for a in e.findall("%sauthor" % XMLNS):
entry.author_name = a.find("%sname" % XMLNS).text
entry.author_email = a.find("%semail" % XMLNS).text
self.mutex.lock()
self.parent.entries.append((self.label, entry))
self.mutex.unlock()
class QCallbacksManager(QObject):
class Request(object):
def __init__(self):
self.event = Event()
self.answer = None
def __call__(self):
raise NotImplementedError()
class LoginRequest(Request):
def __init__(self, backend_name, value):
QCallbacksManager.Request.__init__(self)
self.backend_name = backend_name
self.value = value
def __call__(self):
password, ok = QInputDialog.getText(
None, '%s request' % self.value.label,
'Please enter %s for %s' %
(self.value.label, self.backend_name), QLineEdit.Password)
return password
new_request = Signal()
def __init__(self, weboob, parent=None):
QObject.__init__(self, parent)
self.weboob = weboob
self.weboob.requests.register('login',
self.callback(self.LoginRequest))
self.mutex = QMutex()
self.requests = []
self.new_request.connect(self.do_request)
def callback(self, klass):
def cb(*args, **kwargs):
return self.add_request(klass(*args, **kwargs))
return cb
@Slot()
def do_request(self):
self.mutex.lock()
request = self.requests.pop()
request.answer = request()
request.event.set()
self.mutex.unlock()
def add_request(self, request):
self.mutex.lock()
self.requests.append(request)
self.mutex.unlock()
self.new_request.emit()
request.event.wait()
return request.answer
class QCallbacksManager(QObject):
class Request(object):
def __init__(self):
self.event = Event()
self.answer = None
def __call__(self):
raise NotImplementedError()
class LoginRequest(Request):
def __init__(self, backend_name, value):
QCallbacksManager.Request.__init__(self)
self.backend_name = backend_name
self.value = value
def __call__(self):
password, ok = QInputDialog.getText(None,
'%s request' % self.value.label,
'Please enter %s for %s' % (self.value.label,
self.backend_name),
QLineEdit.Password)
return password
new_request = Signal()
def __init__(self, weboob, parent=None):
QObject.__init__(self, parent)
self.weboob = weboob
self.weboob.callbacks['login'] = self.callback(self.LoginRequest)
self.mutex = QMutex()
self.requests = []
self.new_request.connect(self.do_request)
def callback(self, klass):
def cb(*args, **kwargs):
return self.add_request(klass(*args, **kwargs))
return cb
@Slot()
def do_request(self):
self.mutex.lock()
request = self.requests.pop()
request.answer = request()
request.event.set()
self.mutex.unlock()
def add_request(self, request):
self.mutex.lock()
self.requests.append(request)
self.mutex.unlock()
self.new_request.emit()
request.event.wait()
return request.answer
class MyStreamHandler(StreamHandler):
def __init__(self):
StreamHandler.__init__(self)
def createLock(self):
# must be Recursive (= reentrant)
self._mutex = QMutex(QMutex.Recursive)
def acquire(self):
self._mutex.lock()
def release(self):
self._mutex.unlock()
class MyStreamHandler(StreamHandler):
def __init__(self):
StreamHandler.__init__(self)
def createLock(self):
# must be Recursive (= reentrant)
self._mutex = QMutex(QMutex.Recursive)
def acquire(self):
self._mutex.lock()
def release(self):
self._mutex.unlock()
class VCSPluginThread( QThread ):
" Wrapper for the plugin thread "
def __init__( self, plugin, parent = None ):
QThread.__init__( self, parent )
self.__plugin = plugin
self.__requestQueue = deque()
self.__stopRequest = False
self.__lock = QMutex()
self.__condition = QWaitCondition()
return
def run( self ):
" Thread loop "
while not self.__stopRequest:
self.__lock.lock()
while self.__requestQueue:
path, flag = self.__requestQueue.pop()
self.__lock.unlock()
time.sleep( 0.01 )
self.__processRequest( path, flag )
if self.__stopRequest:
break
self.__lock.lock()
if self.__stopRequest:
self.__lock.unlock()
break
self.__condition.wait( self.__lock )
self.__lock.unlock()
return
def __processRequest( self, path, flag ):
" Processes a single request. It must be exception safe. "
try:
statuses = self.__plugin.getObject().getStatus( path, flag )
for status in statuses:
if len( status ) == 3:
self.emit( SIGNAL( "VCSStatus" ), path + status[ 0 ],
status[ 1 ], status[ 2 ] )
else:
self.emit( SIGNAL( "VCSStatus" ), path, IND_VCS_ERROR,
"The " + self.__plugin.getName() + " plugin "
"does not follow the getStatus() interface "
"agreement" )
except Exception, exc:
self.emit( SIGNAL( "VCSStatus" ), path, IND_VCS_ERROR,
"Exception in " + self.__plugin.getName() +
" plugin while retrieving VCS status: " + str( exc ) )
except:
class Lock(object):
def __init__(self):
self._mutex = QMutex()
def __enter__(self):
self._mutex.lock()
def __exit__(self, type, value, tb):
self._mutex.unlock()
def acquire(self):
self._mutex.lock()
def release(self):
self._mutex.unlock()
class Lock(object):
def __init__(self):
self._mutex = QMutex()
def __enter__(self):
self._mutex.lock()
def __exit__(self, type, value, tb):
self._mutex.unlock()
def acquire(self):
self._mutex.lock()
def release(self):
self._mutex.unlock()
class Evaluate(QThread):
"""Thread used to insolate calculation process in entities (stream, project
and equipment, so gui can response while calculation is in process"""
def __init__(self, parent=None):
super(Evaluate, self).__init__(parent)
self.mutex = QMutex()
def start(self, entity, kwargs):
self.entity = entity
self.kwargs = kwargs
QThread.start(self)
def run(self):
self.mutex.lock()
self.entity(**self.kwargs)
self.mutex.unlock()
class SQLRenderThread(CachedRenderThread):
"""
RenderThread extending the UniqueMethodRenderThread by supplying a cancel
method for classes with database access. The database object needs to be
given either on the object as .db subobject or explicitly by
setObjectDBObject().
The database object needs to have a 'connection' attribute which supports an
interrupt() method.
"""
def __init__(self, parent=0):
self.dbObjectLock = QMutex(QMutex.Recursive)
self.dbObject = {}
CachedRenderThread.__init__(self, parent)
def setObject(self, classObject, *args, **param):
self.dbObjectLock.lock()
if classObject in self.dbObject:
del self.dbObject[classObject]
self.dbObjectLock.unlock()
CachedRenderThread.setObject(self, classObject, *args, **param)
def setObjectDBObject(self, classObject, db):
self.dbObjectLock.lock()
self.dbObject[classObject] = db
self.dbObjectLock.unlock()
def cancelCurrentJob(self):
QMutexLocker(self.renderingLock)
if self.currentlyRenderingJob:
_, classObject, _, _, _ = self.currentlyRenderingJob
try:
QMutexLocker(self.dbObjectLock)
if classObject in self.dbObject:
db = self.dbObject[classObject]
else:
classObjectInst = self.getObjectInstance(classObject)
if hasattr(classObjectInst, 'db'):
db = classObjectInst.db
else:
return False
self.clearCurrentJob()
db.connection.interrupt()
return True
except KeyError:
return False
class ComputationThread(QThread):
def __init__(self, animation_thread):
QThread.__init__(self)
self.is_running = True
self.mutex = QMutex()
self.other_thread = animation_thread
def stop(self):
self.is_running = False
def run(self):
while self.is_running:
self.mutex.lock() # locking may not be necessary
raw_input("")
#self.other_thread.stop( )
self.mutex.unlock()
print " run method ends "
class UdpClient(QThread): def __init__(self, parent): QThread.__init__(self, parent) self.prnt = parent self.udp = QUdpSocket() addr = QHostAddress(QHostAddress.Any) print 'bind to:', addr.toString() self.udp.bind(addr, 34001) self.udp.error.connect(self.errorAnalyser) self.udp.readyRead.connect(self.readUdp) print "Binding..." self.STOP = False self.locker = QMutex() def run(self): self.prnt.initAvahiService() while True : if self.udp is not None and self.udp.state() == QAbstractSocket.ConnectedState : self.udp.waitForReadyRead() else : self.msleep(100) if self.STOP and self.udp is not None: self.udp.close(); break print 'UDPClient closed...' self.prnt.changeConnectState.emit() def stop(self): self.locker.lock() self.STOP = True self.locker.unlock() def readUdp(self): while ( self.udp.hasPendingDatagrams() ): data = QByteArray() addr = QHostAddress() port = 0 try : datagramSize = self.udp.pendingDatagramSize() if datagramSize > 0 : (data, addr, port) = self.udp.readDatagram(datagramSize) #print "Datagram: [%s] from %s:%i" % (QString().fromUtf8(data), addr.toString(), port) self.prnt.contactMessage.emit(QString().fromUtf8(data), addr.toString()) except socket.error, err : print '[in readUdp() UdpClient] SocketError1 : ', err except socket.timeout, err : print '[in readUdp() UdpClient] SocketError2 : ', err except :
class ImageTile(object):
def __init__(self, rect):
self._mutex = QMutex()
self.image = QImage(rect.width(), rect.height(),
QImage.Format_ARGB32_Premultiplied)
self.image.fill(0x00000000)
self._topLeft = rect.topLeft()
# Whenever the underlying data changes, the data version is incremented.
# By comparing the data version to the image and request version, it can
# be determined if the content of this tile is recent or needs to be
# re-computed.
# version of the data
self.dataVer = 0
# version of self.image
#
# If self.imgVer < self.dataVer, the image needs to be re-computed
# from the new data.
self.imgVer = -1
# version of the request that has been generated to update the contents
# of self.image
#
# If self.reqVer == self.dataVer, a request is currently running that will
# eventually replace self.image with the new data.
self.reqVer = -2
def clear(self):
self.image.fill(0)
def paint(self, painter):
self.lock()
painter.drawImage(self._topLeft, self.image)
self.unlock()
def lock(self):
self._mutex.lock()
def unlock(self):
self._mutex.unlock()
class ImageTile(object):
def __init__(self, rect):
self._mutex = QMutex()
self.image = QImage(rect.width(), rect.height(),
QImage.Format_ARGB32_Premultiplied)
self.image.fill(0)
self._topLeft = rect.topLeft()
#Whenever the underlying data changes, the data version is incremented.
#By comparing the data version to the image and request version, it can
#be determined if the content of this tile is recent or needs to be
#re-computed.
#version of the data
self.dataVer = 0
#version of self.image
#
#If self.imgVer < self.dataVer, the image needs to be re-computed
#from the new data.
self.imgVer = -1
#version of the request that has been generated to update the contents
#of self.image
#
#If self.reqVer == self.dataVer, a request is currently running that will
#eventually replace self.image with the new data.
self.reqVer = -2
def clear(self):
self.image.fill(0)
def paint(self, painter):
self.lock()
painter.drawImage(self._topLeft, self.image)
self.unlock()
def lock(self):
self._mutex.lock()
def unlock(self):
self._mutex.unlock()
class DatabaseThread(QThread):
def __init__(self, framework, Data, parent=None):
QThread.__init__(self, parent)
self.framework = framework
self.Data = Data
self.qlock = QMutex()
QObject.connect(self, SIGNAL('quit()'), self.quitHandler)
QObject.connect(self, SIGNAL('started()'), self.startedHandler)
def run(self):
QObject.connect(self, SIGNAL('doConnectDb()'), self.connectDbHandler,
Qt.DirectConnection)
self.exec_()
def close(self):
self.qlock.lock()
try:
self.Data.close()
finally:
self.qlock.unlock()
def quitHandler(self):
self.Data.close()
self.exit(0)
def startedHandler(self):
pass
def connectDb(self, filename, callback):
self.filename = filename
self.callbackObj = callback
QTimer.singleShot(50, self, SIGNAL('doConnectDb()'))
def connectDbHandler(self):
self.qlock.lock()
try:
self.Data.connect(self.filename)
finally:
self.qlock.unlock()
self.callbackObj.emit(SIGNAL('connectDbFinished()'))
class _LoggingMutexQt(_LoggingMutex):
def __init__(self, name):
super(_LoggingMutexQt, self).__init__(name)
from PyQt4.QtCore import QMutex
self.mutex = QMutex()
def currentThreadID(self):
from PyQt4.QtCore import QThread
return QThread.currentThreadId()
def _acquire(self):
self.mutex.lock()
def _release(self):
self.mutex.unlock()
def enterMutex(self):
self.mutex.lock()
def exitMutex(self, *_args, **_kwargs):
self.mutex.unlock()
class AnimationThread(QThread):
def __init__(self):
QThread.__init__(self)
self.is_running = True
self.mutex = QMutex()
def stop(self):
self.is_running = False
def get_state(self):
return self.is_running
def change_state(self):
self.is_running = not self.is_running
def run(self):
while self.is_running:
self.mutex.lock( ) # locking may not be necessary
self.emit(SIGNAL("window_update_request" ))
self.mutex.unlock()
sleep( 0.005 )
class SignalThread(QThread):
def __init__(self, parent=None):
QThread.__init__(self, parent)
self.waitcond = QWaitCondition()
self.mutex = QMutex()
self.isstopped = False
def trigger(self):
"""lock first to make sure the QThread is actually waiting for a signal"""
self.mutex.lock()
self.waitcond.wakeOne()
self.mutex.unlock()
def stopThread(self):
self.mutex.lock()
self.isstopped = True
self.waitcond.wakeOne()
self.mutex.unlock()
def run(self):
self.mutex.lock()
while not self.isstopped:
# just wait, and trigger every time we receive a signal
self.waitcond.wait(self.mutex)
if not self.isstopped:
self.emit(SIGNAL("triggerSignal()"))
self.mutex.unlock()
class communications:
def __init__(self):
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.s.connect((TCP_IP, TCP_PORT))
self.running = QMutex()
self.send_command = '.'
def command(self, comm):
self.running.lock()
self.send_command = comm
self.running.unlock()
def get_data(self):
self.running.lock()
self.s.send(self.send_command.encode())
values = pickle.loads(self.s.recv(BUFFER_SIZE))
self.send_command = '.'
self.running.unlock()
return values
def get_parameters(self):
self.running.lock()
self.s.send(b"?")
params = pickle.loads(self.s.recv(BUFFER_SIZE))
self.running.unlock()
return params
def __del__(self):
self.s.close()
class SignalThread(QThread):
def __init__(self, parent = None):
QThread.__init__(self, parent)
self.waitcond = QWaitCondition()
self.mutex = QMutex()
self.isstopped = False
def trigger(self):
"""lock first to make sure the QThread is actually waiting for a signal"""
self.mutex.lock()
self.waitcond.wakeOne()
self.mutex.unlock()
def stopThread(self):
self.mutex.lock()
self.isstopped = True
self.waitcond.wakeOne()
self.mutex.unlock()
def run(self):
self.mutex.lock()
while not self.isstopped:
# just wait, and trigger every time we receive a signal
self.waitcond.wait(self.mutex)
if not self.isstopped:
self.emit(SIGNAL("triggerSignal()"))
self.mutex.unlock()
class DatabaseThread(QThread):
def __init__(self, framework, Data, parent = None):
QThread.__init__(self, parent)
self.framework = framework
self.Data = Data
self.qlock = QMutex()
QObject.connect(self, SIGNAL('quit()'), self.quitHandler)
QObject.connect(self, SIGNAL('started()'), self.startedHandler)
def run(self):
QObject.connect(self, SIGNAL('doConnectDb()'), self.connectDbHandler, Qt.DirectConnection)
self.exec_()
def close(self):
self.qlock.lock()
try:
self.Data.close()
finally:
self.qlock.unlock()
def quitHandler(self):
self.Data.close()
self.exit(0)
def startedHandler(self):
pass
def connectDb(self, filename, callback):
self.filename = filename
self.callbackObj = callback
QTimer.singleShot(50, self, SIGNAL('doConnectDb()'))
def connectDbHandler(self):
self.qlock.lock()
try:
self.Data.connect(self.filename)
finally:
self.qlock.unlock()
self.callbackObj.emit(SIGNAL('connectDbFinished()'))
class Imap(QRunnable):
def __init__(self, label, parent=None):
QRunnable.__init__(self)
self.imap = None
self.label = label
self.parent = parent
self.mutex = QMutex()
self.login()
def run(self):
for imap_id in self.get_unread():
thread_id = self.get_thread_id(imap_id)
self.mutex.lock()
self.parent.idmap[thread_id] = imap_id
self.mutex.unlock()
def connect(self):
self.imap = imaplib.IMAP4_SSL(IMAP_SERVER, IMAP_PORT)
def login(self):
try:
if not self.imap:
self.connect()
auth = self.imap.login(self.parent.user, self.parent.passwd)
self.logged_in = (auth and auth[0] == "OK")
except imaplib.IMAP4.error:
pass
def logout(self):
pass
def get_unread(self):
self.imap.select(self.label.replace("-", "/"))
result, unseen = self.imap.search(None, "(UNSEEN)")
return unseen[0].split()
def get_thread_id(self, id):
result, data = self.imap.fetch(id, "(X-GM-THRID BODY.PEEK[HEADER])")
return data[0][0].split(" ")[2]
class Imap(QRunnable):
def __init__(self, label, parent=None):
QRunnable.__init__(self)
self.imap = None
self.label = label
self.parent = parent
self.mutex = QMutex()
self.login()
def run(self):
for imap_id in self.get_unread():
thread_id = self.get_thread_id(imap_id)
self.mutex.lock()
self.parent.idmap[thread_id] = imap_id
self.mutex.unlock()
def connect(self):
self.imap = imaplib.IMAP4_SSL(IMAP_SERVER, IMAP_PORT)
def login(self):
try:
if not self.imap:
self.connect()
auth = self.imap.login(self.parent.user, self.parent.passwd)
self.logged_in = (auth and auth[0] == "OK")
except imaplib.IMAP4.error:
pass
def logout(self):
pass
def get_unread(self):
self.imap.select(self.label.replace("-", "/"))
result, unseen = self.imap.search(None, "(UNSEEN)")
return unseen[0].split()
def get_thread_id(self, id):
result, data = self.imap.fetch(id, "(X-GM-THRID BODY.PEEK[HEADER])")
return data[0][0].split(" ")[2]
class WorkerThread(QThread):
do_something = pyqtSignal()
def __init__(self, name="", parent=None):
super().__init__(parent)
self.sleep_interval = 900
self.mutex = QMutex()
self.name = name
self.isStopped = False
def run(self):
while not self.isStopped:
# while True:
self.mutex.lock()
self.do_something.emit()
self.mutex.unlock()
self.msleep(self.sleep_interval)
self.finished.emit()
print(self.name + " finished.")
def stop(self):
with QMutexLocker(self.mutex):
self.isStopped = True
class TypeWorker(QThread):
def __init__(self, *args):
QThread.__init__(self)
self.typeQueue = Queue()
self.regImage = re.compile("(JPEG|JPG|jpg|jpeg|GIF|gif|bmp|BMP|png|PNG|pbm|PBM|pgm|PGM|ppm|PPM|xpm|XPM|xbm|XBM|TIFF|tiff).*", re.IGNORECASE)
self.typeQueue = []
self.setUniq = set()
self.qmutex = QMutex()
self.qsem = QSemaphore()
def enqueue(self, parent, index, node):
self.qmutex.lock()
if long(node.this) not in self.setUniq:
self.typeQueue.insert(0, (parent, index, node))
self.setUniq.add(long(node.this))
self.qsem.release()
self.qmutex.unlock()
def clear(self):
self.qmutex.lock()
self.typeQueue = []
self.setUniq.clear()
self.qsem.acquire(self.qsem.available())
self.qmutex.unlock()
def get(self):
self.qsem.acquire()
self.qmutex.lock()
res = self.typeQueue.pop()
self.setUniq.remove(long(res[2].this))
self.qmutex.unlock()
return res
def isImage(self, ftype):
res = self.regImage.search(ftype)
return res
def run(self):
count = 0
while True:
(parent, index, node) = self.get()
if node.size():
ftype = str(node.dataType())
if parent.imagesthumbnails and self.isImage(ftype):
thumb = ImageThumb()
img = thumb.getImage(ftype, node, index)
if img:
parent.emit(SIGNAL('dataImage'), index, node, img)
class ExtractThread(QThread):
def __init__(self, files, needPrj):
QThread.__init__(self, QThread.currentThread())
self.inFiles = files
self.needPrj = needPrj
self.mutex = QMutex()
self.stopMe = 0
def run(self):
self.mutex.lock()
self.stopMe = 0
self.mutex.unlock()
interrupted = False
for f in self.inFiles:
extractProjection(f, self.needPrj)
self.emit(SIGNAL("fileProcessed()"))
self.mutex.lock()
s = self.stopMe
self.mutex.unlock()
if s == 1:
interrupted = True
break
if not interrupted:
self.emit(SIGNAL("processFinished()"))
else:
self.emit(SIGNAL("processIterrupted()"))
def stop(self):
self.mutex.lock()
self.stopMe = 1
self.mutex.unlock()
QThread.wait(self)
class ExtractThread(QThread):
def __init__(self, files, needPrj):
QThread.__init__(self, QThread.currentThread())
self.inFiles = files
self.needPrj = needPrj
self.mutex = QMutex()
self.stopMe = 0
def run(self):
self.mutex.lock()
self.stopMe = 0
self.mutex.unlock()
interrupted = False
for f in self.inFiles:
extractProjection(f, self.needPrj)
self.emit(SIGNAL("fileProcessed()"))
self.mutex.lock()
s = self.stopMe
self.mutex.unlock()
if s == 1:
interrupted = True
break
if not interrupted:
self.emit(SIGNAL("processFinished()"))
else:
self.emit(SIGNAL("processIterrupted()"))
def stop(self):
self.mutex.lock()
self.stopMe = 1
self.mutex.unlock()
QThread.wait(self)
class Search_worker(QThread):
"""
Defines a `QThread <http://doc.qt.nokia.com/qthread.html>`_ subclass used
to search for a pattern in a directory files.
"""
# Custom signals definitions.
searchFinished = pyqtSignal(list)
"""
This signal is emited by the :class:`Search_worker` class when the search is finished. ( pyqtSignal )
:return: Search results.
:rtype: list
"""
def __init__(self, parent, pattern=None, location=None, settings=None):
"""
Initializes the class.
:param parent: Object parent.
:type parent: QObject
"""
LOGGER.debug("> Initializing '{0}()' class.".format(self.__class__.__name__))
QThread.__init__(self, parent)
# --- Setting class attributes. ---
self.__container = parent
self.__pattern = None
self.pattern = pattern
self.__location = None
self.location = location
self.__settings = None
self.settings = settings
self.__searchResults = None
self.__interrupt = False
self.__lock = QMutex()
#******************************************************************************************************************
#*** Attributes properties.
#******************************************************************************************************************
@property
def container(self):
"""
Property for **self.__container** attribute.
:return: self.__container.
:rtype: QObject
"""
return self.__container
@container.setter
@foundations.exceptions.handleExceptions(foundations.exceptions.ProgrammingError)
def container(self, value):
"""
Setter for **self.__container** attribute.
:param value: Attribute value.
:type value: QObject
"""
raise foundations.exceptions.ProgrammingError(
"{0} | '{1}' attribute is read only!".format(self.__class__.__name__, "container"))
@container.deleter
@foundations.exceptions.handleExceptions(foundations.exceptions.ProgrammingError)
def container(self):
"""
Deleter for **self.__container** attribute.
"""
raise foundations.exceptions.ProgrammingError(
"{0} | '{1}' attribute is not deletable!".format(self.__class__.__name__, "container"))
@property
def pattern(self):
"""
Property for **self.__pattern** attribute.
:return: self.__pattern.
:rtype: unicode
"""
return self.__pattern
@pattern.setter
@foundations.exceptions.handleExceptions(AssertionError)
def pattern(self, value):
"""
Setter for **self.__pattern** attribute.
:param value: Attribute value.
:type value: unicode
"""
if value is not None:
assert type(value) in (unicode, QString), \
"'{0}' attribute: '{1}' type is not 'unicode' or 'QString'!".format("pattern", value)
self.__pattern = value
@pattern.deleter
@foundations.exceptions.handleExceptions(foundations.exceptions.ProgrammingError)
def pattern(self):
"""
Deleter for **self.__pattern** attribute.
"""
raise foundations.exceptions.ProgrammingError(
"{0} | '{1}' attribute is not deletable!".format(self.__class__.__name__, "pattern"))
@property
def location(self):
"""
Property for **self.__location** attribute.
:return: self.__location.
:rtype: Location
"""
return self.__location
@location.setter
@foundations.exceptions.handleExceptions(AssertionError)
def location(self, value):
"""
Setter for **self.__location** attribute.
:param value: Attribute value.
:type value: Location
"""
if value is not None:
assert type(value) is umbra.ui.common.Location, \
"'{0}' attribute: '{1}' type is not 'umbra.ui.common.Location'!".format("location", value)
self.__location = value
@location.deleter
@foundations.exceptions.handleExceptions(foundations.exceptions.ProgrammingError)
def location(self):
"""
Deleter for **self.__location** attribute.
"""
raise foundations.exceptions.ProgrammingError(
"{0} | '{1}' attribute is not deletable!".format(self.__class__.__name__, "location"))
@property
def settings(self):
"""
Property for **self.__settings** attribute.
:return: self.__settings.
:rtype: Location
"""
return self.__settings
@settings.setter
@foundations.exceptions.handleExceptions(AssertionError)
def settings(self, value):
"""
Setter for **self.__settings** attribute.
:param value: Attribute value.
:type value: Location
"""
if value is not None:
assert type(value) is dict, "'{0}' attribute: '{1}' type is not 'dict'!".format("settings", value)
self.__settings = foundations.dataStructures.Structure(**{"caseSensitive" : False,
"wholeWord" : False,
"regularExpressions" : False})
self.__settings.update(value)
@settings.deleter
@foundations.exceptions.handleExceptions(foundations.exceptions.ProgrammingError)
def settings(self):
"""
Deleter for **self.__settings** attribute.
"""
raise foundations.exceptions.ProgrammingError(
"{0} | '{1}' attribute is not deletable!".format(self.__class__.__name__, "settings"))
@property
def searchResults(self):
"""
Property for **self.__searchResults** attribute.
:return: self.__searchResults.
:rtype: list
"""
return self.__searchResults
@searchResults.setter
@foundations.exceptions.handleExceptions(foundations.exceptions.ProgrammingError)
def searchResults(self, value):
"""
Setter for **self.__searchResults** attribute.
:param value: Attribute value.
:type value: list
"""
raise foundations.exceptions.ProgrammingError(
"{0} | '{1}' attribute is read only!".format(self.__class__.__name__, "searchResults"))
@searchResults.deleter
@foundations.exceptions.handleExceptions(foundations.exceptions.ProgrammingError)
def searchResults(self):
"""
Deleter for **self.__searchResults** attribute.
"""
raise foundations.exceptions.ProgrammingError(
"{0} | '{1}' attribute is not deletable!".format(self.__class__.__name__, "searchResults"))
#******************************************************************************************************************
#*** Class methods.
#******************************************************************************************************************
def run(self):
"""
Reimplements the :meth:`QThread.run` method.
"""
self.__search()
def quit(self):
"""
Reimplements the :meth:`QThread.quit` method.
"""
self.__interrupt = True
QThread.quit(self)
def __search(self):
"""
Performs the search.
"""
self.__searchResults = []
editorsFiles = self.__container.defaultTarget in self.__location.targets and \
[editor.file for editor in self.__container.scriptEditor.listEditors()] or []
self.__searchEditorsFiles(editorsFiles)
self.__searchFiles(self.__location.files)
for directory in self.__location.directories:
if self.__interrupt:
return
filesWalker = foundations.walkers.filesWalker(directory,
self.__location.filtersIn,
list(itertools.chain(self.__location.filtersOut,
self.__location.files,
editorsFiles)))
self.__searchFiles(filesWalker)
not self.__interrupt and self.searchFinished.emit(self.__searchResults)
def __searchEditorsFiles(self, files):
"""
Searches in :class:`umbra.components.factory.scriptEditor.scriptEditor.ScriptEditor` class editors files.
:param files: Editor files.
:type files: list
"""
for file in files:
if self.__interrupt:
return
if foundations.io.isReadable(file):
if foundations.io.isBinaryFile(file):
continue
LOGGER.info("{0} | Searching '{1}' file!".format(self.__class__.__name__, file))
editor = self.__container.scriptEditor.getEditor(file)
if not editor:
continue
self.__lock.lock()
occurrences = self.__searchDocument(editor.document(), self.__pattern, self.__settings)
self.__lock.unlock()
occurrences and self.__searchResults.append(SearchResult(file=file,
pattern=self.__pattern,
settings=self.__settings,
occurrences=occurrences))
def __searchFiles(self, files):
"""
Searches in given files.
:param files: Files.
:type files: list
"""
for file in files:
if self.__interrupt:
return
if not foundations.common.pathExists(file):
continue
if foundations.io.isReadable(file):
if foundations.io.isBinaryFile(file):
continue
LOGGER.info("{0} | Searching '{1}' file!".format(self.__class__.__name__, file))
cacheData = self.__container.filesCache.getContent(file)
if not cacheData:
reader = foundations.io.File(file)
content = reader.read()
if content is None:
LOGGER.warning("!> Error occured while reading '{0}' file proceeding to next one!".format(file))
continue
self.__container.filesCache.addContent(**{file : CacheData(content=content, document=None)})
else:
content = cacheData.content
occurrences = self.__searchDocument(QTextDocument(QString(content)), self.__pattern, self.__settings)
occurrences and self.__searchResults.append(SearchResult(file=file,
pattern=self.__pattern,
settings=self.__settings,
occurrences=occurrences))
def __searchDocument(self, document, pattern, settings):
"""
Searches for given pattern occurrences in given document using given settings.
:param document: Document.
:type document: QTextDocument
:param pattern: Pattern.
:type pattern: unicode
:param settings: Search settings.
:type settings: Structure
:return: Matched occurrences.
:rtype: list
"""
pattern = settings.regularExpressions and QRegExp(pattern) or pattern
flags = QTextDocument.FindFlags()
if settings.caseSensitive:
flags = flags | QTextDocument.FindCaseSensitively
if settings.wholeWord:
flags = flags | QTextDocument.FindWholeWords
occurrences = []
block = document.findBlock(0)
cursor = document.find(pattern, block.position(), flags)
while block.isValid() and cursor.position() != -1:
if self.__interrupt:
return
blockCursor = QTextCursor(cursor)
blockCursor.movePosition(QTextCursor.StartOfLine, QTextCursor.MoveAnchor)
blockCursor.movePosition(QTextCursor.EndOfLine, QTextCursor.KeepAnchor)
length = cursor.selectionEnd() - cursor.selectionStart()
occurrences.append(Occurence(line=cursor.blockNumber(),
column=cursor.columnNumber() - length,
length=length,
position=cursor.position() - length,
text=blockCursor.selectedText()))
cursor = document.find(pattern, cursor.position(), flags)
block = block.next()
return occurrences
class VcsStatusMonitorThread(QThread):
"""
Class implementing the VCS status monitor thread base class.
@signal vcsStatusMonitorData(QStringList) emitted to update the VCS status
@signal vcsStatusMonitorStatus(QString, QString) emitted to signal the status of the
monitoring thread (ok, nok, op) and a status message
"""
def __init__(self, interval, projectDir, vcs, parent = None):
"""
Constructor
@param interval new interval in seconds (integer)
@param projectDir project directory to monitor (string or QString)
@param vcs reference to the version control object
@param parent reference to the parent object (QObject)
"""
QThread.__init__(self, parent)
self.setObjectName("VcsStatusMonitorThread")
self.setTerminationEnabled(True)
self.projectDir = QString(projectDir)
self.vcs = vcs
self.interval = interval
self.autoUpdate = False
self.statusList = QStringList()
self.reportedStates = {}
self.shouldUpdate = False
self.monitorMutex = QMutex()
self.monitorCondition = QWaitCondition()
self.__stopIt = False
def run(self):
"""
Protected method implementing the tasks action.
"""
while not self.__stopIt:
# perform the checking task
self.statusList.clear()
self.emit(SIGNAL("vcsStatusMonitorStatus(QString, QString)"),
QString("wait"), self.trUtf8("Waiting for lock"))
try:
locked = self.vcs.vcsExecutionMutex.tryLock(5000)
except TypeError:
locked = self.vcs.vcsExecutionMutex.tryLock()
if locked:
try:
self.emit(SIGNAL("vcsStatusMonitorStatus(QString, QString)"),
QString("op"), self.trUtf8("Checking repository status"))
res, statusMsg = self._performMonitor()
finally:
self.vcs.vcsExecutionMutex.unlock()
if res:
status = QString("ok")
else:
status = QString("nok")
self.emit(SIGNAL("vcsStatusMonitorStatus(QString, QString)"),
QString("send"), self.trUtf8("Sending data"))
self.emit(SIGNAL("vcsStatusMonitorData(QStringList)"),
QStringList(self.statusList))
self.emit(SIGNAL("vcsStatusMonitorStatus(QString, QString)"),
status, statusMsg)
else:
self.emit(SIGNAL("vcsStatusMonitorStatus(QString, QString)"),
QString("timeout"), self.trUtf8("Timed out waiting for lock"))
if self.autoUpdate and self.shouldUpdate:
try:
self.vcs.vcsUpdate(self.projectDir, True)
continue # check again
except TypeError:
pass # compatibility for older VCS plugins
self.shouldUpdate = False
# wait until interval has expired checking for a stop condition
self.monitorMutex.lock()
if not self.__stopIt:
self.monitorCondition.wait(self.monitorMutex, self.interval * 1000)
self.monitorMutex.unlock()
self.exit()
def setInterval(self, interval):
"""
Public method to change the monitor interval.
@param interval new interval in seconds (integer)
"""
locked = self.monitorMutex.tryLock()
self.interval = interval
self.monitorCondition.wakeAll()
if locked:
self.monitorMutex.unlock()
def getInterval(self):
"""
Public method to get the monitor interval.
@return interval in seconds (integer)
"""
return self.interval
def setAutoUpdate(self, auto):
"""
Public method to enable the auto update function.
@param auto status of the auto update function (boolean)
"""
self.autoUpdate = auto
def getAutoUpdate(self):
"""
Public method to retrieve the status of the auto update function.
@return status of the auto update function (boolean)
"""
return self.autoUpdate
def checkStatus(self):
"""
Public method to wake up the status monitor thread.
"""
locked = self.monitorMutex.tryLock()
self.monitorCondition.wakeAll()
if locked:
self.monitorMutex.unlock()
def stop(self):
"""
Public method to stop the monitor thread.
"""
locked = self.monitorMutex.tryLock()
self.__stopIt = True
self.monitorCondition.wakeAll()
if locked:
self.monitorMutex.unlock()
def clearCachedState(self, name):
"""
Public method to clear the cached VCS state of a file/directory.
@param name name of the entry to be cleared (QString or string)
"""
project = e4App().getObject("Project")
key = project.getRelativePath(unicode(name))
try:
del self.reportedStates[key]
except KeyError:
pass
def _performMonitor(self):
"""
Protected method implementing the real monitoring action.
This method must be overridden and populate the statusList member variable
with a list of strings giving the status in the first column and the
path relative to the project directory starting with the third column.
The allowed status flags are:
<ul>
<li>"A" path was added but not yet comitted</li>
<li>"M" path has local changes</li>
<li>"O" path was removed</li>
<li>"R" path was deleted and then re-added</li>
<li>"U" path needs an update</li>
<li>"Z" path contains a conflict</li>
<li>" " path is back at normal</li>
</ul>
@return tuple of flag indicating successful operation (boolean) and
a status message in case of non successful operation (QString)
"""
raise RuntimeError('Not implemented')
class PlottingThread(QThread):
def __init__(self, parent):
QThread.__init__(self)
self.result = None
self.parent = parent
self._stopped = False
self.mutex = QMutex()
self.filePrefix = None
self.fileFormat = None
self.wallColoring = None
self.cellColoring = None
self.pointColoring = None
self.extraDrawing = []
self.pointSize = None
self.pointLineColor = None
self.pointLineThickness = None
self.ellipsisDraw = None
self.overSampling = None
self.wallThickness = None
self.bgColor = None
self.loading = False
self._crop = QRect(0,0,1,1)
self._pix = None
self._end_image_plot = False
self._loading_arguments = {}
self.retryObject = None
def end_step(self):
return len(self.result)+1
def stop(self, value = True):
self.mutex.lock()
self._stopped = value
self.mutex.unlock()
def stopped(self):
self.mutex.lock()
val = self._stopped
self.mutex.unlock()
return val
def nextImage(self):
QCoreApplication.postEvent(self.parent, NextImageEvent())
def abort(self, reason, **others):
e = AbortPlottingEvent(reason)
if others:
e.others = others
QCoreApplication.postEvent(self.parent, e)
def finished(self):
if self.loading:
QCoreApplication.postEvent(self.parent, FinishLoadingEvent())
self.loading = False
else:
QCoreApplication.postEvent(self.parent, FinishPlottingEvent())
def image_ready(self):
QCoreApplication.postEvent(self.parent, ImageReadyPlottingEvent())
def update_nb_images(self, nb):
QCoreApplication.postEvent(self.parent, UpdateNbImageEvent(nb))
@property
def crop_left(self):
return self._crop.left()
@crop_left.setter
def crop_left(self, value):
self._crop.moveLeft(int(value))
@property
def crop_top(self):
return self._crop.top()
@crop_top.setter
def crop_top(self, value):
self._crop.moveTop(int(value))
@property
def crop_width(self):
return self._crop.width()
@crop_width.setter
def crop_width(self, value):
self._crop.setWidth(int(value))
@property
def crop_height(self):
return self._crop.height()
@crop_height.setter
def crop_height(self, value):
self._crop.setHeight(int(value))
def reset_crop(self):
self._crop = QRect(QPoint(0,0), self.img_size)
@property
def crop(self):
return QRect(self._crop)
@crop.deleter
def crop(self):
self.reset_crop()
@property
def end_image_plot(self):
'''
If true, plot the growth data on the end image rather than the start image of the growth calculation.
'''
return self._end_image_plot
@end_image_plot.setter
def end_image_plot(self, value):
self._end_image_plot = bool(value)
@property
def pix(self):
'''Thread-safe image storage.'''
self.mutex.lock()
pix = self._pix
self.mutex.unlock()
return pix
@pix.setter
def pix(self, value):
self.mutex.lock()
self._pix = value
self.mutex.unlock()
def render_valid(self):
if self.result is None:
log_debug("result is None")
return False
if self.parent is None:
log_debug("parent is None")
return False
if self.ellipsisDraw is None:
log_debug("ellipsisDraw is None")
return False
if self.cellColoring is None:
log_debug("cellColoring is None")
return False
if self.wallColoring is None:
log_debug("wallColoring is None")
return False
if self.pointColoring is None:
log_debug("pointColoring is None")
return False
if self.pointSize is None:
log_debug("pointSize is None")
return False
if self.pointLineThickness is None:
log_debug("pointSize is None")
return False
if self.pointLineColor is None:
log_debug("pointSize is None")
return False
if self.wallThickness is None:
log_debug("wallThickness is None")
return False
if self.overSampling is None:
log_debug("overSampling is None")
return False
if self.bgColor is None:
log_debug("bgColor is None")
return False
return True
def valid(self):
if self.filePrefix is None:
log_debug("filePrefix is None")
return False
if not self.filePrefix:
log_debug("filePrefix is Empty")
return False
if self.fileFormat is None:
log_debug("fileFormat is None")
return False
return self.render_valid()
def drawImage(self, imageid):
cache = image_cache.cache
cellColoring = self.cellColoring
wallColoring = self.wallColoring
pointColoring = self.pointColoring
ellipsisDraw = self.ellipsisDraw
overSampling = self.overSampling
extraDrawing = self.extraDrawing
bgColor = self.bgColor.rgb()
result = self.result
if self.result_type == "Data":
data = result
img_name = result.images_name[imageid]
else:
data = result.data
img_name = result.images[imageid]
#scale = data.images_scale[img_name]
min_scale = data.minScale()
img = cache.image(data.image_path(img_name))
img_data = data[img_name]
size = self._crop.size()
pix = QImage(size*overSampling, QImage.Format_ARGB32)
pix.fill(bgColor)
painter = QPainter()
if not painter.begin(pix):
self.abort("Cannot create painter on QImage")
return None, None, None
painter.setRenderHints(QPainter.SmoothPixmapTransform, True)
painter.setRenderHints(QPainter.Antialiasing, True)
if overSampling > 1:
painter.scale(overSampling, overSampling)
painter.translate(-self._crop.topLeft())
painter.save()
painter.translate(self.translate)
log_debug("Translating: %gx%g" % (self.translate.x(), self.translate.y()) )
painter.scale(1/min_scale, 1/min_scale)
painter.save()
matrix = img_data.matrix()
painter.setWorldTransform(matrix, True)
painter.drawImage(QPoint(0,0), img)
painter.restore()
#pt_matrix = QTransform()
#pt_matrix.scale(1/min_scale, 1/min_scale)
#painter.setTransform(pt_matrix, True)
cellColoring.startImage(painter, imageid)
wallColoring.startImage(painter, imageid)
for ed in extraDrawing:
ed.startImage(painter, imageid)
if self.result_type == "Growth":
cells = result.cells[imageid]
walls = result.walls[imageid]
else:
cells = img_data.cells
walls = set()
for cid in img_data.cells:
pts = [ pt for pt in data.cells[cid] if pt in img_data ]
if len(pts) > 1:
for i in range(len(pts)):
walls.add(data.wallId(pts[i-1], pts[i]))
# Now, draw the cells and the ellipsis
for cid in cells:
painter.setPen(Qt.NoPen)
color = cellColoring(imageid, cid)
painter.setBrush(color)
pts = data.cellAtTime(cid, img_data.index)
if pts:
pts.append(pts[0])
ppts = []
for p1,p2 in zip(pts[:-1], pts[1:]):
ppts.append(img_data[p1])
ppts.extend(img_data.walls[p1,p2])
ppts.append(ppts[0])
poly = QPolygonF(ppts)
painter.drawPolygon(poly)
# And draw the walls
wallThickness = self.wallThickness*min_scale
for wid in walls:
color = wallColoring(imageid, wid)
if color.alpha() > 0:
pen = QPen(color)
pen.setWidthF(wallThickness)
painter.setPen(pen)
pts = [img_data[wid[0]]] + img_data.walls[wid[0], wid[1]] + [img_data[wid[1]]]
#painter.drawLine(img_data[wid[0]], img_data[wid[1]])
painter.drawPolyline(*pts)
# Then, draw the points
pointSize = self.pointSize*min_scale
pointLineColor = self.pointLineColor
pointLineThickness = self.pointLineThickness*min_scale
log_debug("pointSize = %g" % pointSize)
for pid in img_data:
color = pointColoring(imageid, pid)
if color.alpha() > 0:
pen = QPen(pointLineColor)
pen.setWidthF(pointLineThickness)
brush = QBrush(color)
painter.setPen(pen)
painter.setBrush(brush)
pos = img_data[pid]
rect = QRectF(pos.x()-pointSize, pos.y()-pointSize, 2*pointSize, 2*pointSize)
painter.drawEllipse(rect)
if ellipsisDraw.plot:
for cid in cells:
pts = data.cellAtTime(cid, img_data.index)
if pts:
pts.append(pts[0])
ppts = []
for p1,p2 in zip(pts[:-1], pts[1:]):
ppts.append(img_data[p1])
ppts.extend(img_data.walls[p1,p2])
ppts.append(ppts[0])
#poly = QPolygonF(ppts)
#painter.drawPolygon(poly)
ellipsisDraw(painter, imageid, cid, ppts, min_scale)
# At last, draw the extra data
for ed in extraDrawing:
ed(painter, imageid)
tr = painter.worldTransform()
painter.restore()
pic_w = wallColoring.finalizeImage(painter, imageid, tr, self.crop)
pic_c = cellColoring.finalizeImage(painter, imageid, tr, self.crop)
for ed in extraDrawing:
ed.finalizeImage(painter, imageid, tr, self.crop)
painter.end()
return pix, pic_w, pic_c
def start(self):
if self.isRunning():
assert not self.rendering_all, "Cannot run twice the rendering of all images with the same object."
return
if parameters.instance.use_thread:
log_debug("Starting rendering thread.")
QThread.start(self)
return False
else:
self.run()
return True
def render_all(self):
self.rendering_all = True
return self.start()
def render_single(self, img_id, retry=False):
if retry:
while self.isRunning():
self.wait(10000)
elif self.isRunning():
return
self.rendering_all = False
self.current_image = img_id
return self.start()
def load(self, filename):
self.loading = True
self.result = filename
return self.start()
def run(self):
if self.loading:
self.run_loader()
elif self.rendering_all:
self.run_full()
else:
self.run_single()
def run_single(self):
img = self.current_image
self.cellColoring.init()
self.wallColoring.init()
self.pointColoring.init()
log_debug("Rendering image %d" % img)
self.pix, self.pic_w, self.pic_c = self.drawImage(img)
if self.pic_w is not None:
log_debug("Has wall image")
if self.pic_c is not None:
log_debug("Has cell image")
if self.pix is not None:
log_debug("Pix correctly rendered")
log_debug("Rendered image %d = %s" % (img, self.pix))
self.image_ready()
def reload(self):
if self.retryObject is None:
return
self._loading_arguments.update(self.retryObject.method_args)
self.load(self.retryObject.filename)
def run_loader(self):
filename = self.result
try:
self.retryObject = None
# First, prepare the data by getting the images and computing how big they
# should be
f = open(filename)
first_line = f.readline()
f.close()
if first_line.startswith("TRKR_VERSION"):
result = Result(None)
result.load(self.result, **self._loading_arguments)
result_type = "Growth"
else:
result = TrackingData()
result.load(self.result, **self._loading_arguments)
result_type = "Data"
self.result = result
self.result_type = result_type
if result_type == "Data":
data = result
images = data.images_name
if data.cells:
self.has_cells = True
self.has_walls = True
else:
self.has_cells = False
self.has_walls = False
self.has_points = bool(data.cell_points)
else:
data = result.data
images = result.images
self.has_cells = False
self.has_walls = False
self.has_points = False
self.images = images
cache = image_cache.cache
self.update_nb_images(len(result))
bbox = QRectF()
ms = data.minScale()
for i in range(len(result)):
img_name = images[i]
img_data = data[img_name]
img = cache.image(data.image_path(img_name))
matrix = QTransform()
matrix = img_data.matrix()
sc = QTransform()
sc.scale(1.0/ms, 1.0/ms)
matrix *= sc
r = QRectF(img.rect())
rbox = matrix.map(QPolygonF(r)).boundingRect()
bbox |= rbox
log_debug("Image '%s':\n\tSize = %gx%g\n\tTransformed = %gx%g %+g %+g\n\tGlobal bbox = %gx%g %+g %+g\n" %
(img_name, r.width(), r.height(), rbox.width(), rbox.height(), rbox.left(), rbox.top(),
bbox.width(), bbox.height(), bbox.left(), bbox.top()))
log_debug("Matrix:\n%g\t%g\t%g\n%g\t%g\t%g\n" %
(matrix.m11(), matrix.m12(), matrix.dx(), matrix.m21(), matrix.m22(), matrix.dy()))
if result_type == "Growth":
if result.cells[i]:
self.has_cells = True
if result.walls[i]:
self.has_walls = True
self.has_points = bool(result.data.cell_points)
self.nextImage()
translate = bbox.topLeft()
translate *= -1
self.translate = translate
size = bbox.size().toSize()
self.img_size = size
self._crop = QRect(QPoint(0,0), size)
self.finished()
self._loading_arguments = {} # All done, we don't need that anymore
except RetryTrackingDataException as ex:
ex.filename = filename
self.retryObject = ex
self.finished()
return
except Exception as ex:
_, _, exceptionTraceback = sys.exc_info()
self.abort(ex, traceback=exceptionTraceback)
raise
def run_full(self):
if not self.valid():
self.abort("Object was not correctly initialized")
return
self.stop(False)
painter = None
try:
result = self.result
self.update_nb_images(len(result))
# if self.result_type == "Data":
# data = result
# images = result.images_name
# else:
# data = result.data
# images = result.images
# cache = image_cache.cache
cellColoring = self.cellColoring
wallColoring = self.wallColoring
pointColoring = self.pointColoring
file_format = self.fileFormat
file_pattern = "%s%%0%dd.%s" % (self.filePrefix, len(str(len(result))), file_format)
wall_file_pattern = "%s%%0%dd_wall.%s" % (self.filePrefix, len(str(len(result))), file_format)
cell_file_pattern = "%s%%0%dd_cell.%s" % (self.filePrefix, len(str(len(result))), file_format)
cellColoring.init()
wallColoring.init()
pointColoring.init()
self.nextImage()
for i in range(len(result)):
if self.stopped():
self.abort("User interruption")
return
pix, pic_w, pic_c = self.drawImage(i)
pix.save(file_pattern % (i+1), file_format)
if pic_w is not None:
self.saveExtra(pic_w, wall_file_pattern % (i+1), file_format)
if pic_c is not None:
self.saveExtra(pic_c, cell_file_pattern % (i+1), file_format)
self.nextImage()
self.finished()
except Exception as ex:
if painter is not None:
painter.end()
_, _, exceptionTraceback = sys.exc_info()
self.abort(ex, traceback=exceptionTraceback)
raise
def saveExtra(self, picture, file_name, file_format):
rect = picture.boundingRect()
pix = QImage(rect.size(), QImage.Format_ARGB32)
pix.fill(QColor(0, 0, 0, 0).rgba())
paint = QPainter()
paint.begin(pix)
paint.drawPicture(rect.topLeft()*-1, picture)
paint.end()
pix.save(file_name, file_format)
class PointsInPolygonThread(QThread):
def __init__(self, inPoly, inPoints, fieldName, outPath, encoding,
attributeList, statisticSelector):
QThread.__init__(self, QThread.currentThread())
self.mutex = QMutex()
self.stopMe = 0
self.interrupted = False
self.layerPoly = inPoly
self.layerPoints = inPoints
self.fieldName = fieldName
self.outPath = outPath
self.encoding = encoding
self.attributeList = attributeList
self.statistics = statisticSelector.currentText()
def run(self):
self.mutex.lock()
self.stopMe = 0
self.mutex.unlock()
interrupted = False
polyProvider = self.layerPoly.dataProvider()
pointProvider = self.layerPoints.dataProvider()
fieldList = ftools_utils.getFieldList(self.layerPoly)
index = polyProvider.fieldNameIndex(unicode(self.fieldName))
if index == -1:
index = polyProvider.fields().count()
fieldList.append(
QgsField(unicode(self.fieldName), QVariant.Int, "int", 10, 0,
self.tr("point count field")))
# Add the selected vector fields to the output polygon vector layer
selectedItems = self.attributeList.selectedItems()
for item in selectedItems:
global typeDouble
columnName = unicode(item.text() + "_" + self.statistics)
index = polyProvider.fieldNameIndex(unicode(columnName))
if index == -1:
if item.type(
) == typeDouble or self.statistics == "mean" or self.statistics == "stddev":
fieldList.append(
QgsField(columnName, QVariant.Double, "double", 24, 15,
"Value"))
else:
fieldList.append(
QgsField(columnName, QVariant.Int, "int", 10, 0,
"Value"))
sRs = polyProvider.crs()
if QFile(self.outPath).exists():
if not QgsVectorFileWriter.deleteShapeFile(self.outPath):
return
writer = QgsVectorFileWriter(self.outPath, self.encoding, fieldList,
polyProvider.geometryType(), sRs)
spatialIndex = ftools_utils.createIndex(pointProvider)
self.emit(SIGNAL("rangeChanged(int)"), polyProvider.featureCount())
polyFeat = QgsFeature()
pntFeat = QgsFeature()
outFeat = QgsFeature()
inGeom = QgsGeometry()
polyFit = polyProvider.getFeatures()
while polyFit.nextFeature(polyFeat):
inGeom = polyFeat.geometry()
atMap = polyFeat.attributes()
outFeat.setAttributes(atMap)
outFeat.setGeometry(inGeom)
count = 0
pointList = []
hasIntersection = True
pointList = spatialIndex.intersects(inGeom.boundingBox())
if len(pointList) > 0:
hasIntersection = True
else:
hasIntersection = False
if hasIntersection:
valueList = {}
for item in selectedItems:
valueList[item.text()] = []
for p in pointList:
pointProvider.getFeatures(QgsFeatureRequest().setFilterFid(
p)).nextFeature(pntFeat)
tmpGeom = QgsGeometry(pntFeat.geometry())
if inGeom.intersects(tmpGeom):
count += 1
for item in selectedItems:
valueList[item.text()].append(
pntFeat.attribute(item.text()))
self.mutex.lock()
s = self.stopMe
self.mutex.unlock()
if s == 1:
interrupted = True
break
atMap.append(count)
# Compute the statistical values for selected vector attributes
for item in selectedItems:
values = valueList[item.text()]
# Check if the input contains non-numeric values
non_numeric_values = False
for value in values:
if not isinstance(value, type(
float())) and not isinstance(
value, type(int())):
non_numeric_values = True
break
# Jump over invalid values
if non_numeric_values is True:
continue
if values and len(values) > 0:
if self.statistics == "sum":
value = reduce(myAdder, values)
elif self.statistics == "mean":
value = reduce(myAdder, values) / float(
len(values))
elif self.statistics == "min":
values.sort()
value = values[0]
elif self.statistics == "max":
values.sort()
value = values[-1]
elif self.statistics == "stddev":
value = two_pass_variance(values)
value = math.sqrt(value)
atMap.append(value)
outFeat.setAttributes(atMap)
writer.addFeature(outFeat)
self.emit(SIGNAL("updateProgress()"))
self.mutex.lock()
s = self.stopMe
self.mutex.unlock()
if s == 1:
interrupted = True
break
del writer
if not interrupted:
self.emit(SIGNAL("processingFinished()"))
else:
self.emit(SIGNAL("processingInterrupted()"))
def stop(self):
self.mutex.lock()
self.stopMe = 1
self.mutex.unlock()
QThread.wait(self)
class SequenceBuilderFormCapture(QObject):
def __init__(self, framework, parent = None):
QObject.__init__(self, parent)
self.framework = framework
self.targets = {}
self.source_urls = {}
self.source_parameters = {}
self.target_parameters = {}
self.sequence_transitions = {}
self.is_tracking = False
self.qlock = QMutex()
self.postDataExtractor = self.framework.getContentExtractor().getExtractor('post-data')
def set_source_url(self, request_id, url):
if not self.is_tracking:
return
self.qlock.lock()
try:
request_id = str(request_id)
self.source_urls[request_id] = url
if request_id not in self.source_parameters:
self.source_parameters[request_id] = {}
self.source_parameters[request_id] = self.process_url(url, self.source_parameters[request_id])
finally:
self.qlock.unlock()
def process_url(self, url, parameters):
splitted = urlparse.urlsplit(url)
if splitted.query:
qs_values = urlparse.parse_qs(splitted.query, True)
position = 0
for name, value in qs_values.items():
position += 1
parameters[SequenceParameter(url, 'Query', name, '', position)] = value
if splitted.fragment:
qs_values = urlparse.parse_qs(splitted.fragment, True)
position = 0
for name, value in qs_values.items():
position += 1
parameters[SequenceParameter(url, 'Fragment', name, '', position)] = value
return parameters
def store_source_parameter(self, request_id, position, name, Type, value):
if not self.is_tracking:
return
self.qlock.lock()
try:
request_id = str(request_id)
url = self.source_urls[request_id]
sequenceParameter = SequenceParameter(url, 'Form', name, Type, position)
self.source_parameters[request_id][sequenceParameter] = [value]
finally:
self.qlock.unlock()
def process_target_request(self, response_id, originating_request_id, method, url, request_headers, request_body):
if not self.is_tracking:
return
self.qlock.lock()
try:
response_id = str(response_id)
originating_request_id = str(originating_request_id)
self.targets[response_id] = originating_request_id
if response_id not in self.target_parameters:
self.target_parameters[response_id] = {}
self.target_parameters[response_id] = self.process_url(url, self.target_parameters[response_id])
results = self.postDataExtractor.process_request(request_headers, request_body)
if results:
position = 0
# TODO: support non-name/value pair types
for name, value, Type in results.name_values:
position += 1
self.target_parameters[response_id][SequenceParameter(url, method, name, Type, position)] = value
finally:
self.qlock.unlock()
def start_tracking(self):
self.is_tracking = True
def stop_tracking(self):
self.is_tracking = False
def set_sequence_transition(self, requestId, originatingResponseId, previousRequestId):
if not self.is_tracking:
return
self.qlock.lock()
try:
print(('transition sequence; requestId=%s, originatingResponseId=%s, prev=%s' % (requestId, originatingResponseId, previousRequestId)))
self.sequence_transitions[requestId] = originatingResponseId
if previousRequestId in self.source_parameters:
if requestId not in self.source_parameters:
self.source_parameters[requestId] = {}
parameters = self.source_parameters[requestId]
for param, value in self.source_parameters[previousRequestId].items():
parameters[param] = value
if previousRequestId in self.source_urls:
self.source_urls[requestId] = self.source_urls[previousRequestId]
finally:
self.qlock.unlock()
def get_sequence_transition(self, requestId):
if requestId in self.sequence_transitions:
return str(self.sequence_transitions[requestId])
else:
return ''
def allParameters(self, response_ids):
# print(self.source_parameters)
response_ids = [str(m) for m in response_ids]
response_ids.sort()
parameters = []
source_parameters_for_response = {}
print(('targets', self.targets))
for response_id in response_ids:
request_id = self.targets.get(response_id)
if request_id and request_id in self.source_parameters:
# TODO: FIX THIS properly!
# source parameters can be collected multiple times
if response_id not in source_parameters_for_response:
source_parameters_for_response[response_id] = {}
parameters_for_response = source_parameters_for_response[response_id]
for param, values in self.source_parameters[request_id].items():
if param not in parameters_for_response:
parameters_for_response[param] = values
for response_id in response_ids:
if response_id in self.target_parameters:
if response_id in source_parameters_for_response:
for param, values in source_parameters_for_response[response_id].items():
responseId = self.get_sequence_transition(request_id)
parameters.append((responseId, request_id, param, values, 'source'))
request_id = self.targets.get(response_id)
for param, values in self.target_parameters[response_id].items():
parameters.append((response_id, request_id, param, values, 'target'))
return parameters
class DataTaker(QThread):
"""
This object goal is to run the script from the configuration file. It
can access all instrument in its instrument hub (see Drivers->Tool.py-->InstrumentHub)
As it is child from QThread one can launch it using its inherited .start() method.
"""
def __init__(self, lock, instr_hub, parent=None):
print("DTT created")
super(DataTaker, self).__init__(parent)
self.instr_hub = instr_hub
self.connect(self.instr_hub, SIGNAL("changed_list()"),
self.reset_lists)
self.lock = lock
self.stopped = True
self.paused = False
self.mutex = QMutex()
self.completed = False
self.DEBUG = IOTool.get_debug_setting()
#the script which is run everytime the user call the routine "run"
self.script_file_name = ''
#a dict that can be populated by variable the user would like to change
#in real time while the script is ran
self.user_variables = {}
self.t_start = None
# scriptize the intruments and their parameters
self.reset_lists()
def __del__(self):
print("DTT deleted")
def initialize(self, first_time=False):
self.stopped = False
self.completed = False
if first_time:
for key, inst in list(self.instruments.items()):
# there's a none object in the Instruments list, ignore it
if inst:
inst.initialize()
def reset_lists(self):
"""
If changes are made to the InstrumentHub, the DataTaker will not acknowledge them
unless using this method
"""
# print("\tChange instruments in datataker...")
self.instruments = self.instr_hub.get_instrument_list()
self.port_param_pairs = self.instr_hub.get_port_param_pairs()
#print("\t...instruments updated in datataker")
def update_user_variables(self, adict):
"""
Replace the user variables by updated ones
"""
if isinstance(adict, dict):
self.user_variables = adict
def assign_user_variable(self, key, value_type=float, default=None):
"""this is used to change variables while data are being taken
"""
#the key exists
if key in self.user_variables:
if value_type == None:
return self.user_variables[key]
else:
if isinstance(value_type, type):
try:
return value_type(self.user_variables[key])
except ValueError:
print("Wrong type conversion applied on \
user variable")
return self.user_variables[key]
else:
print("Wrong type used for user variable")
return self.user_variables[key]
else:
if default == None:
print("The user variable key %s isn't defined" % key)
else:
#assign the default value to the key entry
self.user_variables[key] = default
#make a recursive call to the method
return self.assign_user_variable(key, value_type)
def run(self):
print("DTT begin run")
self.stopped = False
# open another file which contains the script to follow for this
# particular measurement
# try:
script = open(self.script_file_name)
print("open script " + self.script_file_name)
# check for syntax errors
try:
# py_compile.compile(script_file_name)
code = compile(script.read(), script.name, 'exec')
exec(code)
except py_compile.PyCompileError:
print("Syntax error detected")
script.close()
# except:
# print ("+"*10)+"ERROR"+("+"*10)
# print "Your script file failed to open:\n"
# print self.script_file_name
# print ("+"*10)+"ERROR"+("+"*10)
# print
self.completed = True
self.emit(SIGNAL("script_finished(bool)"), self.completed)
# self.stopped = True
print("DTT run over")
def set_script(self, script_fname):
self.script_file_name = script_fname
def stop(self):
try:
self.mutex.lock()
self.stopped = True
print("DTT stopped")
finally:
self.mutex.unlock()
def pause(self):
print("DTT paused")
self.paused = True
def resume(self):
print("DTT resumed")
self.paused = False
def isPaused(self):
return self.paused
def isStopped(self):
return self.stopped
def check_stopped_or_paused(self):
while True:
if (not self.paused) or self.stopped:
return self.stopped
time.sleep(0.1)
def read_spectrum(self, port):
spectrum_data = self.instruments[port].aquire_spectrum
self.emit(SIGNAL("spectrum_data(PyQt_PyObject)"), spectrum_data)
def read_data(self):
"""
Call the method "measure" for each instrument in InstrumentHub.
It collect the different values of corresponding parameters and
emit a signal which will be catch by other instance for further
treatment.
"""
# param_set = []
data_set = []
for port, param in self.port_param_pairs:
inst = self.instruments[port]
if inst != '' and inst != None:
# if inst !='TIME' and inst!='' and inst!= None:
data_set.append(inst.measure(param))
# param_set.append(inst.channels_names[param])
# elif inst =='TIME':
# data_set.append(round(t_meas,2))
# param_set.append('TIME')
else:
data_set.append(0)
# param_set.append('')
# send data back to the mother ship as an array of floats, but only
# self.emit(SIGNAL("data(PyQt_PyObject)"), np.array(data_set))
self.emit(SIGNAL("data(PyQt_PyObject)"), data_set)
class _FTSearchThread(QThread):
'''This thread performs full text search in the background'''
searchFinished = pyqtSignal()
searchError = pyqtSignal()
def __init__(self, searcher, parent):
QThread.__init__(self, parent)
self._searcher = searcher
self._quit = False
self._mutex = QMutex()
self._pending = QWaitCondition()
self._collector = None
self._query = None
self._result = None
def run(self):
while not self._quit:
self._mutex.lock()
if not self._query:
self._pending.wait(self._mutex)
query = self._query
self._query = None
self._mutex.unlock()
# search
if query:
(query_str1, query_str2, itemtypes, limit, highlight,
merge) = query
self._mutex.lock()
collector = self._searcher.make_collector(limit)
self._collector = collector
self._mutex.unlock()
try:
result = self._searcher.search(collector, query_str1,
query_str2, itemtypes,
highlight)
except:
self._mutex.lock()
self._result = None
self._mutex.unlock()
self.searchError.emit()
else:
if collector.aborted:
pass
else:
self._mutex.lock()
self._result = (merge, result)
self._mutex.unlock()
self.searchFinished.emit()
self._mutex.lock()
self._collector = None
self._mutex.unlock()
def cancel(self):
self._mutex.lock()
self._query = None
if self._collector:
self._collector.abort()
self._mutex.unlock()
def quit(self):
self._mutex.lock()
if self._collector:
self._collector.abort()
self._query = None
self._quit = True
self._mutex.unlock()
self._pending.wakeAll()
def update_query(self,
query_str1=None,
query_str2=None,
itemtypes=(),
limit=1000,
highlight=False,
merge=False):
self._mutex.lock()
if self._collector:
self._collector.abort()
self._query = (query_str1, query_str2, itemtypes, limit, highlight,
merge)
self._mutex.unlock()
self._pending.wakeAll()
def take_result(self):
self._mutex.lock()
r = self._result
self._result = None
self._mutex.unlock()
return r
class PointsInPolygonThread(QThread):
def __init__( self, inPoly, inPoints, fieldName, outPath, encoding, attributeList, statisticSelector):
QThread.__init__( self, QThread.currentThread() )
self.mutex = QMutex()
self.stopMe = 0
self.interrupted = False
self.layerPoly = inPoly
self.layerPoints = inPoints
self.fieldName = fieldName
self.outPath = outPath
self.encoding = encoding
self.attributeList = attributeList
self.statistics = statisticSelector.currentText()
def run(self):
self.mutex.lock()
self.stopMe = 0
self.mutex.unlock()
interrupted = False
polyProvider = self.layerPoly.dataProvider()
pointProvider = self.layerPoints.dataProvider()
fieldList = ftools_utils.getFieldList(self.layerPoly)
index = polyProvider.fieldNameIndex(unicode(self.fieldName))
if index == -1:
index = polyProvider.fields().count()
fieldList.append( QgsField(unicode(self.fieldName), QVariant.Int, "int", 10, 0, self.tr("point count field")) )
# Add the selected vector fields to the output polygon vector layer
selectedItems = self.attributeList.selectedItems()
for item in selectedItems:
global typeDouble
columnName = unicode(item.text() + "_" + self.statistics)
index = polyProvider.fieldNameIndex(unicode(columnName))
if index == -1:
if item.type() == typeDouble or self.statistics == "mean" or self.statistics == "stddev":
fieldList.append( QgsField(columnName, QVariant.Double, "double", 24, 15, "Value") )
else:
fieldList.append( QgsField(columnName, QVariant.Int, "int", 10, 0, "Value") )
sRs = polyProvider.crs()
if QFile(self.outPath).exists():
if not QgsVectorFileWriter.deleteShapeFile(self.outPath):
return
writer = QgsVectorFileWriter(self.outPath, self.encoding, fieldList,
polyProvider.geometryType(), sRs)
spatialIndex = ftools_utils.createIndex( pointProvider )
self.emit(SIGNAL("rangeChanged(int)"), polyProvider.featureCount() )
polyFeat = QgsFeature()
pntFeat = QgsFeature()
outFeat = QgsFeature()
inGeom = QgsGeometry()
polyFit = polyProvider.getFeatures()
while polyFit.nextFeature(polyFeat):
inGeom = polyFeat.geometry()
atMap = polyFeat.attributes()
outFeat.setAttributes(atMap)
outFeat.setGeometry(inGeom)
count = 0
pointList = []
hasIntersection = True
pointList = spatialIndex.intersects(inGeom.boundingBox())
if len(pointList) > 0:
hasIntersection = True
else:
hasIntersection = False
if hasIntersection:
valueList = {}
for item in selectedItems:
valueList[item.text()] = []
for p in pointList:
pointProvider.getFeatures( QgsFeatureRequest().setFilterFid( p ) ).nextFeature( pntFeat )
tmpGeom = QgsGeometry(pntFeat.geometry())
if inGeom.intersects(tmpGeom):
count += 1
for item in selectedItems:
valueList[item.text()].append(pntFeat.attribute(item.text()))
self.mutex.lock()
s = self.stopMe
self.mutex.unlock()
if s == 1:
interrupted = True
break
atMap.append(count)
# Compute the statistical values for selected vector attributes
for item in selectedItems:
values = valueList[item.text()]
# Check if the input contains non-numeric values
non_numeric_values = False
for value in values:
if not isinstance(value, type(float())) and not isinstance(value, type(int())):
non_numeric_values = True
break
# Jump over invalid values
if non_numeric_values is True:
continue
if values and len(values) > 0:
if self.statistics == "sum":
value = reduce(myAdder, values)
elif self.statistics == "mean":
value = reduce(myAdder, values) / float(len(values))
elif self.statistics == "min":
values.sort()
value = values[0]
elif self.statistics == "max":
values.sort()
value = values[-1]
elif self.statistics == "stddev":
value = two_pass_variance(values)
value = math.sqrt(value)
atMap.append(value)
outFeat.setAttributes(atMap)
writer.addFeature(outFeat)
self.emit( SIGNAL( "updateProgress()" ) )
self.mutex.lock()
s = self.stopMe
self.mutex.unlock()
if s == 1:
interrupted = True
break
del writer
if not interrupted:
self.emit( SIGNAL( "processingFinished()" ) )
else:
self.emit( SIGNAL( "processingInterrupted()" ) )
def stop(self):
self.mutex.lock()
self.stopMe = 1
self.mutex.unlock()
QThread.wait( self )
class _OperationsSpool(QObject):
"""
A spool of operations of a certain type. Can use many threads.
"""
def __init__(self, parent):
"""
Pprepare spools and threads
Must be overriden to set :
- type : one of "content", "action", "authentication", "network"
- max_threads : max number of threads (default 1)
"""
super(_OperationsSpool, self).__init__(parent)
self.max_threads = 1
self.threads = [] # list of threads running operations
self.mutex = QMutex()
self.spool = deque() # list of remaining operations
self.running = [] # list of running operations
self.done = [] # list of finished operations
self.errors = [] # list of failed operations
self.waiting = deque() # list of operations not yet added to the spool
def count_all(self):
"""
Return a hash with count of operations for each list in this spool
"""
result = {}
for type in ('spool', 'running', 'done', 'errors', 'waiting', ):
result[type] = len(getattr(self, type))
return result
def count_running(self):
"""
Count "alive" operations : spool + running + waiting
"""
by_type = self.count_all()
return by_type['spool'] + by_type['running'] + by_type['waiting']
def add(self, operation):
"""
Add an operation to the waiting spool
"""
self.waiting.append(operation)
self.check_waiting_list()
def check_waiting_list(self):
"""
This method will check conditions about the operation
before to put it in the spool queue (max_same, kill_same, opposites...)
Thi method is threadsafe (mutex.lock is here to ensure that only one
waiting operation is checked at a time)
"""
self.mutex.lock()
try:
# get operation
operation = self.waiting.popleft()
# check operations specificities
can_add, to_remove = self._test_opposite(operation)
for op in to_remove:
op.status = 'rejected'
self.spool.remove(op)
if can_add:
can_add, to_remove = self._test_same(operation)
for op in to_remove:
op.status = 'rejected'
self.spool.remove(op)
# finally add the operation to the spool only if we can add
# the operation
if can_add:
self.spool.append(operation)
self.ping_threads()
else:
operation.status = 'rejected'
finally:
self.mutex.unlock()
def _test_opposite(self, operation):
"""
If the operation has an opposite name, this method check opposites in
the spool. If an opposite is found and the kill_opposite flag is True,
then the opposite operation is removed from the spool, but if the flag
is False, then the current operation is cancelled.
"""
to_remove = []
can_add = True
if operation.opposite:
for op in self.spool:
if op.is_opposite_of(operation):
if operation.kill_opposite:
to_remove.append(op)
else:
can_add = False
return can_add, to_remove
def _test_same(self, operation):
"""
If the operation has a positive max_same, this method will check for
same operations in the spool and if at least one, return False (and an
empty list of operations to remove).
Else if we have a kill_same, we check for same operations in the spool
and return True (operation still valid) and the list of removals
"""
to_remove = []
can_add = True
# check max_same
if operation.max_same:
count_same = 0
for op in self.spool:
if op.is_same_as(operation):
count_same += 1
if count_same >= operation.max_same:
can_add = False
break
# check max_same
elif operation.kill_same:
to_remove = []
for op in self.spool:
if op.is_same_as(operation):
to_remove.append(op)
return can_add, to_remove
def next(self):
"""
Return the next operation to run, remove it from the spool queue,
and move it to the running one.
Will be called by the next thread which has finished working on its
previous operation. This method is threadsafe (mutex.lock is here to
ensure only one thread at a time can claim a operation)
"""
self.mutex.lock()
try:
operation = self.spool.popleft()
self.running.append(operation)
return operation
except:
return None
finally:
self.mutex.unlock()
def ask_for_network(self):
"""
We ask for network
"""
self.parent().ask_for_network()
def ask_for_authentication(self):
"""
We ask for a new authentication
"""
self.parent().ask_for_authentication()
def is_ready(self):
"""
Check if the spool is ready to work (we are both connected and
authenticated)
"""
return self.parent().is_ready()
def ping_threads(self):
"""
Ping all threads to let them check if new operations are available
in the spool. A new thread is created if needed (and possible)
"""
one_available = False
for thread in self.threads:
if not thread.isRunning():
one_available = True
thread.ping()
if not one_available and len(self.threads) < self.max_threads:
thread = OperationsThread(spool=self)
QObject.connect(thread, SIGNALS['operation_started'], self.operation_started, Qt.QueuedConnection)
QObject.connect(thread, SIGNALS['operation_ended'], self.operation_ended, Qt.QueuedConnection)
self.threads.append(thread)
thread.ping()
def operation_started(self, operation_id):
"""
Forward the "operation_started" from the spool to the operations manager
"""
log("[op-start] %s\n" % Operation.get_by_id(operation_id))
operation = Operation.get_by_id(operation_id)
self.parent().emit(SIGNALS['%s_started' % operation.name], operation_id)
self.parent().emit(SIGNALS['operation_started'], operation_id)
def operation_ended(self, operation_id):
"""
Forward the "operation_ended" from the spool to the operations manager
"""
# get the corresponding operation
operation = Operation.get_by_id(operation_id)
# update the spool
self.running.remove(operation)
if operation.status == 'done':
self.done.append(operation)
else:
self.errors.append(operation)
# alert that an operation has finished
log("[op-end] %s\n" % operation)
self.parent().emit(SIGNALS['operation_ended'], operation_id)
# then alert for this specific operation
status = 'done'
operation = Operation.get_by_id(operation_id)
if operation.status != 'done':
status = 'error'
self.parent().emit(SIGNALS['%s_%s' % (operation.name, status)], operation_id)
class Rexarm():
"""!
@brief This class describes a rexarm.
The configuration of the rexarm is loaded from a .csv file as described in the Rexarm.initialize.
"""
# Class constants
# Convert config file type strings to string to type functions
STR_TO_TYPE = {
'bool': lambda s: s.lower() in s.lower() in ['true', '1', 't', 'y', 'yes'],
'str': str,
'float': float,
'int': int
}
def __init__(self):
"""!
@brief Constructs a new instance.
Starts the rexarm run thread but does not initialise the Joints. Call Rexarm.initialize to initialise the
Joints.
"""
# Thread locks
self._cmd_mutex = QMutex()
self._serial_mutex = QMutex()
# Serial bus
self.dxlbus = None
# Gripper
self.gripper = None
self.gripper_state = True
# State
self.estop = False
self.initialized = False
# Update rexarm every update_period seconds
self.update_period = 0.01
# Cmds
self.new_speed_cmds = False
self.speed_cmds = None
self.new_torque_limit_cmds = False
self.torque_limit_cmds = None
self.new_position_cmds = False
self.position_cmds = None
# Feedback
self.position_fb = None
self.speed_fb = None
self.load_fb = None
self.temp_fb = None
# Joints
self._joints = []
self.open_angle = 0
self.close_angle = -60
# Data collection flag
self.collect_flag = False
# Start run thread
self.run_thread = RexarmThread(self)
self.run_thread.start()
def initialize(self, config_file=script_path+'/config/rexarm_config.csv'):
"""!
@brief Initializes the rexarm from given configuration file.
Initializes the Joints and serial port
@param config_file The configuration file see Rexarm._config_joints function for more details on the
config file.
@return True is succes False otherwise
"""
self.initialized = False
# Wait for other threads to finish with the rexarm instead of locking every single call
time.sleep(1)
# Get a serial port interface
self._serial_mutex.lock()
if self.dxlbus is not None:
self.dxlbus.close()
self.dxlbus = DXL_BUS(DEVICENAME, BAUDRATE)
self.port_num = self.dxlbus.port()
self._serial_mutex.unlock()
""" Read in rexarm configs from config file """
if not self._config_joints(config_file):
return self.initialized
""" Commanded Values """
self.num_joints = len(self._joints)
self.position = [0.0] * self.num_joints # degrees
self.speed = [1.0] * self.num_joints # 0 to 1
self.max_torque = [1.0] * self.num_joints # 0 to 1
""" Feedback Values """
self.position_fb = [0.0] * self.num_joints # degrees
self.speed_fb = [0.0] * self.num_joints # 0 to 1
self.load_fb = [0.0] * self.num_joints # -1 to 1
self.temp_fb = [0.0] * self.num_joints # Celsius
self.move_fb = [0] * self.num_joints
""" Gripper Vaules """
self.gripper = None
for joint in self._joints:
if joint.is_gripper:
self.gripper = joint
if self.gripper is not None:
self.gripper.torque_limit = 1.0
self.gripper.speed = 0.8
# Reset estop and initialized
self.estop = False
self.initialized = True
return self.initialized
def _config_joints(self, config_file):
"""!
@brief Configure the Joints and initialize them
TODO: Find the physical properties of the Rexarm (such as angle limits and DH parameters) and write a config
file for the Rexarm. The config file should be formated as a csv file with the following properties:
* The first row lists the exact names of the member variables of the Joint class
* The second row list the types of the associated member variables, Rexarm.STR_TO_TYPE dict defines the
avaliable types
* Each following row defines thoes member variables for a joint in the rexarm
* There should be no spaces after a comma
You may add any new member variables as you wish to the config file in any order. By default they will be
added as a readonly property of the Joint class. If special behaviors are desired for a new member variable
then they must be added to the Joint.__init__ parameter list and handled in that function.
@param config_file The configuration file
@return True if all joints initialize correctly, False if any joint fails
"""
self._joints = []
with open(config_file) as configs:
reader = csv.reader(configs)
title_row = next(reader)
type_strs = next(reader)
types = [Rexarm.STR_TO_TYPE[s] for s in type_strs]
for row in reader:
values = {name: dtype(value) for name, value, dtype in zip(title_row, row, types)}
joint = Joint(self._serial_mutex, self.port_num, **values)
self._joints.append(joint)
if not joint.initialize():
return False
return True
def _ensure_initialized(func):
"""!
@brief Decorator to skip the function if the rexarm is not initialized.
@param func The function to wrap
@return The wraped function
"""
def func_out(self, *args, **kwargs):
if self.initialized:
return func(self, *args, **kwargs)
else:
print('WARNING: Trying to use the Rexarm before initialized')
return func_out
@_ensure_initialized
def open_gripper(self):
"""!
@brief TODO: Tell the gripper to open.
"""
new_position = self.position_fb
new_position[5] = self.open_angle
self.set_positions(new_position)
@_ensure_initialized
def open_gripper_blocking(self):
"""!
@brief TODO: Open a gripper and block until it is.
"""
pass
def is_gripper_open(self):
"""!
@brief TODO: Determines if gripper open.
@return True if gripper open, False otherwise.
"""
if abs(self.position_fb[5] - self.open_angle) < 0.2:
return True
else:
return False
@_ensure_initialized
def close_gripper(self):
"""!
@brief TODO Closes a gripper.
"""
new_position = self.position_fb
new_position[5] = self.close_angle
self.set_positions(new_position)
@_ensure_initialized
def close_gripper_blocking(self):
"""!
@brief TODO Closes a gripper and block until it is.
"""
pass
def is_gripper_close(self):
"""!
@brief TODO Determines if gripper close.
@return True if gripper close, False otherwise.
"""
pass
def toggle_gripper(self):
if self.is_gripper_open():
self.open_gripper()
else:
self.close_gripper()
@_ensure_initialized
def toggle_gripper(self):
"""!
@brief TODO Toggle the gripper between open and close
"""
pass
@_ensure_initialized
def toggle_gripper_blocking(self):
"""!
@brief TODO Toggle the gripper between open and close and block until done
"""
pass
def set_positions(self, joint_angles):
"""!
@brief Sets the positions.
@param joint_angles The joint angles
"""
self._cmd_mutex.lock()
self.position_cmds = joint_angles
self.new_position_cmds = True
self._cmd_mutex.unlock()
def set_speeds_normalized_all(self, speed):
"""!
@brief Sets the speeds normalized all.
@param speed The speed
"""
self._cmd_mutex.lock()
self.speed_cmds = [speed] * len(self._joints)
self.new_speed_cmds = True
self._cmd_mutex.unlock()
def set_speeds_normalized(self, speeds):
"""!
@brief Sets the speeds normalized.
@param speeds The speeds
"""
self._cmd_mutex.lock()
self.speed_cmds = speeds
self.new_speed_cmds = True
self._cmd_mutex.unlock()
def set_speeds(self, speeds):
"""!
@brief Sets the speeds.
@param speeds The speeds
"""
self._cmd_mutex.lock()
self.speed_cmds = [None] * len(self._joints)
for i in range(len(self._joints)):
cmd = abs(speeds[i] / self._joints[i].max_speed)
if (cmd < 3.0 / 1023.0):
cmd = 3.0 / 1023.0
self.speed_cmds[i] = cmd
self.new_speed_cmds = True
self._cmd_mutex.unlock()
def set_torque_limits(self, torques):
"""!
@brief Sets the torque limits.
Can't be used when estoped.
@param torques The torques
"""
if self.estop:
print('WARNING: Torques are zero when estoped')
return
self._cmd_mutex.lock()
self.torque_limit_cmds = torques
self.new_torque_limit_cmds = True
self._cmd_mutex.unlock()
@_ensure_initialized
def _send_commands(self):
"""!
@brief Sends commands.
Send the most recent commands to the joints or set torques to zero if estoped.
"""
# Copy current commands so other threads need not wait for the serial communications
self._cmd_mutex.lock()
torque_limit_cmds = None
speed_cmds = None
position_cmds = None
if self.new_torque_limit_cmds:
torque_limit_cmds = self.torque_limit_cmds.copy()
if self.new_speed_cmds:
speed_cmds = self.speed_cmds.copy()
if self.new_position_cmds:
position_cmds = self.position_cmds.copy()
self.new_torque_limit_cmds = False
self.new_speed_cmds = False
self.new_position_cmds = False
self._cmd_mutex.unlock()
# Set torques to zero when estoped
if self.estop:
torque_limit_cmds = [0] * len(self._joints)
for joint in self._joints:
joint.torque_limit = 0
return
# Write the commands
for i, joint in enumerate(self._joints):
if position_cmds is not None:
joint.position = position_cmds[i]
# Gripper torque and speed handled seperatly
if joint.is_gripper:
continue
if torque_limit_cmds is not None:
joint.torque_limit = torque_limit_cmds[i]
if speed_cmds is not None:
joint.speed = speed_cmds[i]
def disable_torque(self):
"""!
@brief Disables the torque and estops.
"""
self.estop = True
def get_positions(self):
"""!
@brief Gets the positions.
@return The positions.
"""
return self.position_fb
def get_speeds(self):
"""!
@brief Gets the speeds.
@return The speeds.
"""
return self.speed_fb
def get_loads(self):
"""!
@brief Gets the loads.
@return The loads.
"""
return self.load_fb
def get_temps(self):
"""!
@brief Gets the temps.
@return The temps.
"""
return self.temp_fb
def get_moving_status(self):
"""!
@brief Gets the moving status.
@return The moving status.
"""
return self.move_fb
def get_errors(self):
"""!
@brief Gets the serial errors.
@return The serial errors.
"""
return [joint.serial_errors for joint in self._joints]
@_ensure_initialized
def _get_feedback(self):
"""!
@brief Updates all the feedback variables of the joints.
"""
self.position_fb = [joint.position_fb for joint in self._joints]
self.speed_fb = [joint.speed_fb for joint in self._joints]
self.load_fb = [joint.load_fb for joint in self._joints]
self.temp_fb = [joint.temp_fb for joint in self._joints]
self.move_fb = [joint.move_fb for joint in self._joints]
@_ensure_initialized
def get_wrist_pose(self):
"""!
@brief TODO Get the wrist pose.
@return The wrist pose as [x, y, z, phi].
"""
return [0, 0, 0, 0]
def get_dh_parameters(self):
"""!
@brief Gets the dh parameters.
@return The dh parameters.
"""
return [[joint.dh_a, joint.dh_alpha, joint.dh_d, joint.dh_theta] for joint in self._joints]
@_ensure_initialized
def run(self):
"""!
@brief Update the servos feedback and commands
Run in a seperate thread see RexarmThread.
"""
# Update feedback
self._get_feedback()
if self.collect_flag:
with open("rexarm_pos_data.csv", 'a') as inFile:
inFile.write(str(time.time()) + ",")
for pos in self.position_fb:
inFile.write(str(pos) + ",")
inFile.write("\n")
# Send new commands
self._send_commands()
class MessageSender:
"""
static fileds:
context: to keep track of msg req<->resp mechanism msg context is attached.
Thanks to this value it is possible to find right response for given request.
There are context ids reserved for particular messages handling:
0: free text which should be displayed in console window
1: digidiag frame data
Message sender cant assign reserved_context id when message is created and sent.
"""
reserved_context = (
0,
1,
)
context = Uint16(
len(reserved_context)) #start with value greater than reserved_context
lock = False
class ID:
"""
Message IDs reperesenting given procedure in EMU BT board
"""
txt_message = 0
write_to_page = 1
rxflush = 2
bootloader = 11
disable_btlrd = 12
reset = 14
run_main_app_btl = 254
@classmethod
def translate_id(cls, m_id):
try:
return [
p[0] for p in MessageSender.ID.__dict__.items()
if p[1] == m_id
][0]
except IndexError:
return None
def __init__(self, tx_interface):
self.mutex = QMutex()
self.__transmit = tx_interface
def __send_m(self, msg, m_id):
"""
Send createad message.
Avoid using recovered context_ids
"""
context = MessageSender.context
MessageSender.context += 1
while MessageSender.context in MessageSender.reserved_context:
MessageSender.context += 1
translated_m_id = MessageSender.ID.translate_id(m_id)
m_logger.debug("Sent message with context: {}, id: {}({})".format(
context, translated_m_id, m_id))
m_logger.debug(msg[11:30])
self.__transmit(msg)
return context
def peek_context(self):
return MessageSender.context + 1
def send(self, m_id, body=b'NULL'):
"""
Polymorphic method for send
"""
return self.__send(m_id, body)
def send_raw_msg(self, body):
"""
Polymorphic method for send
"""
return self.__send(m_id=None, body=body)
def __send(self, m_id=None, body='NULL'):
msg = create_message(
msg_id=m_id, body=body,
context=MessageSender.context) if m_id is not None else body
self.mutex.lock()
context = self.__send_m(msg, m_id)
self.mutex.unlock()
m_logger.debug("message sending unlocked")
return context
class SpatialIdxThread(QThread):
def __init__(self, layers, isFiles):
QThread.__init__(self, QThread.currentThread())
self.layers = layers
self.isFiles = isFiles
self.mutex = QMutex()
self.stopMe = 0
self.errors = []
def run(self):
self.mutex.lock()
self.stopMe = 0
self.mutex.unlock()
interrupted = False
if self.isFiles:
for layer in self.layers:
vl = QgsVectorLayer(layer, "tmp", "ogr")
provider = vl.dataProvider()
if provider.capabilities(
) & QgsVectorDataProvider.CreateSpatialIndex:
if not provider.createSpatialIndex():
self.errors.append(layer)
else:
self.errors.append(layer)
self.emit(SIGNAL("layerProcessed()"))
self.mutex.lock()
s = self.stopMe
self.mutex.unlock()
if s == 1:
interrupted = True
break
else:
for layer in self.layers:
vl = ftools_utils.getVectorLayerByName(layer)
provider = vl.dataProvider()
if provider.capabilities(
) & QgsVectorDataProvider.CreateSpatialIndex:
if not provider.createSpatialIndex():
self.errors.append(layer)
else:
self.errors.append(layer)
self.emit(SIGNAL("layerProcessed()"))
self.mutex.lock()
s = self.stopMe
self.mutex.unlock()
if s == 1:
interrupted = True
break
if not interrupted:
self.emit(SIGNAL("processFinished( PyQt_PyObject )"), self.errors)
else:
self.emit(SIGNAL("processInterrupted()"))
def stop(self):
self.mutex.lock()
self.stopMe = 1
self.mutex.unlock()
QThread.wait(self)
class Walker(QThread):
COMMON_WORDS_THRESHOLD = 250
MIN_WORD_LEN = 3
MAX_WORD_LEN = 25
INVALID_FIRST_OR_LAST = frozenset("0123456789_")
STRIPHTML_RE = re.compile(r"<[^>]*?>", re.IGNORECASE | re.MULTILINE)
ENTITY_RE = re.compile(r"&(\w+?);|&#(\d+?);")
SPLIT_RE = re.compile(r"\W+", re.IGNORECASE | re.MULTILINE)
def __init__(self,
index,
lock,
files,
filenamesForWords,
commonWords,
parent=None):
super(Walker, self).__init__(parent)
self.index = index
self.lock = lock
self.files = files
self.filenamesForWords = filenamesForWords
self.commonWords = commonWords
self.stopped = False
self.mutex = QMutex()
self.completed = False
def stop(self):
try:
self.mutex.lock()
self.stopped = True
finally:
self.mutex.unlock()
def isStopped(self):
try:
self.mutex.lock()
return self.stopped
finally:
self.mutex.unlock()
def run(self):
self.processFiles()
self.stop()
self.emit(SIGNAL("finished(bool,int)"), self.completed, self.index)
def processFiles(self):
def unichrFromEntity(match):
text = match.group(match.lastindex)
if text.isdigit():
return unichr(int(text))
u = htmlentitydefs.name2codepoint.get(text)
return unichr(u) if u is not None else ""
for fname in self.files:
if self.isStopped():
return
words = set()
fh = None
try:
fh = codecs.open(fname, "r", "UTF8", "ignore")
text = fh.read()
except (IOError, OSError), e:
sys.stderr.write("Error: {0}\n".format(e))
continue
finally:
class WorkerThread(QThread):
''' Schedule things to do
Wait all time or launch/wake other threads to recompute max amount of node to
display with or without zoom, refresh/redraw paint area, etc. .
'''
def __init__(self, parent = None):
super(WorkerThread, self).__init__(parent)
self.mutex = QMutex()
self.condition = QWaitCondition()
self.restart = False
self.abort = False
self.timeline = parent
self.countThread = parent.countThread
self.populateThread = parent.populateThread
self.maxOccThread = parent.maxOccThread
def __del__(self):
self.mutex.lock()
self.abort = True
self.condition.wakeOne()
self.mutex.unlock()
self.wait()
def render(self):
locker = QMutexLocker(self.mutex)
if not self.isRunning() and not self.countThread.isRunning() and not self.populateThread.isRunning() and not self.maxOccThread.isRunning():
self.start(QThread.LowPriority)
else:
self.restart = True
self.condition.wakeOne()
def run(self):
while True:
self.mutex.lock()
# Fetch value from timeline
nodeCount = self.timeline.nodeCount
dataListsCreated = self.timeline.dataListsCreated
if dataListsCreated:
self.timeline.findMaxValue()
xHop = self.timeline.xHop
maxOcc = self.timeline.maxOcc
self.mutex.unlock()
if not nodeCount and not self.restart and not self.countThread.isRunning():
self.countThread.start()
elif nodeCount and not dataListsCreated and not self.restart and not self.populateThread.isRunning():
self.populateThread.start()
elif nodeCount and dataListsCreated and xHop and not maxOcc and not self.restart and not self.maxOccThread.isRunning():
self.maxOccThread.start()
elif nodeCount and dataListsCreated and xHop and maxOcc and not self.restart and not self.maxOccThread.isRunning():
self.emit(SIGNAL('refresh'), True)
if self.abort:
return
self.mutex.lock()
if not self.restart:
self.condition.wait(self.mutex)
self.restart = False
self.mutex.unlock()
class MessageReceiver:
""""
Checks given rx buffer if message is present there.
struct Tail
{
uint8_t tail_start = TAIL_START_MARK; // '<'
uint16_t id = 0;
uint16_t context = 0;
uint16_t msg_len = 0;
uint16_t body_crc= 0;
uint8_t tail_end = TAIL_END_MARK; // '>'
}
uint16_t tail_crc= 0;
"""
TAIL_LEN = 10
TAIL_START_MARK = ord('<')
TAIL_END_MARK = ord('>')
TAIL_CRC_SHIFT_POS = 3
ts = time.time()
LOCKED = False
def __init__(self, rx_buffer):
self.rx_buffer = rx_buffer
self.__mean_rx_time = MeanCalculator()
self.mutex = QMutex()
self.t0 = time.time()
def check_tail(self, peek_buff):
init_find = peek_buff.find(MessageReceiver.TAIL_START_MARK)
peek_buff_len = len(peek_buff)
for i in range(peek_buff_len - MessageReceiver.TAIL_LEN + init_find):
latest_find = i + init_find
try:
tail_start_mark_pos = peek_buff[latest_find:].find(
MessageReceiver.TAIL_START_MARK)
tail_end_mark_pos = tail_start_mark_pos + MessageReceiver.TAIL_LEN - 1
tail_end_mark = peek_buff[latest_find:][tail_end_mark_pos]
except IndexError:
return False
try:
if tail_end_mark == MessageReceiver.TAIL_END_MARK:
peek_buff = peek_buff[latest_find:]
tail = peek_buff[tail_start_mark_pos:tail_end_mark_pos]
_full_tail = peek_buff[
tail_start_mark_pos:tail_end_mark_pos + 1]
_id = struct.unpack('H', tail[1:3])[0]
_context = struct.unpack('H', tail[3:5])[0]
_msg_len = struct.unpack('H', tail[5:7])[0]
_body_crc = tail[7:9]
_tail_crc = peek_buff[tail_end_mark_pos +
1:tail_end_mark_pos +
MessageReceiver.TAIL_CRC_SHIFT_POS]
tail_integrity = _tail_crc == crc_bytes(
_full_tail) # tail integrity check
if _id < len(RxMessage.rx_id_tuple) and _msg_len < MAX_PACKET_SIZE and _context < 0xffff \
and self.rx_buffer.available() > _msg_len and tail_integrity:
return _id, _context, _msg_len, _body_crc, tail_start_mark_pos + latest_find, tail_end_mark_pos + latest_find
except struct.error:
return False
return False
def get_message(self):
t0 = time.time()
ret_rxmsg = None
if self.rx_buffer.available() >= MessageReceiver.TAIL_LEN:
self.mutex.lock()
peek_buff = self.rx_buffer.peek()
check_tail_result = self.check_tail(peek_buff)
self.mutex.unlock()
if check_tail_result:
_id, _context, _msg_len, _crc, tail_start_mark_pos, tail_end_mark_pos = check_tail_result
self.mutex.lock()
msg_body = self.rx_buffer.read(tail_end_mark_pos + 1 + 2)
self.mutex.unlock()
msg_body = msg_body[:tail_start_mark_pos]
MessageReceiver.ts = time.time()
crc_check = RxMessage.RxId.ack if _crc == crc_bytes(
msg_body) else RxMessage.RxId.nack
rxmsg = RxMessage(msg_id=_id,
crc_check=crc_check,
length=len(msg_body),
context=_context,
body=msg_body)
m_logger.debug(MSG_RX_DBG_TEMPLATE.format(rxmsg))
self.t0 = time.time()
if _crc == crc_bytes(msg_body):
self.__mean_rx_time.count(time.time() - t0)
ret_rxmsg = rxmsg
m_logger.debug("Mean msg extract time: {}".format(
self.__mean_rx_time))
m_logger.debug("Period: {}".format(time.time() - self.t0))
return ret_rxmsg
class BeamshutterWidget(QWidget):
currentShutterState = pyqtSignal(str, name='currentShutterState')
def __init__(self, parent=None):
QWidget.__init__(self, parent)
self.ui = uic.loadUi("Beamshutter.ui", self)
self.control = QAxContainer.QAxWidget(self)
self.control.setControl('{3CE35BF3-1E13-4D2C-8C0B-DEF6314420B3}')
self.control.setProperty("HWSerialNum", 85845031)
self.control.setGeometry(0, 0, 600, 400)
self.mutex = QMutex(mode=QMutex.Recursive)
self.shutterReplied = False
self.shut = Beamshutter(self)
self.worker = WorkerThread()
self.ui.mainLayout.addWidget(self.control)
self.layout().setAlignment(Qt.AlignCenter)
sizePolicy = QSizePolicy(QSizePolicy.Preferred, QSizePolicy.Preferred)
sizePolicy.setHeightForWidth(True)
self.setSizePolicy(sizePolicy)
def heightForWidth(self, width):
return width * 0.75
@pyqtSlot()
def on_BInitializeShutter_clicked(self):
if not self.LEDShutter.isChecked():
self.initialize()
else:
self.finalize()
def initialize(self):
with self.check(timeOut=10):
self.worker.executeFunction(self.shut.initialize)
self.ui.LEDShutter.setChecked(True)
def finalize(self):
with self.check():
self.worker.executeFunction(self.shut.finalize)
self.LEDShutter.setChecked(False)
def enable(self):
with self.check():
self.worker.executeFunction(self.shut.enable)
def disable(self):
with self.check():
self.worker.executeFunction(self.shut.disable)
def _del(self):
if self.ui.LEDShutter.isChecked():
self.finalize()
def heightForWidth(self, w):
return (w)
@contextmanager
def check(self, timeOut=2):
out = datetime.datetime.now() + datetime.timedelta(seconds=timeOut)
self.shutterReplied = shutterReplied = False
yield
while datetime.datetime.now() < out and shutterReplied == False:
self.mutex.lock()
shutterReplied = self.shutterReplied
self.mutex.unlock()
if shutterReplied == False:
raise Exception("ThorlabsShutter timed out and might be Crashed")
class SpatialIdxThread( QThread ):
def __init__( self, layers, isFiles ):
QThread.__init__( self, QThread.currentThread() )
self.layers = layers
self.isFiles = isFiles
self.mutex = QMutex()
self.stopMe = 0
self.errors = []
def run( self ):
self.mutex.lock()
self.stopMe = 0
self.mutex.unlock()
interrupted = False
if self.isFiles:
for layer in self.layers:
vl = QgsVectorLayer( layer, "tmp", "ogr" )
provider = vl.dataProvider()
if provider.capabilities() & QgsVectorDataProvider.CreateSpatialIndex:
if not provider.createSpatialIndex():
self.errors.append( layer )
else:
self.errors.append( layer )
self.emit( SIGNAL( "layerProcessed()" ) )
self.mutex.lock()
s = self.stopMe
self.mutex.unlock()
if s == 1:
interrupted = True
break
else:
for layer in self.layers:
vl = ftools_utils.getVectorLayerByName( layer )
provider = vl.dataProvider()
if provider.capabilities() & QgsVectorDataProvider.CreateSpatialIndex:
if not provider.createSpatialIndex():
self.errors.append( layer )
else:
self.errors.append( layer )
self.emit( SIGNAL( "layerProcessed()" ) )
self.mutex.lock()
s = self.stopMe
self.mutex.unlock()
if s == 1:
interrupted = True
break
if not interrupted:
self.emit( SIGNAL( "processFinished( PyQt_PyObject )" ), self.errors )
else:
self.emit( SIGNAL( "processInterrupted()" ) )
def stop( self ):
self.mutex.lock()
self.stopMe = 1
self.mutex.unlock()
QThread.wait( self )
class DomFuzzerThread(QThread):
UNIQUE_MARKER_BASE = '\'-(/b41ns5xg,)'
UNIQUE_MARKER = '>"' + UNIQUE_MARKER_BASE + '<'
NUM1 = '857345'
NUM2 = '572912'
STANDARD_TESTS = [
'\'"></script><script>alert(%s)</script>' % NUM1,
'\'-alert(%s)-\'' % NUM2, UNIQUE_MARKER
]
def __init__(self,
framework,
queueDataModel,
resultsDataModel,
parent=None):
QThread.__init__(self, parent)
self.framework = framework
self.queueDataModel = queueDataModel
self.resultsDataModel = resultsDataModel
self.qlock = QMutex()
self.qlock_analysis = QMutex()
QObject.connect(self, SIGNAL('quit()'), self.quitHandler)
QObject.connect(self, SIGNAL('started()'), self.startedHandler)
self.re_delim = re.compile(r'([;&])')
self.re_unique_marker_base = re.compile(
re.escape(self.UNIQUE_MARKER_BASE), re.I)
self.pending_fuzz_response_ids = deque()
self.analysis_queue = deque()
self.processed_urls = {}
self.Data = None
self.read_cursor = None
self.read_cursor2 = None
self.write_cursor = None
def db_attach(self):
self.Data = self.framework.getDB()
self.read_cursor = self.Data.allocate_thread_cursor()
self.read_cursor2 = self.Data.allocate_thread_cursor()
self.write_cursor = self.Data.allocate_thread_cursor()
self.populateExistingFuzzData()
def db_detach(self):
self.close_cursor()
self.Data = None
def close_cursor(self):
if self.write_cursor and self.Data:
self.write_cursor.close()
self.Data.release_thread_cursor(self.write_cursor)
self.write_cursor = None
if self.read_cursor2 and self.Data:
self.read_cursor2.close()
self.Data.release_thread_cursor(self.read_cursor2)
self.read_cursor2 = None
if self.read_cursor and self.Data:
self.read_cursor.close()
self.Data.release_thread_cursor(self.read_cursor)
self.read_cursor = None
def run(self):
QObject.connect(self, SIGNAL('populateExistingFuzzData()'),
self.do_populateExistingFuzzData, Qt.DirectConnection)
QObject.connect(self, SIGNAL('clearFuzzQueue()'),
self.do_clearFuzzQueue, Qt.DirectConnection)
QObject.connect(self, SIGNAL('startFuzzing()'), self.do_startFuzzing,
Qt.DirectConnection)
QObject.connect(self, SIGNAL('stopFuzzing()'), self.do_stopFuzzing,
Qt.DirectConnection)
QObject.connect(self, SIGNAL('fuzzItemFinished()'),
self.do_fuzzItemFinished, Qt.DirectConnection)
QObject.connect(self, SIGNAL('generateFuzzValues()'),
self.do_generateFuzzValues, Qt.DirectConnection)
self.exec_()
def quitHandler(self):
self.framework.debug_log('DomFuzzerThread quit...')
self.close_cursor()
self.exit(0)
def startedHandler(self):
self.framework.debug_log('DomFuzzerThread started...')
self.framework.subscribe_database_events(self.db_attach,
self.db_detach)
self.framework.subscribe_populate_domfuzzer_response_id(
self.do_populate_domfuzzer_response_id)
self.framework.subscribe_populate_domfuzzer_response_list(
self.do_populate_domfuzzer_response_list)
def populateExistingFuzzData(self):
QTimer.singleShot(10, self, SIGNAL('populateExistingFuzzData()'))
def clearFuzzQueue(self):
QTimer.singleShot(10, self, SIGNAL('clearFuzzQueue()'))
def startFuzzing(self, fuzz_callback):
self.fuzz_callback = fuzz_callback
QTimer.singleShot(10, self, SIGNAL('startFuzzing()'))
def stopFuzzing(self):
QTimer.singleShot(10, self, SIGNAL('stopFuzzing()'))
def fuzzItemFinished(self, fuzz_id, fuzz_url, html, messages):
self.qlock_analysis.lock()
self.analysis_queue.append((fuzz_id, fuzz_url, html, messages))
self.qlock_analysis.unlock()
QTimer.singleShot(10, self, SIGNAL('fuzzItemFinished()'))
def do_startFuzzing(self):
print('do_startFuzzing')
self.keep_fuzzing = True
self.processed_urls = {}
self.dispatch_next_fuzz_item()
def do_fuzzItemFinished(self):
self.dispatch_next_fuzz_item()
self.qlock_analysis.lock()
try:
fuzz_id, fuzz_url, html, messages = self.analysis_queue.popleft()
self.apply_fuzz_analysis(fuzz_id, fuzz_url, html, messages)
self.Data.update_dom_fuzzer_queue_item_status(
self.write_cursor, fuzz_id, 'C')
finally:
self.qlock_analysis.unlock()
def do_populate_domfuzzer_response_id(self, response_id):
self.qlock.lock()
try:
self.pending_fuzz_response_ids.append(response_id)
finally:
self.qlock.unlock()
QTimer.singleShot(10, self, SIGNAL('generateFuzzValues()'))
def do_populate_domfuzzer_response_list(self, id_list):
self.qlock.lock()
try:
for response_id in id_list:
self.pending_fuzz_response_ids.append(int(response_id))
finally:
self.qlock.unlock()
QTimer.singleShot(10, self, SIGNAL('generateFuzzValues()'))
def do_generateFuzzValues(self):
keep_looping = True
is_locked = False
try:
while keep_looping:
response_id = None
self.qlock.lock()
is_locked = True
if len(self.pending_fuzz_response_ids) > 0:
response_id = self.pending_fuzz_response_ids.popleft()
else:
keep_looping = False
self.qlock.unlock()
is_locked = False
if response_id is not None:
self.generate_fuzz_values(response_id)
finally:
if is_locked:
self.qlock.unlock()
def dispatch_next_fuzz_item(self):
if self.keep_fuzzing:
fuzz_item = self.get_next_fuzz_item()
if fuzz_item:
QObject.emit(
self.fuzz_callback,
SIGNAL('fuzzItemAvailable(int, QByteArray, QUrl)'),
fuzz_item[0], fuzz_item[1], fuzz_item[2])
else:
self.processed_urls = {}
QObject.emit(self.fuzz_callback, SIGNAL('fuzzRunFinished()'))
else:
self.processed_urls = {}
def get_next_fuzz_item(self):
self.qlock.lock()
locked = True
fuzz_item = None
try:
data_item = self.queueDataModel.popleft_data()
if not data_item:
self.keep_fuzzing = False
return
fuzz_id = data_item[DomFuzzerQueueTable.ID]
row = self.Data.read_responses_by_id(
self.read_cursor, data_item[DomFuzzerQueueTable.RESPONSE_ID])
if not row:
self.framework.log_warning('missing response id: %s' %
(response_id))
return
responseItems = interface.data_row_to_response_items(row)
target_url = self.compute_url_from_payload(data_item)
qurl = QUrl.fromEncoded(target_url)
dataContent = responseItems[ResponsesTable.RES_DATA]
# TODO: store reference
fuzz_item = (fuzz_id, dataContent, qurl)
self.qlock.unlock()
locked = False
finally:
if locked:
self.qlock.unlock()
return fuzz_item
def do_stopFuzzing(self):
print('do_stopFuzzing')
self.keep_fuzzing = False
def generate_fuzz_values(self, response_id):
row = self.Data.read_responses_by_id(self.read_cursor, response_id)
if not row:
self.framework.log_warning('missing response id: %s' %
(response_id))
return
responseItems = interface.data_row_to_response_items(row)
url = responseItems[ResponsesTable.URL]
contentType = responseItems[ResponsesTable.RES_CONTENT_TYPE]
responseBody = responseItems[ResponsesTable.RES_DATA]
if url in self.processed_urls:
self.framework.log_warning(
'skipping already scanned url [%s] for now' % (url))
return
else:
self.processed_urls[url] = True
# TODO: need better content type
if 'html' not in contentType:
if b'<html' not in responseBody.lower():
# non-html not supported
self.framework.log_warning(
'skipping not HTML request for [%s]: %s' %
(response_id, url))
return
fuzz_payloads = self.calculate_fuzz_payloads(url)
self.qlock.lock()
try:
# if self.fuzz_queue.has_key(url):
# # TODO: allow for better rescan in future
# self.framework.log_warning('adding already scanned url [%s] for now' % (url))
for payload in fuzz_payloads:
queue_item = [
None, response_id, payload[0], payload[1], payload[2],
payload[3], 'P'
]
rowid = self.Data.add_dom_fuzzer_queue_item(
self.write_cursor, queue_item)
queue_item[0] = rowid
self.queueDataModel.append_data([queue_item])
finally:
self.qlock.unlock()
def do_populateExistingFuzzData(self):
self.qlock.lock()
try:
rows = []
dup_rows = []
already_seen = {}
for row in self.Data.get_dom_fuzzer_queue_items(
self.read_cursor, 'P'):
data_item = [m or '' for m in row]
# check for duplicates and prioritize uniques first
url = data_item[DomFuzzerQueueTable.URL]
target = data_item[DomFuzzerQueueTable.TARGET]
param = data_item[DomFuzzerQueueTable.PARAM]
test = data_item[DomFuzzerQueueTable.TEST]
# TODO: remove this
# if 'fragment' == target or '#' == url[-1]:
# dup_rows.append(data_item)
# continue
splitted = urlparse.urlsplit(url)
if 'url' == target:
dupcheck = urlparse.urlunsplit(
(splitted.scheme, splitted.netloc, splitted.path, '',
''))
else:
qs_values = None
if 'query' == target and splitted.query:
qs_values = urlparse.parse_qs(splitted.query, True)
dupcheck = urlparse.urlunsplit(
(splitted.scheme, splitted.netloc, splitted.path,
'&'.join(list(qs_values.keys())),
splitted.fragment))
elif 'fragment' == target and splitted.fragment:
qs_values = urlparse.parse_qs(splitted.fragment, True)
dupcheck = urlparse.urlunsplit(
(splitted.scheme, splitted.netloc, splitted.path,
splitted.query, '&'.join(list(qs_values.keys()))))
else:
dupcheck = url
dupcheck = '%s||%s||%s' % (dupcheck, param, test)
if dupcheck not in already_seen:
rows.append(data_item)
already_seen[dupcheck] = True
else:
dup_rows.append(data_item)
self.queueDataModel.append_data(rows)
self.queueDataModel.append_data(dup_rows)
rows = []
for row in self.Data.read_dom_fuzzer_results_info(
self.read_cursor):
rows.append([m or '' for m in row])
self.resultsDataModel.append_data(rows)
finally:
self.qlock.unlock()
def do_clearFuzzQueue(self):
self.qlock.lock()
try:
self.Data.clear_dom_fuzzer_queue(self.write_cursor)
self.queueDataModel.clearModel()
finally:
self.qlock.unlock()
def calculate_fuzz_payloads(self, url):
payloads = []
splitted = urlparse.urlsplit(url)
if not splitted.query:
if not splitted.fragment:
payloads.extend(
self.calculate_fuzz_payload_tests(url + '?', 'url', ''))
payloads.extend(
self.calculate_fuzz_payload_tests(url + '#', 'url', ''))
else:
mangled = urlparse.urlunsplit(
(splitted.scheme, splitted.netloc, splitted.path, '', ''))
mangled = '?#' + splitted.fragment
payloads.extend(
self.calculate_fuzz_payload_tests(mangled, 'url', ''))
if splitted.query:
payloads.extend(
self.calculate_fuzz_payload_tests(url, 'query',
splitted.query))
if splitted.fragment:
payloads.extend(
self.calculate_fuzz_payload_tests(url, 'fragment',
splitted.fragment))
if splitted.query and not splitted.fragment:
payloads.extend(self.calculate_fuzz_payload_tests(url, 'url', ''))
mangled = urlparse.urlunsplit((splitted.scheme, splitted.netloc,
splitted.path, '', splitted.query))
payloads.extend(
self.calculate_fuzz_payload_tests(mangled, 'fragment',
splitted.query))
return payloads
def calculate_fuzz_payload_tests(self, url, target, url_field):
payloads = []
for test in self.STANDARD_TESTS:
if '&' in url:
payloads.append((url + '&__', target, '', test + '=1'))
else:
payloads.append((url, target, '', test))
if url_field:
pairs = self.re_delim.split(url_field)
for test in self.STANDARD_TESTS:
for offset in range(0, len(pairs)):
values = pairs[offset]
if values == ';' or values == '&':
continue
if '=' in values:
name, value = values.split('=', 1)
else:
name, value = values, ''
payloads.append((url, target, name, test))
return payloads
def compute_url_from_payload(self, data_item):
url = data_item[DomFuzzerQueueTable.URL]
target = data_item[DomFuzzerQueueTable.TARGET]
param = data_item[DomFuzzerQueueTable.PARAM]
test = data_item[DomFuzzerQueueTable.TEST]
if 'url' == target:
if not param:
return url + test
else:
# TODO: fix me
return url + test + '=X'
splitted = urlparse.urlsplit(url)
if 'fragment' == target:
url_field = splitted.fragment
elif 'query' == target:
url_field = splitted.query
else:
raise Exception('unsupported target: %s' % (target))
if not url_field:
raise Exception('missing URL field in url: %s' % (url))
else:
# TODO: this duplicates previous work, so could consider pre-storing target urls?
url_io = StringIO()
pairs = self.re_delim.split(url_field)
found = False
for offset in range(0, len(pairs)):
values = pairs[offset]
if values == ';' or values == '&':
url_io.write(values)
continue
if '=' in values:
name, value = values.split('=', 1)
separator = '='
else:
name, value = values, ''
separator = ''
if name == param:
value += test
found = True
url_io.write(name)
url_io.write(separator)
url_io.write(value)
if not found:
url_io.write(test)
if 'fragment' == target:
target_url = urlparse.urlunsplit(
(splitted.scheme, splitted.netloc, splitted.path,
splitted.query, url_io.getvalue()))
elif 'query' == target:
target_url = urlparse.urlunsplit(
(splitted.scheme, splitted.netloc, splitted.path,
url_io.getvalue(), splitted.fragment))
return target_url
def apply_fuzz_analysis(self, fuzz_id, fuzz_url, html, messages):
# check higher confidence items first
found = False
confidence = ''
for message in messages:
if 'alert' == message[0] and message[2] in (self.NUM1, self.NUM2):
found = True
confidence = 'High'
if not found:
for test in self.STANDARD_TESTS:
if test in html:
found = True
confidence = 'Medium'
if not found:
if self.re_unique_marker_base.search(html):
confidence = 'Low'
if found:
fuzz_item = self.Data.get_dom_fuzzer_queue_item_by_id(
self.read_cursor2, fuzz_id)
if not fuzz_item:
self.framework.log_warning('missing fuzz_id [%s]' % fuzz_id)
else:
flatten_url = fuzz_url
rendered_data = html.encode('utf-8', 'xmlcharrefreplace')
fuzz_results = [
None, fuzz_id, flatten_url,
fuzz_item[DomFuzzerQueueTable.TARGET],
fuzz_item[DomFuzzerQueueTable.PARAM],
fuzz_item[DomFuzzerQueueTable.TEST], confidence,
rendered_data
]
rowid = self.Data.add_dom_fuzzer_results_item(
self.write_cursor, fuzz_results)
fuzz_results[0] = rowid
self.resultsDataModel.append_data([fuzz_results])
class SplitThread(QThread):
def __init__(self, layer, splitField, encoding, outDir):
QThread.__init__(self, QThread.currentThread())
self.layer = layer
self.field = splitField
self.encoding = encoding
self.outDir = outDir
self.mutex = QMutex()
self.stopMe = 0
self.errors = []
def run(self):
self.mutex.lock()
self.stopMe = 0
self.mutex.unlock()
interrupted = False
outPath = self.outDir
if outPath.find("\\") != -1:
outPath.replace("\\", "/")
if not outPath.endswith("/"):
outPath = outPath + "/"
provider = self.layer.dataProvider()
index = provider.fieldNameIndex(self.field)
unique = ftools_utils.getUniqueValues(provider, int(index))
baseName = unicode( outPath + self.layer.name() + "_" + self.field + "_" )
fieldList = ftools_utils.getFieldList(self.layer)
sRs = provider.crs()
geom = self.layer.wkbType()
inFeat = QgsFeature()
self.emit(SIGNAL("rangeCalculated(PyQt_PyObject)"), len(unique))
for i in unique:
check = QFile(baseName + "_" + unicode(i).strip() + ".shp")
fName = check.fileName()
if check.exists():
if not QgsVectorFileWriter.deleteShapeFile(fName):
self.errors.append( fName )
continue
writer = QgsVectorFileWriter(fName, self.encoding, fieldList, geom, sRs)
fit = provider.getFeatures()
while fit.nextFeature(inFeat):
atMap = inFeat.attributes()
if atMap[index] == i:
writer.addFeature(inFeat)
del writer
self.emit(SIGNAL("valueProcessed()"))
self.mutex.lock()
s = self.stopMe
self.mutex.unlock()
if s == 1:
interrupted = True
break
if not interrupted:
self.emit(SIGNAL("processFinished( PyQt_PyObject )"), self.errors)
else:
self.emit(SIGNAL("processInterrupted()"))
def stop(self):
self.mutex.lock()
self.stopMe = 1
self.mutex.unlock()
QThread.wait(self)
class Walker(QThread):
COMMON_WORDS_THRESHOLD = 250
MIN_WORD_LEN = 3
MAX_WORD_LEN = 25
INVALID_FIRST_OR_LAST = frozenset("0123456789_")
STRIPHTML_RE = re.compile(r"<[^>]*?>", re.IGNORECASE | re.MULTILINE)
ENTITY_RE = re.compile(r"&(\w+?);|&#(\d+?);")
SPLIT_RE = re.compile(r"\W+", re.IGNORECASE | re.MULTILINE)
def __init__(self, lock, parent=None):
super(Walker, self).__init__(parent)
self.lock = lock
self.stopped = False
self.mutex = QMutex()
self.path = None
self.completed = False
def initialize(self, path, filenamesForWords, commonWords):
self.stopped = False
self.path = path
self.filenamesForWords = filenamesForWords
self.commonWords = commonWords
self.completed = False
def stop(self):
try:
self.mutex.lock()
self.stopped = True
finally:
self.mutex.unlock()
def isStopped(self):
try:
self.mutex.lock()
return self.stopped
finally:
self.mutex.unlock()
def run(self):
self.processFiles(self.path)
self.stop()
self.emit(SIGNAL("finished(bool)"), self.completed)
def processFiles(self, path):
def unichrFromEntity(match):
text = match.group(match.lastindex)
if text.isdigit():
return chr(int(text))
u = html.entities.name2codepoint.get(text)
return chr(u) if u is not None else ""
for root, dirs, files in os.walk(path):
if self.isStopped():
return
for name in [
name for name in files if name.endswith((".htm", ".html"))
]:
fname = os.path.join(root, name)
if self.isStopped():
return
words = set()
fh = None
try:
fh = codecs.open(fname, "r", "UTF8", "ignore")
text = fh.read()
except EnvironmentError as e:
sys.stderr.write("Error: {0}\n".format(e))
continue
finally:
if fh is not None:
fh.close()
if self.isStopped():
return
text = self.STRIPHTML_RE.sub("", text)
text = self.ENTITY_RE.sub(unichrFromEntity, text)
text = text.lower()
for word in self.SPLIT_RE.split(text):
if (self.MIN_WORD_LEN <= len(word) <= self.MAX_WORD_LEN
and word[0] not in self.INVALID_FIRST_OR_LAST
and word[-1] not in self.INVALID_FIRST_OR_LAST):
try:
self.lock.lockForRead()
new = word not in self.commonWords
finally:
self.lock.unlock()
if new:
words.add(word)
if self.isStopped():
return
for word in words:
try:
self.lock.lockForWrite()
files = self.filenamesForWords[word]
if len(files) > self.COMMON_WORDS_THRESHOLD:
del self.filenamesForWords[word]
self.commonWords.add(word)
else:
files.add(str(fname))
finally:
self.lock.unlock()
self.emit(SIGNAL("indexed(QString)"), fname)
self.completed = True
class StdErrWrapper(QObject):
_write = pyqtSignal(type(u''))
_flush = pyqtSignal()
def __init__(self, old_stderr):
QObject.__init__(self)
self._old_stderr = old_stderr
self._widget = None
self._mutex = QMutex(QMutex.Recursive)
def setApplication(self, app):
assert (self._widget is None)
widget = QPlainTextEdit()
widget.setWindowTitle(u"Error Console")
widget.resize(486, 300)
widget.appendHtml(
u'<span style="color: green">'
u'An unhandled error occurred.<br>'
u'Sorry for the inconvinience.<br>'
u'Please copy the following text into a bug report:<br><br>'
u'</span>')
app.aboutToQuit.connect(self.restoreStdErr)
self._write.connect(self._write_handler)
self._flush.connect(self._flush_handler)
self._widget = widget
def _write_handler(self, data):
self._mutex.lock()
if self._widget:
self._widget.show()
self._widget.insertPlainText(data)
self._mutex.unlock()
def _flush_handler(self):
self._mutex.lock()
if self._widget:
self._widget.show()
self._mutex.unlock()
def restoreStdErr(self):
self._mutex.lock()
if self._widget:
self._widget.close()
self._widget = None
self._mutex.unlock()
@property
def encoding(self):
return 'utf-8'
def write(self, s):
self._mutex.lock()
if self._widget:
if isinstance(s, bytes):
s = s.decode('utf-8', 'replace')
self._write.emit(s)
else:
self._old_stderr.write(s)
self._mutex.unlock()
def flush(self):
self._mutex.lock()
if self._widget:
self._flush.emit()
else:
self._old_stderr.flush()
self._mutex.unlock()
class Walker(QThread):
finished = Signal(bool, int)
indexed = Signal(QString, int)
COMMON_WORDS_THRESHOLD = 250
MIN_WORD_LEN = 3
MAX_WORD_LEN = 25
INVALID_FIRST_OR_LAST = frozenset("0123456789_")
STRIPHTML_RE = re.compile(r"<[^>]*?>", re.IGNORECASE|re.MULTILINE)
ENTITY_RE = re.compile(r"&(\w+?);|&#(\d+?);")
SPLIT_RE = re.compile(r"\W+", re.IGNORECASE|re.MULTILINE)
def __init__(self, index, lock, files, filenamesForWords,
commonWords, parent=None):
super(Walker, self).__init__(parent)
self.index = index
self.lock = lock
self.files = files
self.filenamesForWords = filenamesForWords
self.commonWords = commonWords
self.stopped = False
self.mutex = QMutex()
self.completed = False
def stop(self):
try:
self.mutex.lock()
self.stopped = True
finally:
self.mutex.unlock()
def isStopped(self):
try:
self.mutex.lock()
return self.stopped
finally:
self.mutex.unlock()
def run(self):
self.processFiles()
self.stop()
self.finished.emit(self.completed, self.index)
def processFiles(self):
for fname in self.files:
if self.isStopped():
return
self.processOneFile(fname)
self.completed = True
def processOneFile(self, fname):
words = set()
text = self.readFile(fname)
if text is None or self.isStopped():
return
text = self.STRIPHTML_RE.sub("", text)
text = self.ENTITY_RE.sub(unichrFromEntity, text)
text = text.lower()
for word in self.SPLIT_RE.split(text):
if (self.MIN_WORD_LEN <= len(word) <=
self.MAX_WORD_LEN and
word[0] not in self.INVALID_FIRST_OR_LAST and
word[-1] not in self.INVALID_FIRST_OR_LAST):
try:
self.lock.lockForRead()
new = word not in self.commonWords
finally:
self.lock.unlock()
if new:
words.add(word)
if self.isStopped():
return
for word in words:
try:
self.lock.lockForWrite()
files = self.filenamesForWords[word]
if len(files) > self.COMMON_WORDS_THRESHOLD:
del self.filenamesForWords[word]
self.commonWords.add(word)
else:
files.add(unicode(fname))
finally:
self.lock.unlock()
self.indexed.emit(fname, self.index)
def readFile(self, fname):
try:
with codecs.open(fname, "r", "UTF8", "ignore") as file:
return file.read()
except (IOError, OSError), err:
sys.stderr.write("Error: {0}\n".format(err))
class SplitThread(QThread):
def __init__(self, layer, splitField, encoding, outDir):
QThread.__init__(self, QThread.currentThread())
self.layer = layer
self.field = splitField
self.encoding = encoding
self.outDir = outDir
self.mutex = QMutex()
self.stopMe = 0
self.errors = []
def run(self):
self.mutex.lock()
self.stopMe = 0
self.mutex.unlock()
interrupted = False
outPath = self.outDir
if outPath.find("\\") != -1:
outPath.replace("\\", "/")
if not outPath.endswith("/"):
outPath = outPath + "/"
provider = self.layer.dataProvider()
index = provider.fieldNameIndex(self.field)
unique = ftools_utils.getUniqueValues(provider, int(index))
baseName = unicode(outPath + self.layer.name() + "_" + self.field +
"_")
fieldList = ftools_utils.getFieldList(self.layer)
sRs = provider.crs()
geom = self.layer.wkbType()
inFeat = QgsFeature()
self.emit(SIGNAL("rangeCalculated(PyQt_PyObject)"), len(unique))
for i in unique:
check = QFile(baseName + "_" + unicode(i).strip() + ".shp")
fName = check.fileName()
if check.exists():
if not QgsVectorFileWriter.deleteShapeFile(fName):
self.errors.append(fName)
continue
writer = QgsVectorFileWriter(fName, self.encoding, fieldList, geom,
sRs)
fit = provider.getFeatures()
while fit.nextFeature(inFeat):
atMap = inFeat.attributes()
if atMap[index] == i:
writer.addFeature(inFeat)
del writer
self.emit(SIGNAL("valueProcessed()"))
self.mutex.lock()
s = self.stopMe
self.mutex.unlock()
if s == 1:
interrupted = True
break
if not interrupted:
self.emit(SIGNAL("processFinished( PyQt_PyObject )"), self.errors)
else:
self.emit(SIGNAL("processInterrupted()"))
def stop(self):
self.mutex.lock()
self.stopMe = 1
self.mutex.unlock()
QThread.wait(self)
class GrowthComputationThread(QThread):
def __init__(self, parent):
QThread.__init__(self)
self.parent = parent
self.data = None
self.list_img = None
self._method = None
self._cells_selection = None
self.mutex = QMutex()
self.filename = None
self._stop = False
def __del__(self):
cleanQObject(self)
@property
def method(self):
return self._method
@method.setter
def method(self, method):
assert hasattr(method, "__call__"), "Invalid computation method, no '__call__' method"
assert hasattr(method, "thread"), "Invalid computation method, no 'thread' attribute"
assert hasattr(method, "parameters"), "Invalid computation method, no 'parameters' method"
self._method = method
@property
def cells_selection(self):
return self._cells_selection
@cells_selection.setter
def cells_selection(self, cells_selection):
assert hasattr(cells_selection, "__call__"), "Invalid cells selection method, no '__call__' method"
assert hasattr(cells_selection, "parameters"), "Invalid cells selection method, no 'parameters' method"
self._cells_selection = cells_selection
def valid(self):
if self.parent is None:
return False
if self.data is None:
return False
if self.list_img is None:
return False
if self.method is None:
return False
if self.filename is None:
return False
return True
def run(self):
self._stop = False
if not self.valid():
self.abort()
return
method = self.method
method.thread = self
result = method(self.list_img, self.data, self.cells_selection)
if result is None:
self.abort()
return
self.save(result)
self.finished()
def start(self):
from .parameters import instance
if instance.use_thread:
QThread.start(self)
else:
self.run()
def save(self, result):
result.method_params = self.method.parameters()
result.cells_selection_params = self.cells_selection.parameters()
return result.save(self.filename)
def stop(self):
self.mutex.lock()
self._stop = True
self.mutex.unlock()
def stopped(self):
self.mutex.lock()
stop = self._stop
self.mutex.unlock()
return stop
def nbOutputImages(self):
return self.method.nbOutputImages(self.list_img, self.data)
def abort(self):
QCoreApplication.postEvent(self.parent, AbortImageGrowthEvent())
def nextImage(self):
QCoreApplication.postEvent(self.parent, NextImageGrowthEvent())
def finished(self):
QCoreApplication.postEvent(self.parent, FinishImageGrowthEvent())
class Walker(QThread):
COMMON_WORDS_THRESHOLD = 250
MIN_WORD_LEN = 3
MAX_WORD_LEN = 25
INVALID_FIRST_OR_LAST = frozenset("0123456789_")
STRIPHTML_RE = re.compile(r"<[^>]*?>", re.IGNORECASE|re.MULTILINE)
ENTITY_RE = re.compile(r"&(\w+?);|&#(\d+?);")
SPLIT_RE = re.compile(r"\W+", re.IGNORECASE|re.MULTILINE)
def __init__(self, lock, parent=None):
super(Walker, self).__init__(parent)
self.lock = lock
self.stopped = False
self.mutex = QMutex()
self.path = None
self.completed = False
def initialize(self, path, filenamesForWords, commonWords):
self.stopped = False
self.path = path
self.filenamesForWords = filenamesForWords
self.commonWords = commonWords
self.completed = False
def stop(self):
try:
self.mutex.lock()
self.stopped = True
finally:
self.mutex.unlock()
def isStopped(self):
try:
self.mutex.lock()
return self.stopped
finally:
self.mutex.unlock()
def run(self):
self.processFiles(self.path)
self.stop()
self.emit(SIGNAL("finished(bool)"), self.completed)
def processFiles(self, path):
def unichrFromEntity(match):
text = match.group(match.lastindex)
if text.isdigit():
return unichr(int(text))
u = htmlentitydefs.name2codepoint.get(text)
return unichr(u) if u is not None else ""
for root, dirs, files in os.walk(path):
if self.isStopped():
return
for name in [name for name in files
if name.endswith((".htm", ".html"))]:
fname = os.path.join(root, name)
if self.isStopped():
return
words = set()
fh = None
try:
fh = codecs.open(fname, "r", "UTF8", "ignore")
text = fh.read()
except (IOError, OSError), e:
sys.stderr.write("Error: {0}\n".format(e))
continue
finally:
if fh is not None:
fh.close()
class ShapeMergeThread(QThread):
def __init__(self, dir, shapes, inputEncoding, outputFileName, outputEncoding):
QThread.__init__(self, QThread.currentThread())
self.baseDir = dir
self.shapes = shapes
self.inputEncoding = inputEncoding
self.outputFileName = outputFileName
self.outputEncoding = outputEncoding
self.mutex = QMutex()
self.stopMe = 0
def run(self):
self.mutex.lock()
self.stopMe = 0
self.mutex.unlock()
interrupted = False
# create attribute list with uniquie fields
# from all selected layers
mergedFields = []
self.emit(SIGNAL("rangeChanged( PyQt_PyObject )"), len(self.shapes))
self.emit(SIGNAL("checkStarted()"))
shapeIndex = 0
fieldMap = {}
for fileName in self.shapes:
layerPath = QFileInfo(self.baseDir + "/" + fileName).absoluteFilePath()
newLayer = QgsVectorLayer(layerPath, QFileInfo(layerPath).baseName(), "ogr")
if not newLayer.isValid():
continue
newLayer.setProviderEncoding(self.inputEncoding)
vprovider = newLayer.dataProvider()
fieldMap[shapeIndex] = {}
fieldIndex = 0
for layerField in vprovider.fields():
fieldFound = False
for mergedFieldIndex, mergedField in enumerate(mergedFields):
if mergedField.name() == layerField.name() and mergedField.type() == layerField.type():
fieldFound = True
fieldMap[shapeIndex][fieldIndex] = mergedFieldIndex
if mergedField.length() < layerField.length():
# suit the field size to the field of this layer
mergedField.setLength(layerField.length())
break
if not fieldFound:
fieldMap[shapeIndex][fieldIndex] = len(mergedFields)
mergedFields.append(layerField)
fieldIndex += 1
shapeIndex += 1
self.emit(SIGNAL("featureProcessed()"))
self.emit(SIGNAL("checkFinished()"))
# get information about shapefiles
layerPath = QFileInfo(self.baseDir + "/" + self.shapes[0]).absoluteFilePath()
newLayer = QgsVectorLayer(layerPath, QFileInfo(layerPath).baseName(), "ogr")
self.crs = newLayer.crs()
self.geom = newLayer.wkbType()
vprovider = newLayer.dataProvider()
fields = QgsFields()
for f in mergedFields:
fields.append(f)
writer = QgsVectorFileWriter(
self.outputFileName, self.outputEncoding,
fields, self.geom, self.crs)
shapeIndex = 0
for fileName in self.shapes:
layerPath = QFileInfo(self.baseDir + "/" + fileName).absoluteFilePath()
newLayer = QgsVectorLayer(layerPath, QFileInfo(layerPath).baseName(), "ogr")
if not newLayer.isValid():
continue
newLayer.setProviderEncoding(self.inputEncoding)
vprovider = newLayer.dataProvider()
nFeat = vprovider.featureCount()
self.emit(SIGNAL("rangeChanged( PyQt_PyObject )"), nFeat)
self.emit(SIGNAL("fileNameChanged( PyQt_PyObject )"), fileName)
inFeat = QgsFeature()
outFeat = QgsFeature()
inGeom = QgsGeometry()
fit = vprovider.getFeatures()
while fit.nextFeature(inFeat):
mergedAttrs = [""] * len(mergedFields)
# fill available attributes with values
fieldIndex = 0
for v in inFeat.attributes():
if shapeIndex in fieldMap and fieldIndex in fieldMap[shapeIndex]:
mergedAttrs[fieldMap[shapeIndex][fieldIndex]] = v
fieldIndex += 1
if inFeat.geometry() is not None:
inGeom = QgsGeometry(inFeat.geometry())
outFeat.setGeometry(inGeom)
outFeat.setAttributes(mergedAttrs)
writer.addFeature(outFeat)
self.emit(SIGNAL("featureProcessed()"))
self.emit(SIGNAL("shapeProcessed()"))
self.mutex.lock()
s = self.stopMe
self.mutex.unlock()
if s == 1:
interrupted = True
break
shapeIndex += 1
del writer
if not interrupted:
self.emit(SIGNAL("processingFinished()"))
else:
self.emit(SIGNAL("processingInterrupted()"))
def stop(self):
self.mutex.lock()
self.stopMe = 1
self.mutex.unlock()
QThread.wait(self)