class QtStreamHandler(logging.Handler):
def __init__(self, parent, main):
logging.Handler.__init__(self)
self.parent = parent
self.main = main
self.textWidget = parent
self.formater = logging.Formatter("%(asctime)s - %(levelname)s -> %(message)s")
self.logString = ""
def getLoggingCompleteText(self):
return self.logString
def createLock(self):
self.mutex = QMutex()
def acquire(self):
self.mutex.lock()
def release(self):
self.mutex.unlock()
def emit(self, record):
# self.textWidget.appendPlainText(self.formater.format(record))
formattedText = self.formater.format(record)
#separate text from date
index = formattedText.find('->')
self.textWidget.setTextColor(QColor("green"))
self.textWidget.insertPlainText(formattedText[:index])
self.textWidget.setTextColor(QColor("blue"))
self.textWidget.insertPlainText(formattedText[index:] + "\n")
self.textWidget.moveCursor(QTextCursor.EndOfLine)
self.textWidget.ensureCursorVisible()
self.logString += formattedText + "\n";
class MainW(QtGui.QMainWindow):
"""
Mainw to host and run one single thread
"""
def __init__(self):
QtGui.QMainWindow.__init__(self)
self.mutex = QMutex()
self.thr = GuiThread(process=self.fun, period=0.1)
self.thr2 = GuiThread(process=self.fun2, period=0.1)
self.buff = CircIoBuffer(byte_size=15 * len('rafal miecznik'))
self.thr.start()
self.thr2.start()
time.sleep(2)
print(self.buff.read())
for d in dir(self.mutex):
print(d)
def fun(self):
print(self.mutex.tryLock())
self.buff.write(' rafal')
self.mutex.unlock()
#print self.thr.currentThreadId()
def fun2(self):
self.mutex.lock()
self.buff.write(' miecznik')
class Feed(QRunnable):
def __init__(self, label, parent=None):
QRunnable.__init__(self)
self.label = label
self.parent = parent
self.data = None
self.mutex = QMutex()
self.setAutoDelete(True)
def run(self):
self.check()
def check(self):
try:
self.fetch()
except (urllib2.URLError, urllib2.HTTPError) as err:
self.parent.error.emit(str(err))
else:
self.read()
if self.data:
self.parse()
def fetch(self):
auth_handler = urllib2.HTTPBasicAuthHandler()
auth_handler.add_password(REALM, HOST, self.parent.user,
self.parent.passwd)
opener = urllib2.build_opener(auth_handler)
urllib2.install_opener(opener)
url = "%s/%s" % (URL_ATOM, self.label) if self.label else URL_ATOM
self.conn = urllib2.urlopen(url)
def read(self):
code = self.conn.getcode()
if code != 200:
self.conn.close()
self.parent.error.emit(
"HTTP Error %d: %s" %
(code, BaseHTTPRequestHandler.responses[code]))
else:
self.data = self.conn.read()
self.conn.close()
def parse(self):
tree = et.fromstring(self.data)
for e in tree.findall("%sentry" % XMLNS):
entry = Entry()
entry.title = e.find("%stitle" % XMLNS).text
entry.summary = e.find("%ssummary" % XMLNS).text
entry.link = e.find("%slink" % XMLNS).attrib["href"]
entry.modified = e.find("%smodified" % XMLNS).text
entry.issued = e.find("%sissued" % XMLNS).text
entry.id = e.find("%sid" % XMLNS).text
for a in e.findall("%sauthor" % XMLNS):
entry.author_name = a.find("%sname" % XMLNS).text
entry.author_email = a.find("%semail" % XMLNS).text
self.mutex.lock()
self.parent.entries.append((self.label, entry))
self.mutex.unlock()
class Feed(QRunnable):
def __init__(self, label, parent=None):
QRunnable.__init__(self)
self.label = label
self.parent = parent
self.data = None
self.mutex = QMutex()
self.setAutoDelete(True)
def run(self):
self.check()
def check(self):
try:
self.fetch()
except (urllib2.URLError, urllib2.HTTPError) as err:
self.parent.error.emit(str(err))
else:
self.read()
if self.data:
self.parse()
def fetch(self):
auth_handler = urllib2.HTTPBasicAuthHandler()
auth_handler.add_password(REALM, HOST,
self.parent.user, self.parent.passwd)
opener = urllib2.build_opener(auth_handler)
urllib2.install_opener(opener)
url = "%s/%s" % (URL_ATOM, self.label) if self.label else URL_ATOM
self.conn = urllib2.urlopen(url)
def read(self):
code = self.conn.getcode()
if code != 200:
self.conn.close()
self.parent.error.emit("HTTP Error %d: %s" % (
code, BaseHTTPRequestHandler.responses[code]))
else:
self.data = self.conn.read()
self.conn.close()
def parse(self):
tree = et.fromstring(self.data)
for e in tree.findall("%sentry" % XMLNS):
entry = Entry()
entry.title = e.find("%stitle" % XMLNS).text
entry.summary = e.find("%ssummary" % XMLNS).text
entry.link = e.find("%slink" % XMLNS).attrib["href"]
entry.modified = e.find("%smodified" % XMLNS).text
entry.issued = e.find("%sissued" % XMLNS).text
entry.id = e.find("%sid" % XMLNS).text
for a in e.findall("%sauthor" % XMLNS):
entry.author_name = a.find("%sname" % XMLNS).text
entry.author_email = a.find("%semail" % XMLNS).text
self.mutex.lock()
self.parent.entries.append((self.label, entry))
self.mutex.unlock()
class QCallbacksManager(QObject):
class Request(object):
def __init__(self):
self.event = Event()
self.answer = None
def __call__(self):
raise NotImplementedError()
class LoginRequest(Request):
def __init__(self, backend_name, value):
QCallbacksManager.Request.__init__(self)
self.backend_name = backend_name
self.value = value
def __call__(self):
password, ok = QInputDialog.getText(
None, '%s request' % self.value.label,
'Please enter %s for %s' %
(self.value.label, self.backend_name), QLineEdit.Password)
return password
new_request = Signal()
def __init__(self, weboob, parent=None):
QObject.__init__(self, parent)
self.weboob = weboob
self.weboob.requests.register('login',
self.callback(self.LoginRequest))
self.mutex = QMutex()
self.requests = []
self.new_request.connect(self.do_request)
def callback(self, klass):
def cb(*args, **kwargs):
return self.add_request(klass(*args, **kwargs))
return cb
@Slot()
def do_request(self):
self.mutex.lock()
request = self.requests.pop()
request.answer = request()
request.event.set()
self.mutex.unlock()
def add_request(self, request):
self.mutex.lock()
self.requests.append(request)
self.mutex.unlock()
self.new_request.emit()
request.event.wait()
return request.answer
class QCallbacksManager(QObject):
class Request(object):
def __init__(self):
self.event = Event()
self.answer = None
def __call__(self):
raise NotImplementedError()
class LoginRequest(Request):
def __init__(self, backend_name, value):
QCallbacksManager.Request.__init__(self)
self.backend_name = backend_name
self.value = value
def __call__(self):
password, ok = QInputDialog.getText(None,
'%s request' % self.value.label,
'Please enter %s for %s' % (self.value.label,
self.backend_name),
QLineEdit.Password)
return password
new_request = Signal()
def __init__(self, weboob, parent=None):
QObject.__init__(self, parent)
self.weboob = weboob
self.weboob.callbacks['login'] = self.callback(self.LoginRequest)
self.mutex = QMutex()
self.requests = []
self.new_request.connect(self.do_request)
def callback(self, klass):
def cb(*args, **kwargs):
return self.add_request(klass(*args, **kwargs))
return cb
@Slot()
def do_request(self):
self.mutex.lock()
request = self.requests.pop()
request.answer = request()
request.event.set()
self.mutex.unlock()
def add_request(self, request):
self.mutex.lock()
self.requests.append(request)
self.mutex.unlock()
self.new_request.emit()
request.event.wait()
return request.answer
class ImporterThread(QThread):
def __init__(self, framework, parent=None):
QThread.__init__(self, parent)
self.framework = framework
self.qlock = QMutex()
self.cursor = None
QObject.connect(self, SIGNAL('quit()'), self.quitHandler)
QObject.connect(self, SIGNAL('started()'), self.startedHandler)
def run(self):
QObject.connect(self, SIGNAL('do_runImport()'), self.handle_runImport,
Qt.DirectConnection)
self.exec_()
def quitHandler(self):
if self.cursor:
self.cursor.close()
self.exit(0)
def startedHandler(self):
pass
def runImport(self, importers, proxy_file, source, callback):
self.importers = importers
self.proxy_filelist = [proxy_file]
self.source = source
self.callbackObj = callback
QTimer.singleShot(50, self, SIGNAL('do_runImport()'))
def runImportList(self, importers, proxy_filelist, source, callback):
self.importers = importers
self.proxy_filelist = proxy_filelist
self.source = source
self.callbackObj = callback
QTimer.singleShot(50, self, SIGNAL('do_runImport()'))
def handle_runImport(self):
if self.qlock.tryLock():
try:
for proxy_file in self.proxy_filelist:
try:
self.framework.debug_log('attempting import of %s' %
(str(proxy_file)))
self.importers.process_import(str(proxy_file),
self.framework,
self.source)
except Exception as ex:
self.framework.report_exception(ex)
finally:
self.qlock.unlock()
self.callbackObj.emit(SIGNAL('runImportFinished()'))
class MyStreamHandler(StreamHandler):
def __init__(self):
StreamHandler.__init__(self)
def createLock(self):
# must be Recursive (= reentrant)
self._mutex = QMutex(QMutex.Recursive)
def acquire(self):
self._mutex.lock()
def release(self):
self._mutex.unlock()
class MyStreamHandler(StreamHandler):
def __init__(self):
StreamHandler.__init__(self)
def createLock(self):
# must be Recursive (= reentrant)
self._mutex = QMutex(QMutex.Recursive)
def acquire(self):
self._mutex.lock()
def release(self):
self._mutex.unlock()
class Lock(object):
def __init__(self):
self._mutex = QMutex()
def __enter__(self):
self._mutex.lock()
def __exit__(self, type, value, tb):
self._mutex.unlock()
def acquire(self):
self._mutex.lock()
def release(self):
self._mutex.unlock()
class ImporterThread(QThread):
def __init__(self, framework, parent = None):
QThread.__init__(self, parent)
self.framework = framework
self.qlock = QMutex()
self.cursor = None
QObject.connect(self, SIGNAL('quit()'), self.quitHandler)
QObject.connect(self, SIGNAL('started()'), self.startedHandler)
def run(self):
QObject.connect(self, SIGNAL('do_runImport()'), self.handle_runImport, Qt.DirectConnection)
self.exec_()
def quitHandler(self):
if self.cursor:
self.cursor.close()
self.exit(0)
def startedHandler(self):
pass
def runImport(self, importers, proxy_file, source, callback):
self.importers = importers
self.proxy_filelist = [proxy_file]
self.source = source
self.callbackObj = callback
QTimer.singleShot(50, self, SIGNAL('do_runImport()'))
def runImportList(self, importers, proxy_filelist, source, callback):
self.importers = importers
self.proxy_filelist = proxy_filelist
self.source = source
self.callbackObj = callback
QTimer.singleShot(50, self, SIGNAL('do_runImport()'))
def handle_runImport(self):
if self.qlock.tryLock():
try:
for proxy_file in self.proxy_filelist:
try:
self.framework.debug_log('attempting import of %s' % (str(proxy_file)))
self.importers.process_import(str(proxy_file), self.framework, self.source)
except Exception as ex:
self.framework.report_exception(ex)
finally:
self.qlock.unlock()
self.callbackObj.emit(SIGNAL('runImportFinished()'))
class Lock(object):
def __init__(self):
self._mutex = QMutex()
def __enter__(self):
self._mutex.lock()
def __exit__(self, type, value, tb):
self._mutex.unlock()
def acquire(self):
self._mutex.lock()
def release(self):
self._mutex.unlock()
class Evaluate(QThread):
"""Thread used to insolate calculation process in entities (stream, project
and equipment, so gui can response while calculation is in process"""
def __init__(self, parent=None):
super(Evaluate, self).__init__(parent)
self.mutex = QMutex()
def start(self, entity, kwargs):
self.entity = entity
self.kwargs = kwargs
QThread.start(self)
def run(self):
self.mutex.lock()
self.entity(**self.kwargs)
self.mutex.unlock()
class UdpClient(QThread): def __init__(self, parent): QThread.__init__(self, parent) self.prnt = parent self.udp = QUdpSocket() addr = QHostAddress(QHostAddress.Any) print 'bind to:', addr.toString() self.udp.bind(addr, 34001) self.udp.error.connect(self.errorAnalyser) self.udp.readyRead.connect(self.readUdp) print "Binding..." self.STOP = False self.locker = QMutex() def run(self): self.prnt.initAvahiService() while True : if self.udp is not None and self.udp.state() == QAbstractSocket.ConnectedState : self.udp.waitForReadyRead() else : self.msleep(100) if self.STOP and self.udp is not None: self.udp.close(); break print 'UDPClient closed...' self.prnt.changeConnectState.emit() def stop(self): self.locker.lock() self.STOP = True self.locker.unlock() def readUdp(self): while ( self.udp.hasPendingDatagrams() ): data = QByteArray() addr = QHostAddress() port = 0 try : datagramSize = self.udp.pendingDatagramSize() if datagramSize > 0 : (data, addr, port) = self.udp.readDatagram(datagramSize) #print "Datagram: [%s] from %s:%i" % (QString().fromUtf8(data), addr.toString(), port) self.prnt.contactMessage.emit(QString().fromUtf8(data), addr.toString()) except socket.error, err : print '[in readUdp() UdpClient] SocketError1 : ', err except socket.timeout, err : print '[in readUdp() UdpClient] SocketError2 : ', err except :
class ComputationThread(QThread):
def __init__(self, animation_thread):
QThread.__init__(self)
self.is_running = True
self.mutex = QMutex()
self.other_thread = animation_thread
def stop(self):
self.is_running = False
def run(self):
while self.is_running:
self.mutex.lock() # locking may not be necessary
raw_input("")
#self.other_thread.stop( )
self.mutex.unlock()
print " run method ends "
class SQLRenderThread(CachedRenderThread):
"""
RenderThread extending the UniqueMethodRenderThread by supplying a cancel
method for classes with database access. The database object needs to be
given either on the object as .db subobject or explicitly by
setObjectDBObject().
The database object needs to have a 'connection' attribute which supports an
interrupt() method.
"""
def __init__(self, parent=0):
self.dbObjectLock = QMutex(QMutex.Recursive)
self.dbObject = {}
CachedRenderThread.__init__(self, parent)
def setObject(self, classObject, *args, **param):
self.dbObjectLock.lock()
if classObject in self.dbObject:
del self.dbObject[classObject]
self.dbObjectLock.unlock()
CachedRenderThread.setObject(self, classObject, *args, **param)
def setObjectDBObject(self, classObject, db):
self.dbObjectLock.lock()
self.dbObject[classObject] = db
self.dbObjectLock.unlock()
def cancelCurrentJob(self):
QMutexLocker(self.renderingLock)
if self.currentlyRenderingJob:
_, classObject, _, _, _ = self.currentlyRenderingJob
try:
QMutexLocker(self.dbObjectLock)
if classObject in self.dbObject:
db = self.dbObject[classObject]
else:
classObjectInst = self.getObjectInstance(classObject)
if hasattr(classObjectInst, 'db'):
db = classObjectInst.db
else:
return False
self.clearCurrentJob()
db.connection.interrupt()
return True
except KeyError:
return False
class ImageTile(object):
def __init__(self, rect):
self._mutex = QMutex()
self.image = QImage(rect.width(), rect.height(),
QImage.Format_ARGB32_Premultiplied)
self.image.fill(0x00000000)
self._topLeft = rect.topLeft()
# Whenever the underlying data changes, the data version is incremented.
# By comparing the data version to the image and request version, it can
# be determined if the content of this tile is recent or needs to be
# re-computed.
# version of the data
self.dataVer = 0
# version of self.image
#
# If self.imgVer < self.dataVer, the image needs to be re-computed
# from the new data.
self.imgVer = -1
# version of the request that has been generated to update the contents
# of self.image
#
# If self.reqVer == self.dataVer, a request is currently running that will
# eventually replace self.image with the new data.
self.reqVer = -2
def clear(self):
self.image.fill(0)
def paint(self, painter):
self.lock()
painter.drawImage(self._topLeft, self.image)
self.unlock()
def lock(self):
self._mutex.lock()
def unlock(self):
self._mutex.unlock()
class ImageTile(object):
def __init__(self, rect):
self._mutex = QMutex()
self.image = QImage(rect.width(), rect.height(),
QImage.Format_ARGB32_Premultiplied)
self.image.fill(0)
self._topLeft = rect.topLeft()
#Whenever the underlying data changes, the data version is incremented.
#By comparing the data version to the image and request version, it can
#be determined if the content of this tile is recent or needs to be
#re-computed.
#version of the data
self.dataVer = 0
#version of self.image
#
#If self.imgVer < self.dataVer, the image needs to be re-computed
#from the new data.
self.imgVer = -1
#version of the request that has been generated to update the contents
#of self.image
#
#If self.reqVer == self.dataVer, a request is currently running that will
#eventually replace self.image with the new data.
self.reqVer = -2
def clear(self):
self.image.fill(0)
def paint(self, painter):
self.lock()
painter.drawImage(self._topLeft, self.image)
self.unlock()
def lock(self):
self._mutex.lock()
def unlock(self):
self._mutex.unlock()
class DatabaseThread(QThread):
def __init__(self, framework, Data, parent=None):
QThread.__init__(self, parent)
self.framework = framework
self.Data = Data
self.qlock = QMutex()
QObject.connect(self, SIGNAL('quit()'), self.quitHandler)
QObject.connect(self, SIGNAL('started()'), self.startedHandler)
def run(self):
QObject.connect(self, SIGNAL('doConnectDb()'), self.connectDbHandler,
Qt.DirectConnection)
self.exec_()
def close(self):
self.qlock.lock()
try:
self.Data.close()
finally:
self.qlock.unlock()
def quitHandler(self):
self.Data.close()
self.exit(0)
def startedHandler(self):
pass
def connectDb(self, filename, callback):
self.filename = filename
self.callbackObj = callback
QTimer.singleShot(50, self, SIGNAL('doConnectDb()'))
def connectDbHandler(self):
self.qlock.lock()
try:
self.Data.connect(self.filename)
finally:
self.qlock.unlock()
self.callbackObj.emit(SIGNAL('connectDbFinished()'))
class AnimationThread(QThread):
def __init__(self):
QThread.__init__(self)
self.is_running = True
self.mutex = QMutex()
def stop(self):
self.is_running = False
def get_state(self):
return self.is_running
def change_state(self):
self.is_running = not self.is_running
def run(self):
while self.is_running:
self.mutex.lock( ) # locking may not be necessary
self.emit(SIGNAL("window_update_request" ))
self.mutex.unlock()
sleep( 0.005 )
class _LoggingMutexQt(_LoggingMutex):
def __init__(self, name):
super(_LoggingMutexQt, self).__init__(name)
from PyQt4.QtCore import QMutex
self.mutex = QMutex()
def currentThreadID(self):
from PyQt4.QtCore import QThread
return QThread.currentThreadId()
def _acquire(self):
self.mutex.lock()
def _release(self):
self.mutex.unlock()
def enterMutex(self):
self.mutex.lock()
def exitMutex(self, *_args, **_kwargs):
self.mutex.unlock()
class communications:
def __init__(self):
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.s.connect((TCP_IP, TCP_PORT))
self.running = QMutex()
self.send_command = '.'
def command(self, comm):
self.running.lock()
self.send_command = comm
self.running.unlock()
def get_data(self):
self.running.lock()
self.s.send(self.send_command.encode())
values = pickle.loads(self.s.recv(BUFFER_SIZE))
self.send_command = '.'
self.running.unlock()
return values
def get_parameters(self):
self.running.lock()
self.s.send(b"?")
params = pickle.loads(self.s.recv(BUFFER_SIZE))
self.running.unlock()
return params
def __del__(self):
self.s.close()
class DatabaseThread(QThread):
def __init__(self, framework, Data, parent = None):
QThread.__init__(self, parent)
self.framework = framework
self.Data = Data
self.qlock = QMutex()
QObject.connect(self, SIGNAL('quit()'), self.quitHandler)
QObject.connect(self, SIGNAL('started()'), self.startedHandler)
def run(self):
QObject.connect(self, SIGNAL('doConnectDb()'), self.connectDbHandler, Qt.DirectConnection)
self.exec_()
def close(self):
self.qlock.lock()
try:
self.Data.close()
finally:
self.qlock.unlock()
def quitHandler(self):
self.Data.close()
self.exit(0)
def startedHandler(self):
pass
def connectDb(self, filename, callback):
self.filename = filename
self.callbackObj = callback
QTimer.singleShot(50, self, SIGNAL('doConnectDb()'))
def connectDbHandler(self):
self.qlock.lock()
try:
self.Data.connect(self.filename)
finally:
self.qlock.unlock()
self.callbackObj.emit(SIGNAL('connectDbFinished()'))
class SignalThread(QThread):
def __init__(self, parent=None):
QThread.__init__(self, parent)
self.waitcond = QWaitCondition()
self.mutex = QMutex()
self.isstopped = False
def trigger(self):
"""lock first to make sure the QThread is actually waiting for a signal"""
self.mutex.lock()
self.waitcond.wakeOne()
self.mutex.unlock()
def stopThread(self):
self.mutex.lock()
self.isstopped = True
self.waitcond.wakeOne()
self.mutex.unlock()
def run(self):
self.mutex.lock()
while not self.isstopped:
# just wait, and trigger every time we receive a signal
self.waitcond.wait(self.mutex)
if not self.isstopped:
self.emit(SIGNAL("triggerSignal()"))
self.mutex.unlock()
class SignalThread(QThread):
def __init__(self, parent = None):
QThread.__init__(self, parent)
self.waitcond = QWaitCondition()
self.mutex = QMutex()
self.isstopped = False
def trigger(self):
"""lock first to make sure the QThread is actually waiting for a signal"""
self.mutex.lock()
self.waitcond.wakeOne()
self.mutex.unlock()
def stopThread(self):
self.mutex.lock()
self.isstopped = True
self.waitcond.wakeOne()
self.mutex.unlock()
def run(self):
self.mutex.lock()
while not self.isstopped:
# just wait, and trigger every time we receive a signal
self.waitcond.wait(self.mutex)
if not self.isstopped:
self.emit(SIGNAL("triggerSignal()"))
self.mutex.unlock()
class Imap(QRunnable):
def __init__(self, label, parent=None):
QRunnable.__init__(self)
self.imap = None
self.label = label
self.parent = parent
self.mutex = QMutex()
self.login()
def run(self):
for imap_id in self.get_unread():
thread_id = self.get_thread_id(imap_id)
self.mutex.lock()
self.parent.idmap[thread_id] = imap_id
self.mutex.unlock()
def connect(self):
self.imap = imaplib.IMAP4_SSL(IMAP_SERVER, IMAP_PORT)
def login(self):
try:
if not self.imap:
self.connect()
auth = self.imap.login(self.parent.user, self.parent.passwd)
self.logged_in = (auth and auth[0] == "OK")
except imaplib.IMAP4.error:
pass
def logout(self):
pass
def get_unread(self):
self.imap.select(self.label.replace("-", "/"))
result, unseen = self.imap.search(None, "(UNSEEN)")
return unseen[0].split()
def get_thread_id(self, id):
result, data = self.imap.fetch(id, "(X-GM-THRID BODY.PEEK[HEADER])")
return data[0][0].split(" ")[2]
class Imap(QRunnable):
def __init__(self, label, parent=None):
QRunnable.__init__(self)
self.imap = None
self.label = label
self.parent = parent
self.mutex = QMutex()
self.login()
def run(self):
for imap_id in self.get_unread():
thread_id = self.get_thread_id(imap_id)
self.mutex.lock()
self.parent.idmap[thread_id] = imap_id
self.mutex.unlock()
def connect(self):
self.imap = imaplib.IMAP4_SSL(IMAP_SERVER, IMAP_PORT)
def login(self):
try:
if not self.imap:
self.connect()
auth = self.imap.login(self.parent.user, self.parent.passwd)
self.logged_in = (auth and auth[0] == "OK")
except imaplib.IMAP4.error:
pass
def logout(self):
pass
def get_unread(self):
self.imap.select(self.label.replace("-", "/"))
result, unseen = self.imap.search(None, "(UNSEEN)")
return unseen[0].split()
def get_thread_id(self, id):
result, data = self.imap.fetch(id, "(X-GM-THRID BODY.PEEK[HEADER])")
return data[0][0].split(" ")[2]
class WorkerThread(QThread):
do_something = pyqtSignal()
def __init__(self, name="", parent=None):
super().__init__(parent)
self.sleep_interval = 900
self.mutex = QMutex()
self.name = name
self.isStopped = False
def run(self):
while not self.isStopped:
# while True:
self.mutex.lock()
self.do_something.emit()
self.mutex.unlock()
self.msleep(self.sleep_interval)
self.finished.emit()
print(self.name + " finished.")
def stop(self):
with QMutexLocker(self.mutex):
self.isStopped = True
class VCSPluginThread( QThread ):
" Wrapper for the plugin thread "
def __init__( self, plugin, parent = None ):
QThread.__init__( self, parent )
self.__plugin = plugin
self.__requestQueue = deque()
self.__stopRequest = False
self.__lock = QMutex()
self.__condition = QWaitCondition()
return
def run( self ):
" Thread loop "
while not self.__stopRequest:
self.__lock.lock()
while self.__requestQueue:
path, flag = self.__requestQueue.pop()
self.__lock.unlock()
time.sleep( 0.01 )
self.__processRequest( path, flag )
if self.__stopRequest:
break
self.__lock.lock()
if self.__stopRequest:
self.__lock.unlock()
break
self.__condition.wait( self.__lock )
self.__lock.unlock()
return
def __processRequest( self, path, flag ):
" Processes a single request. It must be exception safe. "
try:
statuses = self.__plugin.getObject().getStatus( path, flag )
for status in statuses:
if len( status ) == 3:
self.emit( SIGNAL( "VCSStatus" ), path + status[ 0 ],
status[ 1 ], status[ 2 ] )
else:
self.emit( SIGNAL( "VCSStatus" ), path, IND_VCS_ERROR,
"The " + self.__plugin.getName() + " plugin "
"does not follow the getStatus() interface "
"agreement" )
except Exception, exc:
self.emit( SIGNAL( "VCSStatus" ), path, IND_VCS_ERROR,
"Exception in " + self.__plugin.getName() +
" plugin while retrieving VCS status: " + str( exc ) )
except:
class ThreadSerial(QThread):
dataUpdate = pyqtSignal(str)
def __init__(self, envs):
super(ThreadSerial, self).__init__()
self.serial = envs.get('SERIAL')
self.mutex = QMutex()
self.exit = False
def run(self):
debug("[+] Listening {}".format(self.serial.port))
while not self.exit:
if self.mutex.tryLock(100):
try:
data = self.serial.read_all()
finally:
self.mutex.unlock()
if data:
self.dataUpdate.emit(data)
time.sleep(0.001)
def pause(self, timeout=1000):
return self.mutex.tryLock(timeout)
def resume(self):
self.mutex.unlock()
def read_no_wait(self):
data = self.serial.read_all()
self.dataUpdate.emit(data)
return data
def write_no_wait(self, data):
self.serial.write(unicode(data).encode('utf-8'))
def write(self, data):
if self.mutex.tryLock(100):
try:
self.serial.write(unicode(data).encode('utf-8'))
finally:
self.mutex.unlock()
def terminate(self, timeout=1000):
debug("[-] Terminate {}".format(self.serial.port))
self.exit = True
if not self.wait(timeout):
QThread.terminate(self)
self.serial.close()
class TypeWorker(QThread):
def __init__(self, *args):
QThread.__init__(self)
self.typeQueue = Queue()
self.regImage = re.compile("(JPEG|JPG|jpg|jpeg|GIF|gif|bmp|BMP|png|PNG|pbm|PBM|pgm|PGM|ppm|PPM|xpm|XPM|xbm|XBM|TIFF|tiff).*", re.IGNORECASE)
self.typeQueue = []
self.setUniq = set()
self.qmutex = QMutex()
self.qsem = QSemaphore()
def enqueue(self, parent, index, node):
self.qmutex.lock()
if long(node.this) not in self.setUniq:
self.typeQueue.insert(0, (parent, index, node))
self.setUniq.add(long(node.this))
self.qsem.release()
self.qmutex.unlock()
def clear(self):
self.qmutex.lock()
self.typeQueue = []
self.setUniq.clear()
self.qsem.acquire(self.qsem.available())
self.qmutex.unlock()
def get(self):
self.qsem.acquire()
self.qmutex.lock()
res = self.typeQueue.pop()
self.setUniq.remove(long(res[2].this))
self.qmutex.unlock()
return res
def isImage(self, ftype):
res = self.regImage.search(ftype)
return res
def run(self):
count = 0
while True:
(parent, index, node) = self.get()
if node.size():
ftype = str(node.dataType())
if parent.imagesthumbnails and self.isImage(ftype):
thumb = ImageThumb()
img = thumb.getImage(ftype, node, index)
if img:
parent.emit(SIGNAL('dataImage'), index, node, img)
class ExtractThread(QThread):
def __init__(self, files, needPrj):
QThread.__init__(self, QThread.currentThread())
self.inFiles = files
self.needPrj = needPrj
self.mutex = QMutex()
self.stopMe = 0
def run(self):
self.mutex.lock()
self.stopMe = 0
self.mutex.unlock()
interrupted = False
for f in self.inFiles:
extractProjection(f, self.needPrj)
self.emit(SIGNAL("fileProcessed()"))
self.mutex.lock()
s = self.stopMe
self.mutex.unlock()
if s == 1:
interrupted = True
break
if not interrupted:
self.emit(SIGNAL("processFinished()"))
else:
self.emit(SIGNAL("processIterrupted()"))
def stop(self):
self.mutex.lock()
self.stopMe = 1
self.mutex.unlock()
QThread.wait(self)
class ExtractThread(QThread):
def __init__(self, files, needPrj):
QThread.__init__(self, QThread.currentThread())
self.inFiles = files
self.needPrj = needPrj
self.mutex = QMutex()
self.stopMe = 0
def run(self):
self.mutex.lock()
self.stopMe = 0
self.mutex.unlock()
interrupted = False
for f in self.inFiles:
extractProjection(f, self.needPrj)
self.emit(SIGNAL("fileProcessed()"))
self.mutex.lock()
s = self.stopMe
self.mutex.unlock()
if s == 1:
interrupted = True
break
if not interrupted:
self.emit(SIGNAL("processFinished()"))
else:
self.emit(SIGNAL("processIterrupted()"))
def stop(self):
self.mutex.lock()
self.stopMe = 1
self.mutex.unlock()
QThread.wait(self)
class SpatialIdxThread( QThread ):
def __init__( self, layers, isFiles ):
QThread.__init__( self, QThread.currentThread() )
self.layers = layers
self.isFiles = isFiles
self.mutex = QMutex()
self.stopMe = 0
self.errors = []
def run( self ):
self.mutex.lock()
self.stopMe = 0
self.mutex.unlock()
interrupted = False
if self.isFiles:
for layer in self.layers:
vl = QgsVectorLayer( layer, "tmp", "ogr" )
provider = vl.dataProvider()
if provider.capabilities() & QgsVectorDataProvider.CreateSpatialIndex:
if not provider.createSpatialIndex():
self.errors.append( layer )
else:
self.errors.append( layer )
self.emit( SIGNAL( "layerProcessed()" ) )
self.mutex.lock()
s = self.stopMe
self.mutex.unlock()
if s == 1:
interrupted = True
break
else:
for layer in self.layers:
vl = ftools_utils.getVectorLayerByName( layer )
provider = vl.dataProvider()
if provider.capabilities() & QgsVectorDataProvider.CreateSpatialIndex:
if not provider.createSpatialIndex():
self.errors.append( layer )
else:
self.errors.append( layer )
self.emit( SIGNAL( "layerProcessed()" ) )
self.mutex.lock()
s = self.stopMe
self.mutex.unlock()
if s == 1:
interrupted = True
break
if not interrupted:
self.emit( SIGNAL( "processFinished( PyQt_PyObject )" ), self.errors )
else:
self.emit( SIGNAL( "processInterrupted()" ) )
def stop( self ):
self.mutex.lock()
self.stopMe = 1
self.mutex.unlock()
QThread.wait( self )
class AnalyzerThread(QThread):
def __init__(self, framework, parent=None):
QThread.__init__(self, parent)
self.framework = framework
self.callback_object = None
self.qtimer = QTimer()
self.qlock = QMutex()
QObject.connect(self, SIGNAL('quit()'), self.handle_quit)
QObject.connect(self, SIGNAL('started()'), self.handle_started)
self.Data = None
self.read_cursor = None
self.cursor = None
def db_attach(self):
self.Data = self.framework.getDB()
self.read_cursor = self.Data.allocate_thread_cursor()
self.cursor = self.Data.allocate_thread_cursor()
def db_detach(self):
self.close_cursor()
self.Data = None
def close_cursor(self):
if self.read_cursor:
self.read_cursor.close()
self.Data.release_thread_cursor(self.read_cursor)
self.read_cursor = None
if self.cursor:
self.cursor.close()
self.Data.release_thread_cursor(self.cursor)
self.cursor = None
def run(self):
QObject.connect(self, SIGNAL('do_runAnalysis()'),
self.handle_runAnalysis, Qt.DirectConnection)
self.exec_()
def runAnalysis(self, callback_object):
self.callback_object = callback_object
QTimer.singleShot(50, self, SIGNAL('do_runAnalysis()'))
def handle_quit(self):
self.framework.debug_log('AnalyzerThread quit...')
self.close_cursor()
self.exit(0)
def handle_started(self):
self.framework.debug_log('AnalyzerThread started...')
self.framework.subscribe_database_events(self.db_attach,
self.db_detach)
def handle_runAnalysis(self):
fullanalysistext = ''
if not self.qlock.tryLock():
self.framework.debug_log('failed to acquire lock for analyzers')
else:
try:
fullanalysistext = self.analyze_content()
except Exception as e:
self.framework.report_exception(e)
finally:
self.qlock.unlock()
if self.callback_object:
self.callback_object.emit(SIGNAL('runAnalysisFinished(QString)'),
str(fullanalysistext))
def result_type_to_string(self, resulttype):
"""Reads type and package information from a result, and returns it as a string"""
resultclass = resulttype.__class__
return "".join((resultclass.__module__, ".", resultclass.__name__))
def analyze_content(self):
""" Perform analysis on the captured content"""
#TODO: NEW DB THREAD TO HOLD RESPONSES, ANOTHER FOR WRITING RESULTS
scopeController = self.framework.getScopeController()
response = self.Data.read_all_responses(self.read_cursor)
response_IDs = []
for row in response:
dbrow = [m or '' for m in row]
Id = dbrow[ResponsesTable.ID]
url = dbrow[ResponsesTable.URL]
if scopeController.isUrlInScope(url, url):
response_IDs.append(Id)
#Instantiate all found analyzers
analyzerobjects = AnalyzerList(self.framework)
analyzerobjects.instantiate_analyzers()
analysisrunid = self.Data.analysis_start(self.cursor)
#TODO - Consider threading from here down
for x in analyzerobjects:
#dbconfig=self.Data.get_config_value(self.read_cursor, 'ANALYSIS', str(x.__class__))
#print "dbconfig=%s"%dbconfig
print("class=%s" % x.__class__)
#x.setConfiguration(dbconfig)
x.preanalysis()
x.initResultsData()
resultinstance = x.getResults()
x.analyzerinstanceid = self.Data.analysis_add_analyzer_instance(
self.cursor, analysisrunid,
str(x.__class__).translate('<>'), x.friendlyname, x.desc,
self.result_type_to_string(resultinstance))
fullanalysistext = StringIO()
for Id in response_IDs:
transaction = self.framework.get_request_response(Id)
for analyzer in analyzerobjects:
try:
analyzer.analyzeTransaction(transaction,
analyzer.getResults())
tempanalysisresults = analyzer.getResults()
#If there were results for this page, add them to the DB
if transaction.Id in tempanalysisresults.pages:
pageresultset = self.Data.analysis_add_resultset(
self.cursor, analyzer.analyzerinstanceid,
transaction.Id, False, transaction.responseUrl,
self.result_type_to_string(
tempanalysisresults.pages[transaction.Id]))
for result in tempanalysisresults.pages[
transaction.Id].results:
self.Data.analysis_add_singleresult(
self.cursor,
pageresultset,
result.severity,
result.certainty,
result.type,
result.desc,
#TODO: Consider db structure to handle data field
str(result.data),
result.span,
self.result_type_to_string(result))
for key, value in list(tempanalysisresults.pages[
transaction.Id].stats.items()):
self.Data.analysis_add_stat(
self.cursor, pageresultset, key, value)
except Exception as e:
# TODO: add real debugging support
self.framework.debug_log('Transaction ID: ' +
str(transaction.Id))
self.framework.report_exception(e)
#Post Analysis
for analyzer in analyzerobjects:
results = analyzer.getResults()
analyzer.postanalysis(results)
for context in list(results.overall.keys()):
overallresultset = self.Data.analysis_add_resultset(
self.cursor, analyzer.analyzerinstanceid, None, True,
context,
self.result_type_to_string(results.overall[context]))
for result in results.overall[context].results:
self.Data.analysis_add_singleresult(
self.cursor,
overallresultset,
result.severity,
result.certainty,
result.type,
result.desc,
#TODO: Consider db structure to handle data field
str(result.data),
result.span,
self.result_type_to_string(result))
#print "WRITING:",self.result_type_to_string(result)
for key, value in list(results.overall[context].stats.items()):
self.Data.analysis_add_stat(self.cursor, overallresultset,
key, value)
self.Data.commit()
#Output results to analysis tab
#for analyzer in analyzerobjects:
#fullanalysistext.write(analyzer.getResults().toHTML())
return ''
class MessageSender:
"""
static fileds:
context: to keep track of msg req<->resp mechanism msg context is attached.
Thanks to this value it is possible to find right response for given request.
There are context ids reserved for particular messages handling:
0: free text which should be displayed in console window
1: digidiag frame data
Message sender cant assign reserved_context id when message is created and sent.
"""
reserved_context = (
0,
1,
)
context = Uint16(
len(reserved_context)) #start with value greater than reserved_context
lock = False
class ID:
"""
Message IDs reperesenting given procedure in EMU BT board
"""
txt_message = 0
write_to_page = 1
rxflush = 2
bootloader = 11
disable_btlrd = 12
reset = 14
run_main_app_btl = 254
@classmethod
def translate_id(cls, m_id):
try:
return [
p[0] for p in MessageSender.ID.__dict__.items()
if p[1] == m_id
][0]
except IndexError:
return None
def __init__(self, tx_interface):
self.mutex = QMutex()
self.__transmit = tx_interface
def __send_m(self, msg, m_id):
"""
Send createad message.
Avoid using recovered context_ids
"""
context = MessageSender.context
MessageSender.context += 1
while MessageSender.context in MessageSender.reserved_context:
MessageSender.context += 1
translated_m_id = MessageSender.ID.translate_id(m_id)
m_logger.debug("Sent message with context: {}, id: {}({})".format(
context, translated_m_id, m_id))
m_logger.debug(msg[11:30])
self.__transmit(msg)
return context
def peek_context(self):
return MessageSender.context + 1
def send(self, m_id, body=b'NULL'):
"""
Polymorphic method for send
"""
return self.__send(m_id, body)
def send_raw_msg(self, body):
"""
Polymorphic method for send
"""
return self.__send(m_id=None, body=body)
def __send(self, m_id=None, body='NULL'):
msg = create_message(
msg_id=m_id, body=body,
context=MessageSender.context) if m_id is not None else body
self.mutex.lock()
context = self.__send_m(msg, m_id)
self.mutex.unlock()
m_logger.debug("message sending unlocked")
return context
class SiteMapThread(QThread):
def __init__(self, framework, treeViewModel, parent=None):
QThread.__init__(self, parent)
self.framework = framework
self.treeViewModel = treeViewModel
self.qtimer = QTimer()
self.qlock = QMutex()
self.fillAll = False
QObject.connect(self, SIGNAL('quit()'), self.quitHandler)
QObject.connect(self, SIGNAL('started()'), self.startedHandler)
self.re_set_cookie = re.compile(r'^Set-Cookie2?:\s*(.+)$', re.I | re.M)
self.re_host_name = re.compile(r'^Host:\s*(.+?)$', re.I | re.M)
self.lastId = 0
self.Data = None
self.cursor = None
def db_attach(self):
self.Data = self.framework.getDB()
self.cursor = self.Data.allocate_thread_cursor()
self.populateSiteMap(True)
def db_detach(self):
self.close_cursor()
self.Data = None
def close_cursor(self):
if self.cursor and self.Data:
self.cursor.close()
self.Data.release_thread_cursor(self.cursor)
self.cursor = None
def run(self):
QObject.connect(self, SIGNAL('updateSiteMap()'), self.doUpdateSiteMap,
Qt.DirectConnection)
self.framework.subscribe_response_data_added(self.doUpdateSiteMap)
self.exec_()
def quitHandler(self):
self.framework.debug_log('SiteMapThread quit...')
self.close_cursor()
self.exit(0)
def startedHandler(self):
self.framework.debug_log('SiteMapThread started...')
self.framework.subscribe_database_events(self.db_attach,
self.db_detach)
def populateSiteMap(self, fillAll):
self.fillAll = fillAll
QTimer.singleShot(50, self, SIGNAL('updateSiteMap()'))
def doUpdateSiteMap(self):
if not self.qlock.tryLock():
return
try:
if self.fillAll:
self.fillAll = False
self.treeViewModel.clearModel()
self.lastId = 0
rows = self.Data.get_sitemap_info(self.cursor, self.lastId)
global_cookie_jar = self.framework.get_global_cookie_jar()
count = 0
for row in rows:
count += 1
if 0 == (count % 100):
self.yieldCurrentThread()
rowItems = [m or '' for m in list(row)]
Id = str(rowItems[0])
try:
self.lastId = int(Id)
except ValueError:
pass
# XXX: review all for bytes usage
if isinstance(rowItems[1], bytes):
url = str(rowItems[1], 'utf-8', 'ignore')
else:
url = str(rowItems[1])
status = str(rowItems[2])
response_headers = str(rowItems[3])
request_headers = str(rowItems[4])
# TODO: make configurable
if status in ('400', '404', '500', '501'):
continue
# TODO:
m = self.re_set_cookie.search(response_headers)
if m:
setCookies = m.group(1)
cookieList = QNetworkCookie.parseCookies(setCookies)
global_cookie_jar.setCookiesFromUrl(
cookieList, QUrl.fromEncoded(url))
parsed = urlparse.urlsplit(url)
hostname = ''
if not parsed.hostname:
m = self.re_host_name.search(request_headers)
if m:
hostname = m.group(1).rstrip()
else:
hostname = parsed.hostname
hostname = hostname.lower()
hostloc = urlparse.urlunsplit(
(parsed.scheme, parsed.netloc, '/', '', ''))
rootNode = self.treeViewModel.findOrAddNode(hostname)
hostLocNode = rootNode.findOrAddNode(self.treeViewModel,
hostloc)
pathval = parsed.path
# add directories
parentNode = hostLocNode
parentNode.setResponseId(None, hostloc)
lastSlash = 0
slash = 0
while True:
slash = pathval.find('/', slash + 1)
if slash < 0:
break
dirname = pathval[lastSlash + 1:slash + 1]
parentNode = parentNode.findOrAddNode(
self.treeViewModel, dirname)
parentNode.setResponseId(
None,
urlparse.urlunsplit((parsed.scheme, parsed.netloc,
pathval[0:slash + 1], '', '')))
lastSlash = slash
# add file element
if lastSlash + 1 < len(pathval):
filename = pathval[lastSlash + 1:]
parentNode = parentNode.findOrAddNode(
self.treeViewModel, filename)
parentNode.setResponseId(
None,
urlparse.urlunsplit(
(parsed.scheme, parsed.netloc, pathval, '', '')))
# add query
if parsed.query:
parentNode = parentNode.findOrAddNode(
self.treeViewModel, '?' + parsed.query)
# store the latest Id
# TODO: should determine best candidate to display
parentNode.setResponseId(Id, url)
finally:
self.qlock.unlock()
class SplitThread(QThread):
def __init__(self, layer, splitField, encoding, outDir):
QThread.__init__(self, QThread.currentThread())
self.layer = layer
self.field = splitField
self.encoding = encoding
self.outDir = outDir
self.mutex = QMutex()
self.stopMe = 0
self.errors = []
def run(self):
self.mutex.lock()
self.stopMe = 0
self.mutex.unlock()
interrupted = False
outPath = self.outDir
if outPath.find("\\") != -1:
outPath.replace("\\", "/")
if not outPath.endswith("/"):
outPath = outPath + "/"
provider = self.layer.dataProvider()
index = provider.fieldNameIndex(self.field)
unique = ftools_utils.getUniqueValues(provider, int(index))
baseName = unicode(outPath + self.layer.name() + "_" + self.field +
"_")
fieldList = ftools_utils.getFieldList(self.layer)
sRs = provider.crs()
geom = self.layer.wkbType()
inFeat = QgsFeature()
self.emit(SIGNAL("rangeCalculated(PyQt_PyObject)"), len(unique))
for i in unique:
check = QFile(baseName + "_" + unicode(i).strip() + ".shp")
fName = check.fileName()
if check.exists():
if not QgsVectorFileWriter.deleteShapeFile(fName):
self.errors.append(fName)
continue
writer = QgsVectorFileWriter(fName, self.encoding, fieldList, geom,
sRs)
fit = provider.getFeatures()
while fit.nextFeature(inFeat):
atMap = inFeat.attributes()
if atMap[index] == i:
writer.addFeature(inFeat)
del writer
self.emit(SIGNAL("valueProcessed()"))
self.mutex.lock()
s = self.stopMe
self.mutex.unlock()
if s == 1:
interrupted = True
break
if not interrupted:
self.emit(SIGNAL("processFinished( PyQt_PyObject )"), self.errors)
else:
self.emit(SIGNAL("processInterrupted()"))
def stop(self):
self.mutex.lock()
self.stopMe = 1
self.mutex.unlock()
QThread.wait(self)
class Walker(QThread):
finished = Signal(bool, int)
indexed = Signal(QString, int)
COMMON_WORDS_THRESHOLD = 250
MIN_WORD_LEN = 3
MAX_WORD_LEN = 25
INVALID_FIRST_OR_LAST = frozenset("0123456789_")
STRIPHTML_RE = re.compile(r"<[^>]*?>", re.IGNORECASE|re.MULTILINE)
ENTITY_RE = re.compile(r"&(\w+?);|&#(\d+?);")
SPLIT_RE = re.compile(r"\W+", re.IGNORECASE|re.MULTILINE)
def __init__(self, index, lock, files, filenamesForWords,
commonWords, parent=None):
super(Walker, self).__init__(parent)
self.index = index
self.lock = lock
self.files = files
self.filenamesForWords = filenamesForWords
self.commonWords = commonWords
self.stopped = False
self.mutex = QMutex()
self.completed = False
def stop(self):
try:
self.mutex.lock()
self.stopped = True
finally:
self.mutex.unlock()
def isStopped(self):
try:
self.mutex.lock()
return self.stopped
finally:
self.mutex.unlock()
def run(self):
self.processFiles()
self.stop()
self.finished.emit(self.completed, self.index)
def processFiles(self):
for fname in self.files:
if self.isStopped():
return
self.processOneFile(fname)
self.completed = True
def processOneFile(self, fname):
words = set()
text = self.readFile(fname)
if text is None or self.isStopped():
return
text = self.STRIPHTML_RE.sub("", text)
text = self.ENTITY_RE.sub(unichrFromEntity, text)
text = text.lower()
for word in self.SPLIT_RE.split(text):
if (self.MIN_WORD_LEN <= len(word) <=
self.MAX_WORD_LEN and
word[0] not in self.INVALID_FIRST_OR_LAST and
word[-1] not in self.INVALID_FIRST_OR_LAST):
try:
self.lock.lockForRead()
new = word not in self.commonWords
finally:
self.lock.unlock()
if new:
words.add(word)
if self.isStopped():
return
for word in words:
try:
self.lock.lockForWrite()
files = self.filenamesForWords[word]
if len(files) > self.COMMON_WORDS_THRESHOLD:
del self.filenamesForWords[word]
self.commonWords.add(word)
else:
files.add(unicode(fname))
finally:
self.lock.unlock()
self.indexed.emit(fname, self.index)
def readFile(self, fname):
try:
with codecs.open(fname, "r", "UTF8", "ignore") as file:
return file.read()
except (IOError, OSError), err:
sys.stderr.write("Error: {0}\n".format(err))
class WorkerThread(QThread):
''' Schedule things to do
Wait all time or launch/wake other threads to recompute max amount of node to
display with or without zoom, refresh/redraw paint area, etc. .
'''
def __init__(self, parent = None):
super(WorkerThread, self).__init__(parent)
self.mutex = QMutex()
self.condition = QWaitCondition()
self.restart = False
self.abort = False
self.timeline = parent
self.countThread = parent.countThread
self.populateThread = parent.populateThread
self.maxOccThread = parent.maxOccThread
def __del__(self):
self.mutex.lock()
self.abort = True
self.condition.wakeOne()
self.mutex.unlock()
self.wait()
def render(self):
locker = QMutexLocker(self.mutex)
if not self.isRunning() and not self.countThread.isRunning() and not self.populateThread.isRunning() and not self.maxOccThread.isRunning():
self.start(QThread.LowPriority)
else:
self.restart = True
self.condition.wakeOne()
def run(self):
while True:
self.mutex.lock()
# Fetch value from timeline
nodeCount = self.timeline.nodeCount
dataListsCreated = self.timeline.dataListsCreated
if dataListsCreated:
self.timeline.findMaxValue()
xHop = self.timeline.xHop
maxOcc = self.timeline.maxOcc
self.mutex.unlock()
if not nodeCount and not self.restart and not self.countThread.isRunning():
self.countThread.start()
elif nodeCount and not dataListsCreated and not self.restart and not self.populateThread.isRunning():
self.populateThread.start()
elif nodeCount and dataListsCreated and xHop and not maxOcc and not self.restart and not self.maxOccThread.isRunning():
self.maxOccThread.start()
elif nodeCount and dataListsCreated and xHop and maxOcc and not self.restart and not self.maxOccThread.isRunning():
self.emit(SIGNAL('refresh'), True)
if self.abort:
return
self.mutex.lock()
if not self.restart:
self.condition.wait(self.mutex)
self.restart = False
self.mutex.unlock()
class _OperationsSpool(QObject):
"""
A spool of operations of a certain type. Can use many threads.
"""
def __init__(self, parent):
"""
Pprepare spools and threads
Must be overriden to set :
- type : one of "content", "action", "authentication", "network"
- max_threads : max number of threads (default 1)
"""
super(_OperationsSpool, self).__init__(parent)
self.max_threads = 1
self.threads = [] # list of threads running operations
self.mutex = QMutex()
self.spool = deque() # list of remaining operations
self.running = [] # list of running operations
self.done = [] # list of finished operations
self.errors = [] # list of failed operations
self.waiting = deque() # list of operations not yet added to the spool
def count_all(self):
"""
Return a hash with count of operations for each list in this spool
"""
result = {}
for type in ('spool', 'running', 'done', 'errors', 'waiting', ):
result[type] = len(getattr(self, type))
return result
def count_running(self):
"""
Count "alive" operations : spool + running + waiting
"""
by_type = self.count_all()
return by_type['spool'] + by_type['running'] + by_type['waiting']
def add(self, operation):
"""
Add an operation to the waiting spool
"""
self.waiting.append(operation)
self.check_waiting_list()
def check_waiting_list(self):
"""
This method will check conditions about the operation
before to put it in the spool queue (max_same, kill_same, opposites...)
Thi method is threadsafe (mutex.lock is here to ensure that only one
waiting operation is checked at a time)
"""
self.mutex.lock()
try:
# get operation
operation = self.waiting.popleft()
# check operations specificities
can_add, to_remove = self._test_opposite(operation)
for op in to_remove:
op.status = 'rejected'
self.spool.remove(op)
if can_add:
can_add, to_remove = self._test_same(operation)
for op in to_remove:
op.status = 'rejected'
self.spool.remove(op)
# finally add the operation to the spool only if we can add
# the operation
if can_add:
self.spool.append(operation)
self.ping_threads()
else:
operation.status = 'rejected'
finally:
self.mutex.unlock()
def _test_opposite(self, operation):
"""
If the operation has an opposite name, this method check opposites in
the spool. If an opposite is found and the kill_opposite flag is True,
then the opposite operation is removed from the spool, but if the flag
is False, then the current operation is cancelled.
"""
to_remove = []
can_add = True
if operation.opposite:
for op in self.spool:
if op.is_opposite_of(operation):
if operation.kill_opposite:
to_remove.append(op)
else:
can_add = False
return can_add, to_remove
def _test_same(self, operation):
"""
If the operation has a positive max_same, this method will check for
same operations in the spool and if at least one, return False (and an
empty list of operations to remove).
Else if we have a kill_same, we check for same operations in the spool
and return True (operation still valid) and the list of removals
"""
to_remove = []
can_add = True
# check max_same
if operation.max_same:
count_same = 0
for op in self.spool:
if op.is_same_as(operation):
count_same += 1
if count_same >= operation.max_same:
can_add = False
break
# check max_same
elif operation.kill_same:
to_remove = []
for op in self.spool:
if op.is_same_as(operation):
to_remove.append(op)
return can_add, to_remove
def next(self):
"""
Return the next operation to run, remove it from the spool queue,
and move it to the running one.
Will be called by the next thread which has finished working on its
previous operation. This method is threadsafe (mutex.lock is here to
ensure only one thread at a time can claim a operation)
"""
self.mutex.lock()
try:
operation = self.spool.popleft()
self.running.append(operation)
return operation
except:
return None
finally:
self.mutex.unlock()
def ask_for_network(self):
"""
We ask for network
"""
self.parent().ask_for_network()
def ask_for_authentication(self):
"""
We ask for a new authentication
"""
self.parent().ask_for_authentication()
def is_ready(self):
"""
Check if the spool is ready to work (we are both connected and
authenticated)
"""
return self.parent().is_ready()
def ping_threads(self):
"""
Ping all threads to let them check if new operations are available
in the spool. A new thread is created if needed (and possible)
"""
one_available = False
for thread in self.threads:
if not thread.isRunning():
one_available = True
thread.ping()
if not one_available and len(self.threads) < self.max_threads:
thread = OperationsThread(spool=self)
QObject.connect(thread, SIGNALS['operation_started'], self.operation_started, Qt.QueuedConnection)
QObject.connect(thread, SIGNALS['operation_ended'], self.operation_ended, Qt.QueuedConnection)
self.threads.append(thread)
thread.ping()
def operation_started(self, operation_id):
"""
Forward the "operation_started" from the spool to the operations manager
"""
log("[op-start] %s\n" % Operation.get_by_id(operation_id))
operation = Operation.get_by_id(operation_id)
self.parent().emit(SIGNALS['%s_started' % operation.name], operation_id)
self.parent().emit(SIGNALS['operation_started'], operation_id)
def operation_ended(self, operation_id):
"""
Forward the "operation_ended" from the spool to the operations manager
"""
# get the corresponding operation
operation = Operation.get_by_id(operation_id)
# update the spool
self.running.remove(operation)
if operation.status == 'done':
self.done.append(operation)
else:
self.errors.append(operation)
# alert that an operation has finished
log("[op-end] %s\n" % operation)
self.parent().emit(SIGNALS['operation_ended'], operation_id)
# then alert for this specific operation
status = 'done'
operation = Operation.get_by_id(operation_id)
if operation.status != 'done':
status = 'error'
self.parent().emit(SIGNALS['%s_%s' % (operation.name, status)], operation_id)
class PlottingThread(QThread):
def __init__(self, parent):
QThread.__init__(self)
self.result = None
self.parent = parent
self._stopped = False
self.mutex = QMutex()
self.filePrefix = None
self.fileFormat = None
self.wallColoring = None
self.cellColoring = None
self.pointColoring = None
self.extraDrawing = []
self.pointSize = None
self.pointLineColor = None
self.pointLineThickness = None
self.ellipsisDraw = None
self.overSampling = None
self.wallThickness = None
self.bgColor = None
self.loading = False
self._crop = QRect(0,0,1,1)
self._pix = None
self._end_image_plot = False
self._loading_arguments = {}
self.retryObject = None
def end_step(self):
return len(self.result)+1
def stop(self, value = True):
self.mutex.lock()
self._stopped = value
self.mutex.unlock()
def stopped(self):
self.mutex.lock()
val = self._stopped
self.mutex.unlock()
return val
def nextImage(self):
QCoreApplication.postEvent(self.parent, NextImageEvent())
def abort(self, reason, **others):
e = AbortPlottingEvent(reason)
if others:
e.others = others
QCoreApplication.postEvent(self.parent, e)
def finished(self):
if self.loading:
QCoreApplication.postEvent(self.parent, FinishLoadingEvent())
self.loading = False
else:
QCoreApplication.postEvent(self.parent, FinishPlottingEvent())
def image_ready(self):
QCoreApplication.postEvent(self.parent, ImageReadyPlottingEvent())
def update_nb_images(self, nb):
QCoreApplication.postEvent(self.parent, UpdateNbImageEvent(nb))
@property
def crop_left(self):
return self._crop.left()
@crop_left.setter
def crop_left(self, value):
self._crop.moveLeft(int(value))
@property
def crop_top(self):
return self._crop.top()
@crop_top.setter
def crop_top(self, value):
self._crop.moveTop(int(value))
@property
def crop_width(self):
return self._crop.width()
@crop_width.setter
def crop_width(self, value):
self._crop.setWidth(int(value))
@property
def crop_height(self):
return self._crop.height()
@crop_height.setter
def crop_height(self, value):
self._crop.setHeight(int(value))
def reset_crop(self):
self._crop = QRect(QPoint(0,0), self.img_size)
@property
def crop(self):
return QRect(self._crop)
@crop.deleter
def crop(self):
self.reset_crop()
@property
def end_image_plot(self):
'''
If true, plot the growth data on the end image rather than the start image of the growth calculation.
'''
return self._end_image_plot
@end_image_plot.setter
def end_image_plot(self, value):
self._end_image_plot = bool(value)
@property
def pix(self):
'''Thread-safe image storage.'''
self.mutex.lock()
pix = self._pix
self.mutex.unlock()
return pix
@pix.setter
def pix(self, value):
self.mutex.lock()
self._pix = value
self.mutex.unlock()
def render_valid(self):
if self.result is None:
log_debug("result is None")
return False
if self.parent is None:
log_debug("parent is None")
return False
if self.ellipsisDraw is None:
log_debug("ellipsisDraw is None")
return False
if self.cellColoring is None:
log_debug("cellColoring is None")
return False
if self.wallColoring is None:
log_debug("wallColoring is None")
return False
if self.pointColoring is None:
log_debug("pointColoring is None")
return False
if self.pointSize is None:
log_debug("pointSize is None")
return False
if self.pointLineThickness is None:
log_debug("pointSize is None")
return False
if self.pointLineColor is None:
log_debug("pointSize is None")
return False
if self.wallThickness is None:
log_debug("wallThickness is None")
return False
if self.overSampling is None:
log_debug("overSampling is None")
return False
if self.bgColor is None:
log_debug("bgColor is None")
return False
return True
def valid(self):
if self.filePrefix is None:
log_debug("filePrefix is None")
return False
if not self.filePrefix:
log_debug("filePrefix is Empty")
return False
if self.fileFormat is None:
log_debug("fileFormat is None")
return False
return self.render_valid()
def drawImage(self, imageid):
cache = image_cache.cache
cellColoring = self.cellColoring
wallColoring = self.wallColoring
pointColoring = self.pointColoring
ellipsisDraw = self.ellipsisDraw
overSampling = self.overSampling
extraDrawing = self.extraDrawing
bgColor = self.bgColor.rgb()
result = self.result
if self.result_type == "Data":
data = result
img_name = result.images_name[imageid]
else:
data = result.data
img_name = result.images[imageid]
#scale = data.images_scale[img_name]
min_scale = data.minScale()
img = cache.image(data.image_path(img_name))
img_data = data[img_name]
size = self._crop.size()
pix = QImage(size*overSampling, QImage.Format_ARGB32)
pix.fill(bgColor)
painter = QPainter()
if not painter.begin(pix):
self.abort("Cannot create painter on QImage")
return None, None, None
painter.setRenderHints(QPainter.SmoothPixmapTransform, True)
painter.setRenderHints(QPainter.Antialiasing, True)
if overSampling > 1:
painter.scale(overSampling, overSampling)
painter.translate(-self._crop.topLeft())
painter.save()
painter.translate(self.translate)
log_debug("Translating: %gx%g" % (self.translate.x(), self.translate.y()) )
painter.scale(1/min_scale, 1/min_scale)
painter.save()
matrix = img_data.matrix()
painter.setWorldTransform(matrix, True)
painter.drawImage(QPoint(0,0), img)
painter.restore()
#pt_matrix = QTransform()
#pt_matrix.scale(1/min_scale, 1/min_scale)
#painter.setTransform(pt_matrix, True)
cellColoring.startImage(painter, imageid)
wallColoring.startImage(painter, imageid)
for ed in extraDrawing:
ed.startImage(painter, imageid)
if self.result_type == "Growth":
cells = result.cells[imageid]
walls = result.walls[imageid]
else:
cells = img_data.cells
walls = set()
for cid in img_data.cells:
pts = [ pt for pt in data.cells[cid] if pt in img_data ]
if len(pts) > 1:
for i in range(len(pts)):
walls.add(data.wallId(pts[i-1], pts[i]))
# Now, draw the cells and the ellipsis
for cid in cells:
painter.setPen(Qt.NoPen)
color = cellColoring(imageid, cid)
painter.setBrush(color)
pts = data.cellAtTime(cid, img_data.index)
if pts:
pts.append(pts[0])
ppts = []
for p1,p2 in zip(pts[:-1], pts[1:]):
ppts.append(img_data[p1])
ppts.extend(img_data.walls[p1,p2])
ppts.append(ppts[0])
poly = QPolygonF(ppts)
painter.drawPolygon(poly)
# And draw the walls
wallThickness = self.wallThickness*min_scale
for wid in walls:
color = wallColoring(imageid, wid)
if color.alpha() > 0:
pen = QPen(color)
pen.setWidthF(wallThickness)
painter.setPen(pen)
pts = [img_data[wid[0]]] + img_data.walls[wid[0], wid[1]] + [img_data[wid[1]]]
#painter.drawLine(img_data[wid[0]], img_data[wid[1]])
painter.drawPolyline(*pts)
# Then, draw the points
pointSize = self.pointSize*min_scale
pointLineColor = self.pointLineColor
pointLineThickness = self.pointLineThickness*min_scale
log_debug("pointSize = %g" % pointSize)
for pid in img_data:
color = pointColoring(imageid, pid)
if color.alpha() > 0:
pen = QPen(pointLineColor)
pen.setWidthF(pointLineThickness)
brush = QBrush(color)
painter.setPen(pen)
painter.setBrush(brush)
pos = img_data[pid]
rect = QRectF(pos.x()-pointSize, pos.y()-pointSize, 2*pointSize, 2*pointSize)
painter.drawEllipse(rect)
if ellipsisDraw.plot:
for cid in cells:
pts = data.cellAtTime(cid, img_data.index)
if pts:
pts.append(pts[0])
ppts = []
for p1,p2 in zip(pts[:-1], pts[1:]):
ppts.append(img_data[p1])
ppts.extend(img_data.walls[p1,p2])
ppts.append(ppts[0])
#poly = QPolygonF(ppts)
#painter.drawPolygon(poly)
ellipsisDraw(painter, imageid, cid, ppts, min_scale)
# At last, draw the extra data
for ed in extraDrawing:
ed(painter, imageid)
tr = painter.worldTransform()
painter.restore()
pic_w = wallColoring.finalizeImage(painter, imageid, tr, self.crop)
pic_c = cellColoring.finalizeImage(painter, imageid, tr, self.crop)
for ed in extraDrawing:
ed.finalizeImage(painter, imageid, tr, self.crop)
painter.end()
return pix, pic_w, pic_c
def start(self):
if self.isRunning():
assert not self.rendering_all, "Cannot run twice the rendering of all images with the same object."
return
if parameters.instance.use_thread:
log_debug("Starting rendering thread.")
QThread.start(self)
return False
else:
self.run()
return True
def render_all(self):
self.rendering_all = True
return self.start()
def render_single(self, img_id, retry=False):
if retry:
while self.isRunning():
self.wait(10000)
elif self.isRunning():
return
self.rendering_all = False
self.current_image = img_id
return self.start()
def load(self, filename):
self.loading = True
self.result = filename
return self.start()
def run(self):
if self.loading:
self.run_loader()
elif self.rendering_all:
self.run_full()
else:
self.run_single()
def run_single(self):
img = self.current_image
self.cellColoring.init()
self.wallColoring.init()
self.pointColoring.init()
log_debug("Rendering image %d" % img)
self.pix, self.pic_w, self.pic_c = self.drawImage(img)
if self.pic_w is not None:
log_debug("Has wall image")
if self.pic_c is not None:
log_debug("Has cell image")
if self.pix is not None:
log_debug("Pix correctly rendered")
log_debug("Rendered image %d = %s" % (img, self.pix))
self.image_ready()
def reload(self):
if self.retryObject is None:
return
self._loading_arguments.update(self.retryObject.method_args)
self.load(self.retryObject.filename)
def run_loader(self):
filename = self.result
try:
self.retryObject = None
# First, prepare the data by getting the images and computing how big they
# should be
f = open(filename)
first_line = f.readline()
f.close()
if first_line.startswith("TRKR_VERSION"):
result = Result(None)
result.load(self.result, **self._loading_arguments)
result_type = "Growth"
else:
result = TrackingData()
result.load(self.result, **self._loading_arguments)
result_type = "Data"
self.result = result
self.result_type = result_type
if result_type == "Data":
data = result
images = data.images_name
if data.cells:
self.has_cells = True
self.has_walls = True
else:
self.has_cells = False
self.has_walls = False
self.has_points = bool(data.cell_points)
else:
data = result.data
images = result.images
self.has_cells = False
self.has_walls = False
self.has_points = False
self.images = images
cache = image_cache.cache
self.update_nb_images(len(result))
bbox = QRectF()
ms = data.minScale()
for i in range(len(result)):
img_name = images[i]
img_data = data[img_name]
img = cache.image(data.image_path(img_name))
matrix = QTransform()
matrix = img_data.matrix()
sc = QTransform()
sc.scale(1.0/ms, 1.0/ms)
matrix *= sc
r = QRectF(img.rect())
rbox = matrix.map(QPolygonF(r)).boundingRect()
bbox |= rbox
log_debug("Image '%s':\n\tSize = %gx%g\n\tTransformed = %gx%g %+g %+g\n\tGlobal bbox = %gx%g %+g %+g\n" %
(img_name, r.width(), r.height(), rbox.width(), rbox.height(), rbox.left(), rbox.top(),
bbox.width(), bbox.height(), bbox.left(), bbox.top()))
log_debug("Matrix:\n%g\t%g\t%g\n%g\t%g\t%g\n" %
(matrix.m11(), matrix.m12(), matrix.dx(), matrix.m21(), matrix.m22(), matrix.dy()))
if result_type == "Growth":
if result.cells[i]:
self.has_cells = True
if result.walls[i]:
self.has_walls = True
self.has_points = bool(result.data.cell_points)
self.nextImage()
translate = bbox.topLeft()
translate *= -1
self.translate = translate
size = bbox.size().toSize()
self.img_size = size
self._crop = QRect(QPoint(0,0), size)
self.finished()
self._loading_arguments = {} # All done, we don't need that anymore
except RetryTrackingDataException as ex:
ex.filename = filename
self.retryObject = ex
self.finished()
return
except Exception as ex:
_, _, exceptionTraceback = sys.exc_info()
self.abort(ex, traceback=exceptionTraceback)
raise
def run_full(self):
if not self.valid():
self.abort("Object was not correctly initialized")
return
self.stop(False)
painter = None
try:
result = self.result
self.update_nb_images(len(result))
# if self.result_type == "Data":
# data = result
# images = result.images_name
# else:
# data = result.data
# images = result.images
# cache = image_cache.cache
cellColoring = self.cellColoring
wallColoring = self.wallColoring
pointColoring = self.pointColoring
file_format = self.fileFormat
file_pattern = "%s%%0%dd.%s" % (self.filePrefix, len(str(len(result))), file_format)
wall_file_pattern = "%s%%0%dd_wall.%s" % (self.filePrefix, len(str(len(result))), file_format)
cell_file_pattern = "%s%%0%dd_cell.%s" % (self.filePrefix, len(str(len(result))), file_format)
cellColoring.init()
wallColoring.init()
pointColoring.init()
self.nextImage()
for i in range(len(result)):
if self.stopped():
self.abort("User interruption")
return
pix, pic_w, pic_c = self.drawImage(i)
pix.save(file_pattern % (i+1), file_format)
if pic_w is not None:
self.saveExtra(pic_w, wall_file_pattern % (i+1), file_format)
if pic_c is not None:
self.saveExtra(pic_c, cell_file_pattern % (i+1), file_format)
self.nextImage()
self.finished()
except Exception as ex:
if painter is not None:
painter.end()
_, _, exceptionTraceback = sys.exc_info()
self.abort(ex, traceback=exceptionTraceback)
raise
def saveExtra(self, picture, file_name, file_format):
rect = picture.boundingRect()
pix = QImage(rect.size(), QImage.Format_ARGB32)
pix.fill(QColor(0, 0, 0, 0).rgba())
paint = QPainter()
paint.begin(pix)
paint.drawPicture(rect.topLeft()*-1, picture)
paint.end()
pix.save(file_name, file_format)
class Rexarm():
"""!
@brief This class describes a rexarm.
The configuration of the rexarm is loaded from a .csv file as described in the Rexarm.initialize.
"""
# Class constants
# Convert config file type strings to string to type functions
STR_TO_TYPE = {
'bool': lambda s: s.lower() in s.lower() in ['true', '1', 't', 'y', 'yes'],
'str': str,
'float': float,
'int': int
}
def __init__(self):
"""!
@brief Constructs a new instance.
Starts the rexarm run thread but does not initialise the Joints. Call Rexarm.initialize to initialise the
Joints.
"""
# Thread locks
self._cmd_mutex = QMutex()
self._serial_mutex = QMutex()
# Serial bus
self.dxlbus = None
# Gripper
self.gripper = None
self.gripper_state = True
# State
self.estop = False
self.initialized = False
# Update rexarm every update_period seconds
self.update_period = 0.01
# Cmds
self.new_speed_cmds = False
self.speed_cmds = None
self.new_torque_limit_cmds = False
self.torque_limit_cmds = None
self.new_position_cmds = False
self.position_cmds = None
# Feedback
self.position_fb = None
self.speed_fb = None
self.load_fb = None
self.temp_fb = None
# Joints
self._joints = []
self.open_angle = 0
self.close_angle = -60
# Data collection flag
self.collect_flag = False
# Start run thread
self.run_thread = RexarmThread(self)
self.run_thread.start()
def initialize(self, config_file=script_path+'/config/rexarm_config.csv'):
"""!
@brief Initializes the rexarm from given configuration file.
Initializes the Joints and serial port
@param config_file The configuration file see Rexarm._config_joints function for more details on the
config file.
@return True is succes False otherwise
"""
self.initialized = False
# Wait for other threads to finish with the rexarm instead of locking every single call
time.sleep(1)
# Get a serial port interface
self._serial_mutex.lock()
if self.dxlbus is not None:
self.dxlbus.close()
self.dxlbus = DXL_BUS(DEVICENAME, BAUDRATE)
self.port_num = self.dxlbus.port()
self._serial_mutex.unlock()
""" Read in rexarm configs from config file """
if not self._config_joints(config_file):
return self.initialized
""" Commanded Values """
self.num_joints = len(self._joints)
self.position = [0.0] * self.num_joints # degrees
self.speed = [1.0] * self.num_joints # 0 to 1
self.max_torque = [1.0] * self.num_joints # 0 to 1
""" Feedback Values """
self.position_fb = [0.0] * self.num_joints # degrees
self.speed_fb = [0.0] * self.num_joints # 0 to 1
self.load_fb = [0.0] * self.num_joints # -1 to 1
self.temp_fb = [0.0] * self.num_joints # Celsius
self.move_fb = [0] * self.num_joints
""" Gripper Vaules """
self.gripper = None
for joint in self._joints:
if joint.is_gripper:
self.gripper = joint
if self.gripper is not None:
self.gripper.torque_limit = 1.0
self.gripper.speed = 0.8
# Reset estop and initialized
self.estop = False
self.initialized = True
return self.initialized
def _config_joints(self, config_file):
"""!
@brief Configure the Joints and initialize them
TODO: Find the physical properties of the Rexarm (such as angle limits and DH parameters) and write a config
file for the Rexarm. The config file should be formated as a csv file with the following properties:
* The first row lists the exact names of the member variables of the Joint class
* The second row list the types of the associated member variables, Rexarm.STR_TO_TYPE dict defines the
avaliable types
* Each following row defines thoes member variables for a joint in the rexarm
* There should be no spaces after a comma
You may add any new member variables as you wish to the config file in any order. By default they will be
added as a readonly property of the Joint class. If special behaviors are desired for a new member variable
then they must be added to the Joint.__init__ parameter list and handled in that function.
@param config_file The configuration file
@return True if all joints initialize correctly, False if any joint fails
"""
self._joints = []
with open(config_file) as configs:
reader = csv.reader(configs)
title_row = next(reader)
type_strs = next(reader)
types = [Rexarm.STR_TO_TYPE[s] for s in type_strs]
for row in reader:
values = {name: dtype(value) for name, value, dtype in zip(title_row, row, types)}
joint = Joint(self._serial_mutex, self.port_num, **values)
self._joints.append(joint)
if not joint.initialize():
return False
return True
def _ensure_initialized(func):
"""!
@brief Decorator to skip the function if the rexarm is not initialized.
@param func The function to wrap
@return The wraped function
"""
def func_out(self, *args, **kwargs):
if self.initialized:
return func(self, *args, **kwargs)
else:
print('WARNING: Trying to use the Rexarm before initialized')
return func_out
@_ensure_initialized
def open_gripper(self):
"""!
@brief TODO: Tell the gripper to open.
"""
new_position = self.position_fb
new_position[5] = self.open_angle
self.set_positions(new_position)
@_ensure_initialized
def open_gripper_blocking(self):
"""!
@brief TODO: Open a gripper and block until it is.
"""
pass
def is_gripper_open(self):
"""!
@brief TODO: Determines if gripper open.
@return True if gripper open, False otherwise.
"""
if abs(self.position_fb[5] - self.open_angle) < 0.2:
return True
else:
return False
@_ensure_initialized
def close_gripper(self):
"""!
@brief TODO Closes a gripper.
"""
new_position = self.position_fb
new_position[5] = self.close_angle
self.set_positions(new_position)
@_ensure_initialized
def close_gripper_blocking(self):
"""!
@brief TODO Closes a gripper and block until it is.
"""
pass
def is_gripper_close(self):
"""!
@brief TODO Determines if gripper close.
@return True if gripper close, False otherwise.
"""
pass
def toggle_gripper(self):
if self.is_gripper_open():
self.open_gripper()
else:
self.close_gripper()
@_ensure_initialized
def toggle_gripper(self):
"""!
@brief TODO Toggle the gripper between open and close
"""
pass
@_ensure_initialized
def toggle_gripper_blocking(self):
"""!
@brief TODO Toggle the gripper between open and close and block until done
"""
pass
def set_positions(self, joint_angles):
"""!
@brief Sets the positions.
@param joint_angles The joint angles
"""
self._cmd_mutex.lock()
self.position_cmds = joint_angles
self.new_position_cmds = True
self._cmd_mutex.unlock()
def set_speeds_normalized_all(self, speed):
"""!
@brief Sets the speeds normalized all.
@param speed The speed
"""
self._cmd_mutex.lock()
self.speed_cmds = [speed] * len(self._joints)
self.new_speed_cmds = True
self._cmd_mutex.unlock()
def set_speeds_normalized(self, speeds):
"""!
@brief Sets the speeds normalized.
@param speeds The speeds
"""
self._cmd_mutex.lock()
self.speed_cmds = speeds
self.new_speed_cmds = True
self._cmd_mutex.unlock()
def set_speeds(self, speeds):
"""!
@brief Sets the speeds.
@param speeds The speeds
"""
self._cmd_mutex.lock()
self.speed_cmds = [None] * len(self._joints)
for i in range(len(self._joints)):
cmd = abs(speeds[i] / self._joints[i].max_speed)
if (cmd < 3.0 / 1023.0):
cmd = 3.0 / 1023.0
self.speed_cmds[i] = cmd
self.new_speed_cmds = True
self._cmd_mutex.unlock()
def set_torque_limits(self, torques):
"""!
@brief Sets the torque limits.
Can't be used when estoped.
@param torques The torques
"""
if self.estop:
print('WARNING: Torques are zero when estoped')
return
self._cmd_mutex.lock()
self.torque_limit_cmds = torques
self.new_torque_limit_cmds = True
self._cmd_mutex.unlock()
@_ensure_initialized
def _send_commands(self):
"""!
@brief Sends commands.
Send the most recent commands to the joints or set torques to zero if estoped.
"""
# Copy current commands so other threads need not wait for the serial communications
self._cmd_mutex.lock()
torque_limit_cmds = None
speed_cmds = None
position_cmds = None
if self.new_torque_limit_cmds:
torque_limit_cmds = self.torque_limit_cmds.copy()
if self.new_speed_cmds:
speed_cmds = self.speed_cmds.copy()
if self.new_position_cmds:
position_cmds = self.position_cmds.copy()
self.new_torque_limit_cmds = False
self.new_speed_cmds = False
self.new_position_cmds = False
self._cmd_mutex.unlock()
# Set torques to zero when estoped
if self.estop:
torque_limit_cmds = [0] * len(self._joints)
for joint in self._joints:
joint.torque_limit = 0
return
# Write the commands
for i, joint in enumerate(self._joints):
if position_cmds is not None:
joint.position = position_cmds[i]
# Gripper torque and speed handled seperatly
if joint.is_gripper:
continue
if torque_limit_cmds is not None:
joint.torque_limit = torque_limit_cmds[i]
if speed_cmds is not None:
joint.speed = speed_cmds[i]
def disable_torque(self):
"""!
@brief Disables the torque and estops.
"""
self.estop = True
def get_positions(self):
"""!
@brief Gets the positions.
@return The positions.
"""
return self.position_fb
def get_speeds(self):
"""!
@brief Gets the speeds.
@return The speeds.
"""
return self.speed_fb
def get_loads(self):
"""!
@brief Gets the loads.
@return The loads.
"""
return self.load_fb
def get_temps(self):
"""!
@brief Gets the temps.
@return The temps.
"""
return self.temp_fb
def get_moving_status(self):
"""!
@brief Gets the moving status.
@return The moving status.
"""
return self.move_fb
def get_errors(self):
"""!
@brief Gets the serial errors.
@return The serial errors.
"""
return [joint.serial_errors for joint in self._joints]
@_ensure_initialized
def _get_feedback(self):
"""!
@brief Updates all the feedback variables of the joints.
"""
self.position_fb = [joint.position_fb for joint in self._joints]
self.speed_fb = [joint.speed_fb for joint in self._joints]
self.load_fb = [joint.load_fb for joint in self._joints]
self.temp_fb = [joint.temp_fb for joint in self._joints]
self.move_fb = [joint.move_fb for joint in self._joints]
@_ensure_initialized
def get_wrist_pose(self):
"""!
@brief TODO Get the wrist pose.
@return The wrist pose as [x, y, z, phi].
"""
return [0, 0, 0, 0]
def get_dh_parameters(self):
"""!
@brief Gets the dh parameters.
@return The dh parameters.
"""
return [[joint.dh_a, joint.dh_alpha, joint.dh_d, joint.dh_theta] for joint in self._joints]
@_ensure_initialized
def run(self):
"""!
@brief Update the servos feedback and commands
Run in a seperate thread see RexarmThread.
"""
# Update feedback
self._get_feedback()
if self.collect_flag:
with open("rexarm_pos_data.csv", 'a') as inFile:
inFile.write(str(time.time()) + ",")
for pos in self.position_fb:
inFile.write(str(pos) + ",")
inFile.write("\n")
# Send new commands
self._send_commands()
class PointsInPolygonThread(QThread):
def __init__(self, inPoly, inPoints, fieldName, outPath, encoding,
attributeList, statisticSelector):
QThread.__init__(self, QThread.currentThread())
self.mutex = QMutex()
self.stopMe = 0
self.interrupted = False
self.layerPoly = inPoly
self.layerPoints = inPoints
self.fieldName = fieldName
self.outPath = outPath
self.encoding = encoding
self.attributeList = attributeList
self.statistics = statisticSelector.currentText()
def run(self):
self.mutex.lock()
self.stopMe = 0
self.mutex.unlock()
interrupted = False
polyProvider = self.layerPoly.dataProvider()
pointProvider = self.layerPoints.dataProvider()
fieldList = ftools_utils.getFieldList(self.layerPoly)
index = polyProvider.fieldNameIndex(unicode(self.fieldName))
if index == -1:
index = polyProvider.fields().count()
fieldList.append(
QgsField(unicode(self.fieldName), QVariant.Int, "int", 10, 0,
self.tr("point count field")))
# Add the selected vector fields to the output polygon vector layer
selectedItems = self.attributeList.selectedItems()
for item in selectedItems:
global typeDouble
columnName = unicode(item.text() + "_" + self.statistics)
index = polyProvider.fieldNameIndex(unicode(columnName))
if index == -1:
if item.type(
) == typeDouble or self.statistics == "mean" or self.statistics == "stddev":
fieldList.append(
QgsField(columnName, QVariant.Double, "double", 24, 15,
"Value"))
else:
fieldList.append(
QgsField(columnName, QVariant.Int, "int", 10, 0,
"Value"))
sRs = polyProvider.crs()
if QFile(self.outPath).exists():
if not QgsVectorFileWriter.deleteShapeFile(self.outPath):
return
writer = QgsVectorFileWriter(self.outPath, self.encoding, fieldList,
polyProvider.geometryType(), sRs)
spatialIndex = ftools_utils.createIndex(pointProvider)
self.emit(SIGNAL("rangeChanged(int)"), polyProvider.featureCount())
polyFeat = QgsFeature()
pntFeat = QgsFeature()
outFeat = QgsFeature()
inGeom = QgsGeometry()
polyFit = polyProvider.getFeatures()
while polyFit.nextFeature(polyFeat):
inGeom = polyFeat.geometry()
atMap = polyFeat.attributes()
outFeat.setAttributes(atMap)
outFeat.setGeometry(inGeom)
count = 0
pointList = []
hasIntersection = True
pointList = spatialIndex.intersects(inGeom.boundingBox())
if len(pointList) > 0:
hasIntersection = True
else:
hasIntersection = False
if hasIntersection:
valueList = {}
for item in selectedItems:
valueList[item.text()] = []
for p in pointList:
pointProvider.getFeatures(QgsFeatureRequest().setFilterFid(
p)).nextFeature(pntFeat)
tmpGeom = QgsGeometry(pntFeat.geometry())
if inGeom.intersects(tmpGeom):
count += 1
for item in selectedItems:
valueList[item.text()].append(
pntFeat.attribute(item.text()))
self.mutex.lock()
s = self.stopMe
self.mutex.unlock()
if s == 1:
interrupted = True
break
atMap.append(count)
# Compute the statistical values for selected vector attributes
for item in selectedItems:
values = valueList[item.text()]
# Check if the input contains non-numeric values
non_numeric_values = False
for value in values:
if not isinstance(value, type(
float())) and not isinstance(
value, type(int())):
non_numeric_values = True
break
# Jump over invalid values
if non_numeric_values is True:
continue
if values and len(values) > 0:
if self.statistics == "sum":
value = reduce(myAdder, values)
elif self.statistics == "mean":
value = reduce(myAdder, values) / float(
len(values))
elif self.statistics == "min":
values.sort()
value = values[0]
elif self.statistics == "max":
values.sort()
value = values[-1]
elif self.statistics == "stddev":
value = two_pass_variance(values)
value = math.sqrt(value)
atMap.append(value)
outFeat.setAttributes(atMap)
writer.addFeature(outFeat)
self.emit(SIGNAL("updateProgress()"))
self.mutex.lock()
s = self.stopMe
self.mutex.unlock()
if s == 1:
interrupted = True
break
del writer
if not interrupted:
self.emit(SIGNAL("processingFinished()"))
else:
self.emit(SIGNAL("processingInterrupted()"))
def stop(self):
self.mutex.lock()
self.stopMe = 1
self.mutex.unlock()
QThread.wait(self)
class Walker(QThread):
COMMON_WORDS_THRESHOLD = 250
MIN_WORD_LEN = 3
MAX_WORD_LEN = 25
INVALID_FIRST_OR_LAST = frozenset("0123456789_")
STRIPHTML_RE = re.compile(r"<[^>]*?>", re.IGNORECASE | re.MULTILINE)
ENTITY_RE = re.compile(r"&(\w+?);|&#(\d+?);")
SPLIT_RE = re.compile(r"\W+", re.IGNORECASE | re.MULTILINE)
def __init__(self,
index,
lock,
files,
filenamesForWords,
commonWords,
parent=None):
super(Walker, self).__init__(parent)
self.index = index
self.lock = lock
self.files = files
self.filenamesForWords = filenamesForWords
self.commonWords = commonWords
self.stopped = False
self.mutex = QMutex()
self.completed = False
def stop(self):
try:
self.mutex.lock()
self.stopped = True
finally:
self.mutex.unlock()
def isStopped(self):
try:
self.mutex.lock()
return self.stopped
finally:
self.mutex.unlock()
def run(self):
self.processFiles()
self.stop()
self.emit(SIGNAL("finished(bool,int)"), self.completed, self.index)
def processFiles(self):
def unichrFromEntity(match):
text = match.group(match.lastindex)
if text.isdigit():
return unichr(int(text))
u = htmlentitydefs.name2codepoint.get(text)
return unichr(u) if u is not None else ""
for fname in self.files:
if self.isStopped():
return
words = set()
fh = None
try:
fh = codecs.open(fname, "r", "UTF8", "ignore")
text = fh.read()
except (IOError, OSError), e:
sys.stderr.write("Error: {0}\n".format(e))
continue
finally:
class SVDViewer(QWidget):
svdSelectEvent = QtCore.pyqtSignal()
def __init__(self, array, labels, **options):
QWidget.__init__(self)
assert isinstance(array, np.ndarray),\
"Got a %s instead of an ndarray" % type(array)
# figure out our size
self.width = self.size().width()
self.height = self.size().height()
self.labels = labels
self.magnitudes = None # can be assigned by external information
self.array = np.asarray(array)
self.orig_array = self.array.copy()
self.npoints = self.array.shape[0]
self.scale = self.calculate_scale()
if options.get('jitter', True):
self.add_jitter()
self.layers = []
self.k = self.array.shape[1]
self.projection = Projection(self.k)
# create our internal display
self.painter = QPainter()
self.selected_index = None
self.mouseX = 0
self.mouseY = 0
self.buttons = 0
# avoid racing
self.paint_lock = QMutex()
self.timer = QTimer(self)
self.timer_ticks = 0
self.timer.setInterval(30)
self.timer.timeout.connect(self.timerEvent)
self.timer.start()
self.setMouseTracking(True)
self.default_colors = self.components_to_colors(self.array)[:]
self.update_colors()
# initialize the mapping from projection coordinates to screen
# coordinates
self.reset_view()
def age_timer(self):
self.timer_ticks += 1
if self.timer_ticks == TIMER_MAX:
self.timer.stop()
def activate_timer(self):
if self.timer_ticks >= TIMER_MAX:
self.timer.start()
self.timer_ticks = 0
def stop_timer(self):
self.timer.stop()
def __del__(self):
self.stop_timer()
def calculate_scale(self):
"""
Find roughly the median of axis coordinates, determining
a reasonable zoom level for the initial view. Make sure it's non-zero.
"""
coords = [c for c in np.abs(self.array.flatten()) if c > 0] + [1.0]
coords.sort()
return coords[len(coords)//2]
def add_jitter(self):
self.jitter = np.exp(np.random.normal(size=self.array.shape) / 50.0)
self.array *= self.jitter
def insert_layer(self, pos, layertype, *args):
"""
Add a layer of visualization to this widget, by specifying the
subclass of Layer and whatever arguments its constructor takes.
"""
self.layers.insert(pos, layertype(self, *args))
def add_layer(self, layertype, *args):
"""
Add a layer of visualization to this widget, by specifying the
subclass of Layer and whatever arguments its constructor takes.
"""
self.layers.append(layertype(self, *args))
def reset_view(self):
self.screen_center = np.array([0., 0.])
self.screen_size = np.array([self.scale/10, self.scale/10])
self.projection.reset_projection()
self.set_default_axes()
self.update_screenpts()
self.update()
@staticmethod
def make_svdview(matrix, svdmatrix, magnitudes=None, canonical=None):
widget = SVDViewer(svdmatrix, svdmatrix.row_labels)
if magnitudes is None:
magnitudes = np.array([np.linalg.norm(vec) for vec in svdmatrix])
widget.magnitudes = magnitudes
widget.setup_standard_layers()
widget.set_default_axes()
if canonical is None: canonical = []
for c in canonical:
#svdmatrix[svdmatrix.row_index(c)] *= 4
magnitudes[svdmatrix.row_index(c)] *= 2
widget.insert_layer(1, CanonicalLayer, canonical)
widget.insert_layer(2, LinkLayer, matrix)
widget.insert_layer(3, NetworkLayer, 6)
return widget
@staticmethod
def make_colors(matrix, svdmatrix):
widget = SVDViewer(svdmatrix, svdmatrix.col_labels)
widget.setup_standard_layers()
from csc.concepttools.colors import text_color
colors = [text_color(text) for text in widget.labels]
widget.default_colors = np.clip(np.array(colors), 55, 230)
widget.update_colors()
return widget
@staticmethod
def make(array, labels):
widget = SVDViewer(array, labels)
widget.setup_standard_layers()
return widget
def setup_standard_layers(self):
self.add_layer(PixelRenderingLayer)
self.add_layer(PointLayer)
self.add_layer(LabelLayer, 1000, 2000)
self.add_layer(SelectionLayer)
self.add_layer(SimilarityLayer)
self.add_layer(RotationLayer)
self.add_layer(PanZoomLayer)
def set_default_axes(self):
self.set_axis_to_pc(0, 1)
self.set_axis_to_pc(1, 2)
def set_default_x_axis(self):
self.set_axis_to_pc(0, 1)
def set_default_y_axis(self):
self.set_axis_to_pc(1, 2)
def set_axis_to_pc(self, axis, pc):
"""
Sets an axis to a particular principal component.
"""
pcvec = np.zeros((self.array.shape[1],))
pcvec[pc] = 1.0
if axis == 0:
self.projection.set_x_axis(pcvec)
elif axis == 1:
self.projection.set_y_axis(pcvec)
self.activate_timer()
def set_axis_to_text(self, axis, text):
if isinstance(text, QString):
text = unicode(text)
if not text: return
if text in self.labels:
index = self.labels.index(text)
if axis == 0:
self.projection.set_x_axis(self.array[index,:])
elif axis == 1:
self.projection.set_y_axis(self.array[index,:])
self.activate_timer()
else:
print repr(text), "not in label list"
def is_point_on_screen(self, coords):
return np.all((coords >= np.int32([0, 0])) &
(coords < np.int32([self.width, self.height])),
axis=-1)
def is_on_screen(self, x, y):
return x >= 0 and x < self.width and y >= 0 and y < self.height
def pixel_size(self):
"""
What is the diameter of a single pixel at the current zoom level?
If the x and y scales end up being different, take the geometric mean.
"""
xsize = self.screen_size[0] / self.width
ysize = self.screen_size[1] / self.height
return np.sqrt(xsize*ysize)
def components_to_screen(self, coords):
return self.projection_to_screen(self.projection.components_to_projection(coords))
def projection_to_screen(self, coords):
zoomed = (coords - self.screen_center) / self.screen_size
screen = (zoomed + np.array([self.width, self.height])/2)
return np.int32(screen * FLIP_Y + [0, self.height])
def screen_to_projection(self, screen_coords):
zoomed = ((screen_coords - [0, self.height])*FLIP_Y) - np.array([self.width, self.height])/2
return (zoomed * self.screen_size) + self.screen_center
def components_to_colors(self, coords):
while coords.shape[1] < 5:
coords = np.concatenate([coords, -coords, coords], axis=1)
return np.clip(np.int32(coords[...,3:6]*80/self.scale + 160), 50, 230)
def update_screenpts(self):
self.screenpts = self.components_to_screen(self.array)
def update_colors(self):
self.colors = self.default_colors
def constrain_to_screen(self, points):
return np.clip(points,
np.int32([1., 1.]), np.int32([self.width-2, self.height-2])).T
def distances_from_mouse(self, coords):
mouse = np.int32([self.mouseX, self.mouseY])
offsets = self.screenpts - mouse
return np.sqrt(np.sum(offsets*offsets, axis=1))
def get_nearest_point(self):
return np.argmin(self.distances_from_mouse(self.array))
def select_nearest_point(self):
self.selected_index = self.get_nearest_point()
self.selectEvent(self.selected_index)
def selected_vector(self):
return self.array[self.selected_index]
def selected_label(self):
return self.labels[self.selected_index]
def selectEvent(self, index):
for layer in self.layers:
layer.selectEvent(index)
self.svdSelectEvent.emit()
def focus_on_point(self, text):
index = self.labels.index(text)
if index is None: return
coords = self.projection.components_to_projection(self.array[index])
if not self.is_point_on_screen(coords):
self.screen_center = coords
self.selected_index = index
self.selectEvent(index)
def paintEvent(self, event):
if self.paint_lock.tryLock():
QWidget.paintEvent(self, event)
self.painter.begin(self)
self.painter.setRenderHint(QPainter.Antialiasing, True)
self.painter.setRenderHint(QPainter.TextAntialiasing, True)
try:
for layer in self.layers:
layer.draw(self.painter)
finally:
self.painter.end()
self.paint_lock.unlock()
def resizeEvent(self, sizeEvent):
self.width = sizeEvent.size().width()
self.height = sizeEvent.size().height()
for layer in self.layers:
layer.resize(self.width, self.height)
def mouseMoveEvent(self, mouseEvent):
point = mouseEvent.pos()
self.mouseX = point.x()
self.mouseY = point.y()
for layer in self.layers: layer.mouseMoveEvent(mouseEvent)
if self.leftMouseDown() or self.rightMouseDown():
self.activate_timer()
elif self.timer_ticks >= TIMER_MAX:
self.update()
def timerEvent(self):
self.projection.timerEvent()
for layer in self.layers: layer.timerEvent()
self.update_screenpts()
self.update()
self.age_timer()
def updateMouseButtons(self, event):
self.buttons = event.buttons()
self.modifiers = event.modifiers()
def leftMouseDown(self):
'''
Checks if the left mouse button was pressed at the time of the last event.
Right mouse clicks can be faked by holding a modifier (Ctrl on Macs, Logo key on Windows).
'''
return (self.buttons & Qt.LeftButton) and not (self.modifiers & RIGHT_BUTTON_MODIFIER)
def rightMouseDown(self):
'''
Checks if the right mouse button was pressed at the time of the last event.
Right mouse clicks can be faked by holding a modifier (Ctrl on Macs, Logo key on Windows).
'''
return (self.buttons & Qt.RightButton) or (self.buttons & Qt.LeftButton and self.modifiers & RIGHT_BUTTON_MODIFIER)
def mousePressEvent(self, mouseEvent):
self.updateMouseButtons(mouseEvent)
for layer in self.layers: layer.mousePressEvent(mouseEvent)
self.activate_timer()
def mouseReleaseEvent(self, mouseEvent):
self.updateMouseButtons(mouseEvent)
for layer in self.layers: layer.mouseReleaseEvent(mouseEvent)
self.activate_timer()
def wheelEvent(self, mouseEvent):
for layer in self.layers: layer.wheelEvent(mouseEvent)
self.activate_timer()
def dropLabel(self, index, label):
self.refreshData(self, index)
def refreshData(self):
self.update_screenpts()
self.default_colors = self.components_to_colors(self.array)[:]
self.update_colors()
self.update()
def get_svg_figure(self):
figs = []
for layer in self.layers:
fig = layer.drawSVG()
if fig is not None: figs.append(fig)
return svgfig.Fig(*figs)
def write_svg(self, filename=None):
if filename is None: filename = 'luminoso.svg'
out = open(filename, 'w')
out.write("<svg>\n")
out.write(self.get_svg_figure().SVG().xml())
out.write("\n</svg>\n")
out.close()
class ShapeMergeThread(QThread):
def __init__(self, dir, shapes, inputEncoding, outputFileName, outputEncoding):
QThread.__init__(self, QThread.currentThread())
self.baseDir = dir
self.shapes = shapes
self.inputEncoding = inputEncoding
self.outputFileName = outputFileName
self.outputEncoding = outputEncoding
self.mutex = QMutex()
self.stopMe = 0
def run(self):
self.mutex.lock()
self.stopMe = 0
self.mutex.unlock()
interrupted = False
# create attribute list with uniquie fields
# from all selected layers
mergedFields = []
self.emit(SIGNAL("rangeChanged( PyQt_PyObject )"), len(self.shapes))
self.emit(SIGNAL("checkStarted()"))
shapeIndex = 0
fieldMap = {}
for fileName in self.shapes:
layerPath = QFileInfo(self.baseDir + "/" + fileName).absoluteFilePath()
newLayer = QgsVectorLayer(layerPath, QFileInfo(layerPath).baseName(), "ogr")
if not newLayer.isValid():
continue
newLayer.setProviderEncoding(self.inputEncoding)
vprovider = newLayer.dataProvider()
fieldMap[shapeIndex] = {}
fieldIndex = 0
for layerField in vprovider.fields():
fieldFound = False
for mergedFieldIndex, mergedField in enumerate(mergedFields):
if mergedField.name() == layerField.name() and mergedField.type() == layerField.type():
fieldFound = True
fieldMap[shapeIndex][fieldIndex] = mergedFieldIndex
if mergedField.length() < layerField.length():
# suit the field size to the field of this layer
mergedField.setLength(layerField.length())
break
if not fieldFound:
fieldMap[shapeIndex][fieldIndex] = len(mergedFields)
mergedFields.append(layerField)
fieldIndex += 1
shapeIndex += 1
self.emit(SIGNAL("featureProcessed()"))
self.emit(SIGNAL("checkFinished()"))
# get information about shapefiles
layerPath = QFileInfo(self.baseDir + "/" + self.shapes[0]).absoluteFilePath()
newLayer = QgsVectorLayer(layerPath, QFileInfo(layerPath).baseName(), "ogr")
self.crs = newLayer.crs()
self.geom = newLayer.wkbType()
vprovider = newLayer.dataProvider()
fields = QgsFields()
for f in mergedFields:
fields.append(f)
writer = QgsVectorFileWriter(
self.outputFileName, self.outputEncoding,
fields, self.geom, self.crs)
shapeIndex = 0
for fileName in self.shapes:
layerPath = QFileInfo(self.baseDir + "/" + fileName).absoluteFilePath()
newLayer = QgsVectorLayer(layerPath, QFileInfo(layerPath).baseName(), "ogr")
if not newLayer.isValid():
continue
newLayer.setProviderEncoding(self.inputEncoding)
vprovider = newLayer.dataProvider()
nFeat = vprovider.featureCount()
self.emit(SIGNAL("rangeChanged( PyQt_PyObject )"), nFeat)
self.emit(SIGNAL("fileNameChanged( PyQt_PyObject )"), fileName)
inFeat = QgsFeature()
outFeat = QgsFeature()
inGeom = QgsGeometry()
fit = vprovider.getFeatures()
while fit.nextFeature(inFeat):
mergedAttrs = [""] * len(mergedFields)
# fill available attributes with values
fieldIndex = 0
for v in inFeat.attributes():
if shapeIndex in fieldMap and fieldIndex in fieldMap[shapeIndex]:
mergedAttrs[fieldMap[shapeIndex][fieldIndex]] = v
fieldIndex += 1
if inFeat.geometry() is not None:
inGeom = QgsGeometry(inFeat.geometry())
outFeat.setGeometry(inGeom)
outFeat.setAttributes(mergedAttrs)
writer.addFeature(outFeat)
self.emit(SIGNAL("featureProcessed()"))
self.emit(SIGNAL("shapeProcessed()"))
self.mutex.lock()
s = self.stopMe
self.mutex.unlock()
if s == 1:
interrupted = True
break
shapeIndex += 1
del writer
if not interrupted:
self.emit(SIGNAL("processingFinished()"))
else:
self.emit(SIGNAL("processingInterrupted()"))
def stop(self):
self.mutex.lock()
self.stopMe = 1
self.mutex.unlock()
QThread.wait(self)
class Search_worker(QThread):
"""
Defines a `QThread <http://doc.qt.nokia.com/qthread.html>`_ subclass used
to search for a pattern in a directory files.
"""
# Custom signals definitions.
searchFinished = pyqtSignal(list)
"""
This signal is emited by the :class:`Search_worker` class when the search is finished. ( pyqtSignal )
:return: Search results.
:rtype: list
"""
def __init__(self, parent, pattern=None, location=None, settings=None):
"""
Initializes the class.
:param parent: Object parent.
:type parent: QObject
"""
LOGGER.debug("> Initializing '{0}()' class.".format(self.__class__.__name__))
QThread.__init__(self, parent)
# --- Setting class attributes. ---
self.__container = parent
self.__pattern = None
self.pattern = pattern
self.__location = None
self.location = location
self.__settings = None
self.settings = settings
self.__searchResults = None
self.__interrupt = False
self.__lock = QMutex()
#******************************************************************************************************************
#*** Attributes properties.
#******************************************************************************************************************
@property
def container(self):
"""
Property for **self.__container** attribute.
:return: self.__container.
:rtype: QObject
"""
return self.__container
@container.setter
@foundations.exceptions.handleExceptions(foundations.exceptions.ProgrammingError)
def container(self, value):
"""
Setter for **self.__container** attribute.
:param value: Attribute value.
:type value: QObject
"""
raise foundations.exceptions.ProgrammingError(
"{0} | '{1}' attribute is read only!".format(self.__class__.__name__, "container"))
@container.deleter
@foundations.exceptions.handleExceptions(foundations.exceptions.ProgrammingError)
def container(self):
"""
Deleter for **self.__container** attribute.
"""
raise foundations.exceptions.ProgrammingError(
"{0} | '{1}' attribute is not deletable!".format(self.__class__.__name__, "container"))
@property
def pattern(self):
"""
Property for **self.__pattern** attribute.
:return: self.__pattern.
:rtype: unicode
"""
return self.__pattern
@pattern.setter
@foundations.exceptions.handleExceptions(AssertionError)
def pattern(self, value):
"""
Setter for **self.__pattern** attribute.
:param value: Attribute value.
:type value: unicode
"""
if value is not None:
assert type(value) in (unicode, QString), \
"'{0}' attribute: '{1}' type is not 'unicode' or 'QString'!".format("pattern", value)
self.__pattern = value
@pattern.deleter
@foundations.exceptions.handleExceptions(foundations.exceptions.ProgrammingError)
def pattern(self):
"""
Deleter for **self.__pattern** attribute.
"""
raise foundations.exceptions.ProgrammingError(
"{0} | '{1}' attribute is not deletable!".format(self.__class__.__name__, "pattern"))
@property
def location(self):
"""
Property for **self.__location** attribute.
:return: self.__location.
:rtype: Location
"""
return self.__location
@location.setter
@foundations.exceptions.handleExceptions(AssertionError)
def location(self, value):
"""
Setter for **self.__location** attribute.
:param value: Attribute value.
:type value: Location
"""
if value is not None:
assert type(value) is umbra.ui.common.Location, \
"'{0}' attribute: '{1}' type is not 'umbra.ui.common.Location'!".format("location", value)
self.__location = value
@location.deleter
@foundations.exceptions.handleExceptions(foundations.exceptions.ProgrammingError)
def location(self):
"""
Deleter for **self.__location** attribute.
"""
raise foundations.exceptions.ProgrammingError(
"{0} | '{1}' attribute is not deletable!".format(self.__class__.__name__, "location"))
@property
def settings(self):
"""
Property for **self.__settings** attribute.
:return: self.__settings.
:rtype: Location
"""
return self.__settings
@settings.setter
@foundations.exceptions.handleExceptions(AssertionError)
def settings(self, value):
"""
Setter for **self.__settings** attribute.
:param value: Attribute value.
:type value: Location
"""
if value is not None:
assert type(value) is dict, "'{0}' attribute: '{1}' type is not 'dict'!".format("settings", value)
self.__settings = foundations.dataStructures.Structure(**{"caseSensitive" : False,
"wholeWord" : False,
"regularExpressions" : False})
self.__settings.update(value)
@settings.deleter
@foundations.exceptions.handleExceptions(foundations.exceptions.ProgrammingError)
def settings(self):
"""
Deleter for **self.__settings** attribute.
"""
raise foundations.exceptions.ProgrammingError(
"{0} | '{1}' attribute is not deletable!".format(self.__class__.__name__, "settings"))
@property
def searchResults(self):
"""
Property for **self.__searchResults** attribute.
:return: self.__searchResults.
:rtype: list
"""
return self.__searchResults
@searchResults.setter
@foundations.exceptions.handleExceptions(foundations.exceptions.ProgrammingError)
def searchResults(self, value):
"""
Setter for **self.__searchResults** attribute.
:param value: Attribute value.
:type value: list
"""
raise foundations.exceptions.ProgrammingError(
"{0} | '{1}' attribute is read only!".format(self.__class__.__name__, "searchResults"))
@searchResults.deleter
@foundations.exceptions.handleExceptions(foundations.exceptions.ProgrammingError)
def searchResults(self):
"""
Deleter for **self.__searchResults** attribute.
"""
raise foundations.exceptions.ProgrammingError(
"{0} | '{1}' attribute is not deletable!".format(self.__class__.__name__, "searchResults"))
#******************************************************************************************************************
#*** Class methods.
#******************************************************************************************************************
def run(self):
"""
Reimplements the :meth:`QThread.run` method.
"""
self.__search()
def quit(self):
"""
Reimplements the :meth:`QThread.quit` method.
"""
self.__interrupt = True
QThread.quit(self)
def __search(self):
"""
Performs the search.
"""
self.__searchResults = []
editorsFiles = self.__container.defaultTarget in self.__location.targets and \
[editor.file for editor in self.__container.scriptEditor.listEditors()] or []
self.__searchEditorsFiles(editorsFiles)
self.__searchFiles(self.__location.files)
for directory in self.__location.directories:
if self.__interrupt:
return
filesWalker = foundations.walkers.filesWalker(directory,
self.__location.filtersIn,
list(itertools.chain(self.__location.filtersOut,
self.__location.files,
editorsFiles)))
self.__searchFiles(filesWalker)
not self.__interrupt and self.searchFinished.emit(self.__searchResults)
def __searchEditorsFiles(self, files):
"""
Searches in :class:`umbra.components.factory.scriptEditor.scriptEditor.ScriptEditor` class editors files.
:param files: Editor files.
:type files: list
"""
for file in files:
if self.__interrupt:
return
if foundations.io.isReadable(file):
if foundations.io.isBinaryFile(file):
continue
LOGGER.info("{0} | Searching '{1}' file!".format(self.__class__.__name__, file))
editor = self.__container.scriptEditor.getEditor(file)
if not editor:
continue
self.__lock.lock()
occurrences = self.__searchDocument(editor.document(), self.__pattern, self.__settings)
self.__lock.unlock()
occurrences and self.__searchResults.append(SearchResult(file=file,
pattern=self.__pattern,
settings=self.__settings,
occurrences=occurrences))
def __searchFiles(self, files):
"""
Searches in given files.
:param files: Files.
:type files: list
"""
for file in files:
if self.__interrupt:
return
if not foundations.common.pathExists(file):
continue
if foundations.io.isReadable(file):
if foundations.io.isBinaryFile(file):
continue
LOGGER.info("{0} | Searching '{1}' file!".format(self.__class__.__name__, file))
cacheData = self.__container.filesCache.getContent(file)
if not cacheData:
reader = foundations.io.File(file)
content = reader.read()
if content is None:
LOGGER.warning("!> Error occured while reading '{0}' file proceeding to next one!".format(file))
continue
self.__container.filesCache.addContent(**{file : CacheData(content=content, document=None)})
else:
content = cacheData.content
occurrences = self.__searchDocument(QTextDocument(QString(content)), self.__pattern, self.__settings)
occurrences and self.__searchResults.append(SearchResult(file=file,
pattern=self.__pattern,
settings=self.__settings,
occurrences=occurrences))
def __searchDocument(self, document, pattern, settings):
"""
Searches for given pattern occurrences in given document using given settings.
:param document: Document.
:type document: QTextDocument
:param pattern: Pattern.
:type pattern: unicode
:param settings: Search settings.
:type settings: Structure
:return: Matched occurrences.
:rtype: list
"""
pattern = settings.regularExpressions and QRegExp(pattern) or pattern
flags = QTextDocument.FindFlags()
if settings.caseSensitive:
flags = flags | QTextDocument.FindCaseSensitively
if settings.wholeWord:
flags = flags | QTextDocument.FindWholeWords
occurrences = []
block = document.findBlock(0)
cursor = document.find(pattern, block.position(), flags)
while block.isValid() and cursor.position() != -1:
if self.__interrupt:
return
blockCursor = QTextCursor(cursor)
blockCursor.movePosition(QTextCursor.StartOfLine, QTextCursor.MoveAnchor)
blockCursor.movePosition(QTextCursor.EndOfLine, QTextCursor.KeepAnchor)
length = cursor.selectionEnd() - cursor.selectionStart()
occurrences.append(Occurence(line=cursor.blockNumber(),
column=cursor.columnNumber() - length,
length=length,
position=cursor.position() - length,
text=blockCursor.selectedText()))
cursor = document.find(pattern, cursor.position(), flags)
block = block.next()
return occurrences
class SplitThread(QThread):
def __init__(self, layer, splitField, encoding, outDir):
QThread.__init__(self, QThread.currentThread())
self.layer = layer
self.field = splitField
self.encoding = encoding
self.outDir = outDir
self.mutex = QMutex()
self.stopMe = 0
self.errors = []
def run(self):
self.mutex.lock()
self.stopMe = 0
self.mutex.unlock()
interrupted = False
outPath = self.outDir
if outPath.find("\\") != -1:
outPath.replace("\\", "/")
if not outPath.endswith("/"):
outPath = outPath + "/"
provider = self.layer.dataProvider()
index = provider.fieldNameIndex(self.field)
unique = ftools_utils.getUniqueValues(provider, int(index))
baseName = unicode( outPath + self.layer.name() + "_" + self.field + "_" )
fieldList = ftools_utils.getFieldList(self.layer)
sRs = provider.crs()
geom = self.layer.wkbType()
inFeat = QgsFeature()
self.emit(SIGNAL("rangeCalculated(PyQt_PyObject)"), len(unique))
for i in unique:
check = QFile(baseName + "_" + unicode(i).strip() + ".shp")
fName = check.fileName()
if check.exists():
if not QgsVectorFileWriter.deleteShapeFile(fName):
self.errors.append( fName )
continue
writer = QgsVectorFileWriter(fName, self.encoding, fieldList, geom, sRs)
fit = provider.getFeatures()
while fit.nextFeature(inFeat):
atMap = inFeat.attributes()
if atMap[index] == i:
writer.addFeature(inFeat)
del writer
self.emit(SIGNAL("valueProcessed()"))
self.mutex.lock()
s = self.stopMe
self.mutex.unlock()
if s == 1:
interrupted = True
break
if not interrupted:
self.emit(SIGNAL("processFinished( PyQt_PyObject )"), self.errors)
else:
self.emit(SIGNAL("processInterrupted()"))
def stop(self):
self.mutex.lock()
self.stopMe = 1
self.mutex.unlock()
QThread.wait(self)
class BeamshutterWidget(QWidget):
currentShutterState = pyqtSignal(str, name='currentShutterState')
def __init__(self, parent=None):
QWidget.__init__(self, parent)
self.ui = uic.loadUi("Beamshutter.ui", self)
self.control = QAxContainer.QAxWidget(self)
self.control.setControl('{3CE35BF3-1E13-4D2C-8C0B-DEF6314420B3}')
self.control.setProperty("HWSerialNum", 85845031)
self.control.setGeometry(0, 0, 600, 400)
self.mutex = QMutex(mode=QMutex.Recursive)
self.shutterReplied = False
self.shut = Beamshutter(self)
self.worker = WorkerThread()
self.ui.mainLayout.addWidget(self.control)
self.layout().setAlignment(Qt.AlignCenter)
sizePolicy = QSizePolicy(QSizePolicy.Preferred, QSizePolicy.Preferred)
sizePolicy.setHeightForWidth(True)
self.setSizePolicy(sizePolicy)
def heightForWidth(self, width):
return width * 0.75
@pyqtSlot()
def on_BInitializeShutter_clicked(self):
if not self.LEDShutter.isChecked():
self.initialize()
else:
self.finalize()
def initialize(self):
with self.check(timeOut=10):
self.worker.executeFunction(self.shut.initialize)
self.ui.LEDShutter.setChecked(True)
def finalize(self):
with self.check():
self.worker.executeFunction(self.shut.finalize)
self.LEDShutter.setChecked(False)
def enable(self):
with self.check():
self.worker.executeFunction(self.shut.enable)
def disable(self):
with self.check():
self.worker.executeFunction(self.shut.disable)
def _del(self):
if self.ui.LEDShutter.isChecked():
self.finalize()
def heightForWidth(self, w):
return (w)
@contextmanager
def check(self, timeOut=2):
out = datetime.datetime.now() + datetime.timedelta(seconds=timeOut)
self.shutterReplied = shutterReplied = False
yield
while datetime.datetime.now() < out and shutterReplied == False:
self.mutex.lock()
shutterReplied = self.shutterReplied
self.mutex.unlock()
if shutterReplied == False:
raise Exception("ThorlabsShutter timed out and might be Crashed")
class StdErrWrapper(QObject):
_write = pyqtSignal(type(u''))
_flush = pyqtSignal()
def __init__(self, old_stderr):
QObject.__init__(self)
self._old_stderr = old_stderr
self._widget = None
self._mutex = QMutex(QMutex.Recursive)
def setApplication(self, app):
assert (self._widget is None)
widget = QPlainTextEdit()
widget.setWindowTitle(u"Error Console")
widget.resize(486, 300)
widget.appendHtml(
u'<span style="color: green">'
u'An unhandled error occurred.<br>'
u'Sorry for the inconvinience.<br>'
u'Please copy the following text into a bug report:<br><br>'
u'</span>')
app.aboutToQuit.connect(self.restoreStdErr)
self._write.connect(self._write_handler)
self._flush.connect(self._flush_handler)
self._widget = widget
def _write_handler(self, data):
self._mutex.lock()
if self._widget:
self._widget.show()
self._widget.insertPlainText(data)
self._mutex.unlock()
def _flush_handler(self):
self._mutex.lock()
if self._widget:
self._widget.show()
self._mutex.unlock()
def restoreStdErr(self):
self._mutex.lock()
if self._widget:
self._widget.close()
self._widget = None
self._mutex.unlock()
@property
def encoding(self):
return 'utf-8'
def write(self, s):
self._mutex.lock()
if self._widget:
if isinstance(s, bytes):
s = s.decode('utf-8', 'replace')
self._write.emit(s)
else:
self._old_stderr.write(s)
self._mutex.unlock()
def flush(self):
self._mutex.lock()
if self._widget:
self._flush.emit()
else:
self._old_stderr.flush()
self._mutex.unlock()
class MessageReceiver:
""""
Checks given rx buffer if message is present there.
struct Tail
{
uint8_t tail_start = TAIL_START_MARK; // '<'
uint16_t id = 0;
uint16_t context = 0;
uint16_t msg_len = 0;
uint16_t body_crc= 0;
uint8_t tail_end = TAIL_END_MARK; // '>'
}
uint16_t tail_crc= 0;
"""
TAIL_LEN = 10
TAIL_START_MARK = ord('<')
TAIL_END_MARK = ord('>')
TAIL_CRC_SHIFT_POS = 3
ts = time.time()
LOCKED = False
def __init__(self, rx_buffer):
self.rx_buffer = rx_buffer
self.__mean_rx_time = MeanCalculator()
self.mutex = QMutex()
self.t0 = time.time()
def check_tail(self, peek_buff):
init_find = peek_buff.find(MessageReceiver.TAIL_START_MARK)
peek_buff_len = len(peek_buff)
for i in range(peek_buff_len - MessageReceiver.TAIL_LEN + init_find):
latest_find = i + init_find
try:
tail_start_mark_pos = peek_buff[latest_find:].find(
MessageReceiver.TAIL_START_MARK)
tail_end_mark_pos = tail_start_mark_pos + MessageReceiver.TAIL_LEN - 1
tail_end_mark = peek_buff[latest_find:][tail_end_mark_pos]
except IndexError:
return False
try:
if tail_end_mark == MessageReceiver.TAIL_END_MARK:
peek_buff = peek_buff[latest_find:]
tail = peek_buff[tail_start_mark_pos:tail_end_mark_pos]
_full_tail = peek_buff[
tail_start_mark_pos:tail_end_mark_pos + 1]
_id = struct.unpack('H', tail[1:3])[0]
_context = struct.unpack('H', tail[3:5])[0]
_msg_len = struct.unpack('H', tail[5:7])[0]
_body_crc = tail[7:9]
_tail_crc = peek_buff[tail_end_mark_pos +
1:tail_end_mark_pos +
MessageReceiver.TAIL_CRC_SHIFT_POS]
tail_integrity = _tail_crc == crc_bytes(
_full_tail) # tail integrity check
if _id < len(RxMessage.rx_id_tuple) and _msg_len < MAX_PACKET_SIZE and _context < 0xffff \
and self.rx_buffer.available() > _msg_len and tail_integrity:
return _id, _context, _msg_len, _body_crc, tail_start_mark_pos + latest_find, tail_end_mark_pos + latest_find
except struct.error:
return False
return False
def get_message(self):
t0 = time.time()
ret_rxmsg = None
if self.rx_buffer.available() >= MessageReceiver.TAIL_LEN:
self.mutex.lock()
peek_buff = self.rx_buffer.peek()
check_tail_result = self.check_tail(peek_buff)
self.mutex.unlock()
if check_tail_result:
_id, _context, _msg_len, _crc, tail_start_mark_pos, tail_end_mark_pos = check_tail_result
self.mutex.lock()
msg_body = self.rx_buffer.read(tail_end_mark_pos + 1 + 2)
self.mutex.unlock()
msg_body = msg_body[:tail_start_mark_pos]
MessageReceiver.ts = time.time()
crc_check = RxMessage.RxId.ack if _crc == crc_bytes(
msg_body) else RxMessage.RxId.nack
rxmsg = RxMessage(msg_id=_id,
crc_check=crc_check,
length=len(msg_body),
context=_context,
body=msg_body)
m_logger.debug(MSG_RX_DBG_TEMPLATE.format(rxmsg))
self.t0 = time.time()
if _crc == crc_bytes(msg_body):
self.__mean_rx_time.count(time.time() - t0)
ret_rxmsg = rxmsg
m_logger.debug("Mean msg extract time: {}".format(
self.__mean_rx_time))
m_logger.debug("Period: {}".format(time.time() - self.t0))
return ret_rxmsg
class SiteMapThread(QThread):
def __init__(self, framework, treeViewModel, parent=None):
QThread.__init__(self, parent)
self.framework = framework
self.treeViewModel = treeViewModel
self.qtimer = QTimer()
self.qlock = QMutex()
self.fillAll = False
QObject.connect(self, SIGNAL("quit()"), self.quitHandler)
QObject.connect(self, SIGNAL("started()"), self.startedHandler)
self.re_set_cookie = re.compile(r"^Set-Cookie2?:\s*(.+)$", re.I | re.M)
self.re_host_name = re.compile(r"^Host:\s*(.+?)$", re.I | re.M)
self.lastId = 0
self.Data = None
self.cursor = None
def db_attach(self):
self.Data = self.framework.getDB()
self.cursor = self.Data.allocate_thread_cursor()
self.populateSiteMap(True)
def db_detach(self):
self.close_cursor()
self.Data = None
def close_cursor(self):
if self.cursor and self.Data:
self.cursor.close()
self.Data.release_thread_cursor(self.cursor)
self.cursor = None
def run(self):
QObject.connect(self, SIGNAL("updateSiteMap()"), self.doUpdateSiteMap, Qt.DirectConnection)
self.framework.subscribe_response_data_added(self.doUpdateSiteMap)
self.exec_()
def quitHandler(self):
self.framework.debug_log("SiteMapThread quit...")
self.close_cursor()
self.exit(0)
def startedHandler(self):
self.framework.debug_log("SiteMapThread started...")
self.framework.subscribe_database_events(self.db_attach, self.db_detach)
def populateSiteMap(self, fillAll):
self.fillAll = fillAll
QTimer.singleShot(50, self, SIGNAL("updateSiteMap()"))
def doUpdateSiteMap(self):
if not self.qlock.tryLock():
return
try:
if self.fillAll:
self.fillAll = False
self.treeViewModel.clearModel()
self.lastId = 0
rows = self.Data.get_sitemap_info(self.cursor, self.lastId)
global_cookie_jar = self.framework.get_global_cookie_jar()
count = 0
for row in rows:
count += 1
if 0 == (count % 100):
self.yieldCurrentThread()
rowItems = [m or "" for m in list(row)]
Id = str(rowItems[0])
try:
self.lastId = int(Id)
except ValueError:
pass
# XXX: review all for bytes usage
if isinstance(rowItems[1], bytes):
url = str(rowItems[1], "utf-8", "ignore")
else:
url = str(rowItems[1])
status = str(rowItems[2])
response_headers = str(rowItems[3])
request_headers = str(rowItems[4])
# TODO: make configurable
if status in ("400", "404", "500", "501"):
continue
# TODO:
m = self.re_set_cookie.search(response_headers)
if m:
setCookies = m.group(1)
cookieList = QNetworkCookie.parseCookies(setCookies)
global_cookie_jar.setCookiesFromUrl(cookieList, QUrl.fromEncoded(url))
parsed = urlparse.urlsplit(url)
hostname = ""
if not parsed.hostname:
m = self.re_host_name.search(request_headers)
if m:
hostname = m.group(1).rstrip()
else:
hostname = parsed.hostname
hostname = hostname.lower()
hostloc = urlparse.urlunsplit((parsed.scheme, parsed.netloc, "/", "", ""))
rootNode = self.treeViewModel.findOrAddNode(hostname)
hostLocNode = rootNode.findOrAddNode(self.treeViewModel, hostloc)
pathval = parsed.path
# add directories
parentNode = hostLocNode
parentNode.setResponseId(None, hostloc)
lastSlash = 0
slash = 0
while True:
slash = pathval.find("/", slash + 1)
if slash < 0:
break
dirname = pathval[lastSlash + 1 : slash + 1]
parentNode = parentNode.findOrAddNode(self.treeViewModel, dirname)
parentNode.setResponseId(
None, urlparse.urlunsplit((parsed.scheme, parsed.netloc, pathval[0 : slash + 1], "", ""))
)
lastSlash = slash
# add file element
if lastSlash + 1 < len(pathval):
filename = pathval[lastSlash + 1 :]
parentNode = parentNode.findOrAddNode(self.treeViewModel, filename)
parentNode.setResponseId(None, urlparse.urlunsplit((parsed.scheme, parsed.netloc, pathval, "", "")))
# add query
if parsed.query:
parentNode = parentNode.findOrAddNode(self.treeViewModel, "?" + parsed.query)
# store the latest Id
# TODO: should determine best candidate to display
parentNode.setResponseId(Id, url)
finally:
self.qlock.unlock()
class Walker(QThread):
COMMON_WORDS_THRESHOLD = 250
MIN_WORD_LEN = 3
MAX_WORD_LEN = 25
INVALID_FIRST_OR_LAST = frozenset("0123456789_")
STRIPHTML_RE = re.compile(r"<[^>]*?>", re.IGNORECASE | re.MULTILINE)
ENTITY_RE = re.compile(r"&(\w+?);|&#(\d+?);")
SPLIT_RE = re.compile(r"\W+", re.IGNORECASE | re.MULTILINE)
def __init__(self, lock, parent=None):
super(Walker, self).__init__(parent)
self.lock = lock
self.stopped = False
self.mutex = QMutex()
self.path = None
self.completed = False
def initialize(self, path, filenamesForWords, commonWords):
self.stopped = False
self.path = path
self.filenamesForWords = filenamesForWords
self.commonWords = commonWords
self.completed = False
def stop(self):
try:
self.mutex.lock()
self.stopped = True
finally:
self.mutex.unlock()
def isStopped(self):
try:
self.mutex.lock()
return self.stopped
finally:
self.mutex.unlock()
def run(self):
self.processFiles(self.path)
self.stop()
self.emit(SIGNAL("finished(bool)"), self.completed)
def processFiles(self, path):
def unichrFromEntity(match):
text = match.group(match.lastindex)
if text.isdigit():
return chr(int(text))
u = html.entities.name2codepoint.get(text)
return chr(u) if u is not None else ""
for root, dirs, files in os.walk(path):
if self.isStopped():
return
for name in [
name for name in files if name.endswith((".htm", ".html"))
]:
fname = os.path.join(root, name)
if self.isStopped():
return
words = set()
fh = None
try:
fh = codecs.open(fname, "r", "UTF8", "ignore")
text = fh.read()
except EnvironmentError as e:
sys.stderr.write("Error: {0}\n".format(e))
continue
finally:
if fh is not None:
fh.close()
if self.isStopped():
return
text = self.STRIPHTML_RE.sub("", text)
text = self.ENTITY_RE.sub(unichrFromEntity, text)
text = text.lower()
for word in self.SPLIT_RE.split(text):
if (self.MIN_WORD_LEN <= len(word) <= self.MAX_WORD_LEN
and word[0] not in self.INVALID_FIRST_OR_LAST
and word[-1] not in self.INVALID_FIRST_OR_LAST):
try:
self.lock.lockForRead()
new = word not in self.commonWords
finally:
self.lock.unlock()
if new:
words.add(word)
if self.isStopped():
return
for word in words:
try:
self.lock.lockForWrite()
files = self.filenamesForWords[word]
if len(files) > self.COMMON_WORDS_THRESHOLD:
del self.filenamesForWords[word]
self.commonWords.add(word)
else:
files.add(str(fname))
finally:
self.lock.unlock()
self.emit(SIGNAL("indexed(QString)"), fname)
self.completed = True
class Walker(QThread):
COMMON_WORDS_THRESHOLD = 250
MIN_WORD_LEN = 3
MAX_WORD_LEN = 25
INVALID_FIRST_OR_LAST = frozenset("0123456789_")
STRIPHTML_RE = re.compile(r"<[^>]*?>", re.IGNORECASE|re.MULTILINE)
ENTITY_RE = re.compile(r"&(\w+?);|&#(\d+?);")
SPLIT_RE = re.compile(r"\W+", re.IGNORECASE|re.MULTILINE)
def __init__(self, lock, parent=None):
super(Walker, self).__init__(parent)
self.lock = lock
self.stopped = False
self.mutex = QMutex()
self.path = None
self.completed = False
def initialize(self, path, filenamesForWords, commonWords):
self.stopped = False
self.path = path
self.filenamesForWords = filenamesForWords
self.commonWords = commonWords
self.completed = False
def stop(self):
try:
self.mutex.lock()
self.stopped = True
finally:
self.mutex.unlock()
def isStopped(self):
try:
self.mutex.lock()
return self.stopped
finally:
self.mutex.unlock()
def run(self):
self.processFiles(self.path)
self.stop()
self.emit(SIGNAL("finished(bool)"), self.completed)
def processFiles(self, path):
def unichrFromEntity(match):
text = match.group(match.lastindex)
if text.isdigit():
return unichr(int(text))
u = htmlentitydefs.name2codepoint.get(text)
return unichr(u) if u is not None else ""
for root, dirs, files in os.walk(path):
if self.isStopped():
return
for name in [name for name in files
if name.endswith((".htm", ".html"))]:
fname = os.path.join(root, name)
if self.isStopped():
return
words = set()
fh = None
try:
fh = codecs.open(fname, "r", "UTF8", "ignore")
text = fh.read()
except (IOError, OSError), e:
sys.stderr.write("Error: {0}\n".format(e))
continue
finally:
if fh is not None:
fh.close()
class VcsStatusMonitorThread(QThread):
"""
Class implementing the VCS status monitor thread base class.
@signal vcsStatusMonitorData(QStringList) emitted to update the VCS status
@signal vcsStatusMonitorStatus(QString, QString) emitted to signal the status of the
monitoring thread (ok, nok, op) and a status message
"""
def __init__(self, interval, projectDir, vcs, parent = None):
"""
Constructor
@param interval new interval in seconds (integer)
@param projectDir project directory to monitor (string or QString)
@param vcs reference to the version control object
@param parent reference to the parent object (QObject)
"""
QThread.__init__(self, parent)
self.setObjectName("VcsStatusMonitorThread")
self.setTerminationEnabled(True)
self.projectDir = QString(projectDir)
self.vcs = vcs
self.interval = interval
self.autoUpdate = False
self.statusList = QStringList()
self.reportedStates = {}
self.shouldUpdate = False
self.monitorMutex = QMutex()
self.monitorCondition = QWaitCondition()
self.__stopIt = False
def run(self):
"""
Protected method implementing the tasks action.
"""
while not self.__stopIt:
# perform the checking task
self.statusList.clear()
self.emit(SIGNAL("vcsStatusMonitorStatus(QString, QString)"),
QString("wait"), self.trUtf8("Waiting for lock"))
try:
locked = self.vcs.vcsExecutionMutex.tryLock(5000)
except TypeError:
locked = self.vcs.vcsExecutionMutex.tryLock()
if locked:
try:
self.emit(SIGNAL("vcsStatusMonitorStatus(QString, QString)"),
QString("op"), self.trUtf8("Checking repository status"))
res, statusMsg = self._performMonitor()
finally:
self.vcs.vcsExecutionMutex.unlock()
if res:
status = QString("ok")
else:
status = QString("nok")
self.emit(SIGNAL("vcsStatusMonitorStatus(QString, QString)"),
QString("send"), self.trUtf8("Sending data"))
self.emit(SIGNAL("vcsStatusMonitorData(QStringList)"),
QStringList(self.statusList))
self.emit(SIGNAL("vcsStatusMonitorStatus(QString, QString)"),
status, statusMsg)
else:
self.emit(SIGNAL("vcsStatusMonitorStatus(QString, QString)"),
QString("timeout"), self.trUtf8("Timed out waiting for lock"))
if self.autoUpdate and self.shouldUpdate:
try:
self.vcs.vcsUpdate(self.projectDir, True)
continue # check again
except TypeError:
pass # compatibility for older VCS plugins
self.shouldUpdate = False
# wait until interval has expired checking for a stop condition
self.monitorMutex.lock()
if not self.__stopIt:
self.monitorCondition.wait(self.monitorMutex, self.interval * 1000)
self.monitorMutex.unlock()
self.exit()
def setInterval(self, interval):
"""
Public method to change the monitor interval.
@param interval new interval in seconds (integer)
"""
locked = self.monitorMutex.tryLock()
self.interval = interval
self.monitorCondition.wakeAll()
if locked:
self.monitorMutex.unlock()
def getInterval(self):
"""
Public method to get the monitor interval.
@return interval in seconds (integer)
"""
return self.interval
def setAutoUpdate(self, auto):
"""
Public method to enable the auto update function.
@param auto status of the auto update function (boolean)
"""
self.autoUpdate = auto
def getAutoUpdate(self):
"""
Public method to retrieve the status of the auto update function.
@return status of the auto update function (boolean)
"""
return self.autoUpdate
def checkStatus(self):
"""
Public method to wake up the status monitor thread.
"""
locked = self.monitorMutex.tryLock()
self.monitorCondition.wakeAll()
if locked:
self.monitorMutex.unlock()
def stop(self):
"""
Public method to stop the monitor thread.
"""
locked = self.monitorMutex.tryLock()
self.__stopIt = True
self.monitorCondition.wakeAll()
if locked:
self.monitorMutex.unlock()
def clearCachedState(self, name):
"""
Public method to clear the cached VCS state of a file/directory.
@param name name of the entry to be cleared (QString or string)
"""
project = e4App().getObject("Project")
key = project.getRelativePath(unicode(name))
try:
del self.reportedStates[key]
except KeyError:
pass
def _performMonitor(self):
"""
Protected method implementing the real monitoring action.
This method must be overridden and populate the statusList member variable
with a list of strings giving the status in the first column and the
path relative to the project directory starting with the third column.
The allowed status flags are:
<ul>
<li>"A" path was added but not yet comitted</li>
<li>"M" path has local changes</li>
<li>"O" path was removed</li>
<li>"R" path was deleted and then re-added</li>
<li>"U" path needs an update</li>
<li>"Z" path contains a conflict</li>
<li>" " path is back at normal</li>
</ul>
@return tuple of flag indicating successful operation (boolean) and
a status message in case of non successful operation (QString)
"""
raise RuntimeError('Not implemented')
