class QtStreamHandler(logging.Handler): def __init__(self, parent, main): logging.Handler.__init__(self) self.parent = parent self.main = main self.textWidget = parent self.formater = logging.Formatter("%(asctime)s - %(levelname)s -> %(message)s") self.logString = "" def getLoggingCompleteText(self): return self.logString def createLock(self): self.mutex = QMutex() def acquire(self): self.mutex.lock() def release(self): self.mutex.unlock() def emit(self, record): # self.textWidget.appendPlainText(self.formater.format(record)) formattedText = self.formater.format(record) #separate text from date index = formattedText.find('->') self.textWidget.setTextColor(QColor("green")) self.textWidget.insertPlainText(formattedText[:index]) self.textWidget.setTextColor(QColor("blue")) self.textWidget.insertPlainText(formattedText[index:] + "\n") self.textWidget.moveCursor(QTextCursor.EndOfLine) self.textWidget.ensureCursorVisible() self.logString += formattedText + "\n";
class MainW(QtGui.QMainWindow): """ Mainw to host and run one single thread """ def __init__(self): QtGui.QMainWindow.__init__(self) self.mutex = QMutex() self.thr = GuiThread(process=self.fun, period=0.1) self.thr2 = GuiThread(process=self.fun2, period=0.1) self.buff = CircIoBuffer(byte_size=15 * len('rafal miecznik')) self.thr.start() self.thr2.start() time.sleep(2) print(self.buff.read()) for d in dir(self.mutex): print(d) def fun(self): print(self.mutex.tryLock()) self.buff.write(' rafal') self.mutex.unlock() #print self.thr.currentThreadId() def fun2(self): self.mutex.lock() self.buff.write(' miecznik')
class Feed(QRunnable): def __init__(self, label, parent=None): QRunnable.__init__(self) self.label = label self.parent = parent self.data = None self.mutex = QMutex() self.setAutoDelete(True) def run(self): self.check() def check(self): try: self.fetch() except (urllib2.URLError, urllib2.HTTPError) as err: self.parent.error.emit(str(err)) else: self.read() if self.data: self.parse() def fetch(self): auth_handler = urllib2.HTTPBasicAuthHandler() auth_handler.add_password(REALM, HOST, self.parent.user, self.parent.passwd) opener = urllib2.build_opener(auth_handler) urllib2.install_opener(opener) url = "%s/%s" % (URL_ATOM, self.label) if self.label else URL_ATOM self.conn = urllib2.urlopen(url) def read(self): code = self.conn.getcode() if code != 200: self.conn.close() self.parent.error.emit( "HTTP Error %d: %s" % (code, BaseHTTPRequestHandler.responses[code])) else: self.data = self.conn.read() self.conn.close() def parse(self): tree = et.fromstring(self.data) for e in tree.findall("%sentry" % XMLNS): entry = Entry() entry.title = e.find("%stitle" % XMLNS).text entry.summary = e.find("%ssummary" % XMLNS).text entry.link = e.find("%slink" % XMLNS).attrib["href"] entry.modified = e.find("%smodified" % XMLNS).text entry.issued = e.find("%sissued" % XMLNS).text entry.id = e.find("%sid" % XMLNS).text for a in e.findall("%sauthor" % XMLNS): entry.author_name = a.find("%sname" % XMLNS).text entry.author_email = a.find("%semail" % XMLNS).text self.mutex.lock() self.parent.entries.append((self.label, entry)) self.mutex.unlock()
class Feed(QRunnable): def __init__(self, label, parent=None): QRunnable.__init__(self) self.label = label self.parent = parent self.data = None self.mutex = QMutex() self.setAutoDelete(True) def run(self): self.check() def check(self): try: self.fetch() except (urllib2.URLError, urllib2.HTTPError) as err: self.parent.error.emit(str(err)) else: self.read() if self.data: self.parse() def fetch(self): auth_handler = urllib2.HTTPBasicAuthHandler() auth_handler.add_password(REALM, HOST, self.parent.user, self.parent.passwd) opener = urllib2.build_opener(auth_handler) urllib2.install_opener(opener) url = "%s/%s" % (URL_ATOM, self.label) if self.label else URL_ATOM self.conn = urllib2.urlopen(url) def read(self): code = self.conn.getcode() if code != 200: self.conn.close() self.parent.error.emit("HTTP Error %d: %s" % ( code, BaseHTTPRequestHandler.responses[code])) else: self.data = self.conn.read() self.conn.close() def parse(self): tree = et.fromstring(self.data) for e in tree.findall("%sentry" % XMLNS): entry = Entry() entry.title = e.find("%stitle" % XMLNS).text entry.summary = e.find("%ssummary" % XMLNS).text entry.link = e.find("%slink" % XMLNS).attrib["href"] entry.modified = e.find("%smodified" % XMLNS).text entry.issued = e.find("%sissued" % XMLNS).text entry.id = e.find("%sid" % XMLNS).text for a in e.findall("%sauthor" % XMLNS): entry.author_name = a.find("%sname" % XMLNS).text entry.author_email = a.find("%semail" % XMLNS).text self.mutex.lock() self.parent.entries.append((self.label, entry)) self.mutex.unlock()
class QCallbacksManager(QObject): class Request(object): def __init__(self): self.event = Event() self.answer = None def __call__(self): raise NotImplementedError() class LoginRequest(Request): def __init__(self, backend_name, value): QCallbacksManager.Request.__init__(self) self.backend_name = backend_name self.value = value def __call__(self): password, ok = QInputDialog.getText( None, '%s request' % self.value.label, 'Please enter %s for %s' % (self.value.label, self.backend_name), QLineEdit.Password) return password new_request = Signal() def __init__(self, weboob, parent=None): QObject.__init__(self, parent) self.weboob = weboob self.weboob.requests.register('login', self.callback(self.LoginRequest)) self.mutex = QMutex() self.requests = [] self.new_request.connect(self.do_request) def callback(self, klass): def cb(*args, **kwargs): return self.add_request(klass(*args, **kwargs)) return cb @Slot() def do_request(self): self.mutex.lock() request = self.requests.pop() request.answer = request() request.event.set() self.mutex.unlock() def add_request(self, request): self.mutex.lock() self.requests.append(request) self.mutex.unlock() self.new_request.emit() request.event.wait() return request.answer
class QCallbacksManager(QObject): class Request(object): def __init__(self): self.event = Event() self.answer = None def __call__(self): raise NotImplementedError() class LoginRequest(Request): def __init__(self, backend_name, value): QCallbacksManager.Request.__init__(self) self.backend_name = backend_name self.value = value def __call__(self): password, ok = QInputDialog.getText(None, '%s request' % self.value.label, 'Please enter %s for %s' % (self.value.label, self.backend_name), QLineEdit.Password) return password new_request = Signal() def __init__(self, weboob, parent=None): QObject.__init__(self, parent) self.weboob = weboob self.weboob.callbacks['login'] = self.callback(self.LoginRequest) self.mutex = QMutex() self.requests = [] self.new_request.connect(self.do_request) def callback(self, klass): def cb(*args, **kwargs): return self.add_request(klass(*args, **kwargs)) return cb @Slot() def do_request(self): self.mutex.lock() request = self.requests.pop() request.answer = request() request.event.set() self.mutex.unlock() def add_request(self, request): self.mutex.lock() self.requests.append(request) self.mutex.unlock() self.new_request.emit() request.event.wait() return request.answer
class ImporterThread(QThread): def __init__(self, framework, parent=None): QThread.__init__(self, parent) self.framework = framework self.qlock = QMutex() self.cursor = None QObject.connect(self, SIGNAL('quit()'), self.quitHandler) QObject.connect(self, SIGNAL('started()'), self.startedHandler) def run(self): QObject.connect(self, SIGNAL('do_runImport()'), self.handle_runImport, Qt.DirectConnection) self.exec_() def quitHandler(self): if self.cursor: self.cursor.close() self.exit(0) def startedHandler(self): pass def runImport(self, importers, proxy_file, source, callback): self.importers = importers self.proxy_filelist = [proxy_file] self.source = source self.callbackObj = callback QTimer.singleShot(50, self, SIGNAL('do_runImport()')) def runImportList(self, importers, proxy_filelist, source, callback): self.importers = importers self.proxy_filelist = proxy_filelist self.source = source self.callbackObj = callback QTimer.singleShot(50, self, SIGNAL('do_runImport()')) def handle_runImport(self): if self.qlock.tryLock(): try: for proxy_file in self.proxy_filelist: try: self.framework.debug_log('attempting import of %s' % (str(proxy_file))) self.importers.process_import(str(proxy_file), self.framework, self.source) except Exception as ex: self.framework.report_exception(ex) finally: self.qlock.unlock() self.callbackObj.emit(SIGNAL('runImportFinished()'))
class MyStreamHandler(StreamHandler): def __init__(self): StreamHandler.__init__(self) def createLock(self): # must be Recursive (= reentrant) self._mutex = QMutex(QMutex.Recursive) def acquire(self): self._mutex.lock() def release(self): self._mutex.unlock()
class Lock(object): def __init__(self): self._mutex = QMutex() def __enter__(self): self._mutex.lock() def __exit__(self, type, value, tb): self._mutex.unlock() def acquire(self): self._mutex.lock() def release(self): self._mutex.unlock()
class ImporterThread(QThread): def __init__(self, framework, parent = None): QThread.__init__(self, parent) self.framework = framework self.qlock = QMutex() self.cursor = None QObject.connect(self, SIGNAL('quit()'), self.quitHandler) QObject.connect(self, SIGNAL('started()'), self.startedHandler) def run(self): QObject.connect(self, SIGNAL('do_runImport()'), self.handle_runImport, Qt.DirectConnection) self.exec_() def quitHandler(self): if self.cursor: self.cursor.close() self.exit(0) def startedHandler(self): pass def runImport(self, importers, proxy_file, source, callback): self.importers = importers self.proxy_filelist = [proxy_file] self.source = source self.callbackObj = callback QTimer.singleShot(50, self, SIGNAL('do_runImport()')) def runImportList(self, importers, proxy_filelist, source, callback): self.importers = importers self.proxy_filelist = proxy_filelist self.source = source self.callbackObj = callback QTimer.singleShot(50, self, SIGNAL('do_runImport()')) def handle_runImport(self): if self.qlock.tryLock(): try: for proxy_file in self.proxy_filelist: try: self.framework.debug_log('attempting import of %s' % (str(proxy_file))) self.importers.process_import(str(proxy_file), self.framework, self.source) except Exception as ex: self.framework.report_exception(ex) finally: self.qlock.unlock() self.callbackObj.emit(SIGNAL('runImportFinished()'))
class Evaluate(QThread): """Thread used to insolate calculation process in entities (stream, project and equipment, so gui can response while calculation is in process""" def __init__(self, parent=None): super(Evaluate, self).__init__(parent) self.mutex = QMutex() def start(self, entity, kwargs): self.entity = entity self.kwargs = kwargs QThread.start(self) def run(self): self.mutex.lock() self.entity(**self.kwargs) self.mutex.unlock()
class UdpClient(QThread): def __init__(self, parent): QThread.__init__(self, parent) self.prnt = parent self.udp = QUdpSocket() addr = QHostAddress(QHostAddress.Any) print 'bind to:', addr.toString() self.udp.bind(addr, 34001) self.udp.error.connect(self.errorAnalyser) self.udp.readyRead.connect(self.readUdp) print "Binding..." self.STOP = False self.locker = QMutex() def run(self): self.prnt.initAvahiService() while True : if self.udp is not None and self.udp.state() == QAbstractSocket.ConnectedState : self.udp.waitForReadyRead() else : self.msleep(100) if self.STOP and self.udp is not None: self.udp.close(); break print 'UDPClient closed...' self.prnt.changeConnectState.emit() def stop(self): self.locker.lock() self.STOP = True self.locker.unlock() def readUdp(self): while ( self.udp.hasPendingDatagrams() ): data = QByteArray() addr = QHostAddress() port = 0 try : datagramSize = self.udp.pendingDatagramSize() if datagramSize > 0 : (data, addr, port) = self.udp.readDatagram(datagramSize) #print "Datagram: [%s] from %s:%i" % (QString().fromUtf8(data), addr.toString(), port) self.prnt.contactMessage.emit(QString().fromUtf8(data), addr.toString()) except socket.error, err : print '[in readUdp() UdpClient] SocketError1 : ', err except socket.timeout, err : print '[in readUdp() UdpClient] SocketError2 : ', err except :
class ComputationThread(QThread): def __init__(self, animation_thread): QThread.__init__(self) self.is_running = True self.mutex = QMutex() self.other_thread = animation_thread def stop(self): self.is_running = False def run(self): while self.is_running: self.mutex.lock() # locking may not be necessary raw_input("") #self.other_thread.stop( ) self.mutex.unlock() print " run method ends "
class SQLRenderThread(CachedRenderThread): """ RenderThread extending the UniqueMethodRenderThread by supplying a cancel method for classes with database access. The database object needs to be given either on the object as .db subobject or explicitly by setObjectDBObject(). The database object needs to have a 'connection' attribute which supports an interrupt() method. """ def __init__(self, parent=0): self.dbObjectLock = QMutex(QMutex.Recursive) self.dbObject = {} CachedRenderThread.__init__(self, parent) def setObject(self, classObject, *args, **param): self.dbObjectLock.lock() if classObject in self.dbObject: del self.dbObject[classObject] self.dbObjectLock.unlock() CachedRenderThread.setObject(self, classObject, *args, **param) def setObjectDBObject(self, classObject, db): self.dbObjectLock.lock() self.dbObject[classObject] = db self.dbObjectLock.unlock() def cancelCurrentJob(self): QMutexLocker(self.renderingLock) if self.currentlyRenderingJob: _, classObject, _, _, _ = self.currentlyRenderingJob try: QMutexLocker(self.dbObjectLock) if classObject in self.dbObject: db = self.dbObject[classObject] else: classObjectInst = self.getObjectInstance(classObject) if hasattr(classObjectInst, 'db'): db = classObjectInst.db else: return False self.clearCurrentJob() db.connection.interrupt() return True except KeyError: return False
class ImageTile(object): def __init__(self, rect): self._mutex = QMutex() self.image = QImage(rect.width(), rect.height(), QImage.Format_ARGB32_Premultiplied) self.image.fill(0x00000000) self._topLeft = rect.topLeft() # Whenever the underlying data changes, the data version is incremented. # By comparing the data version to the image and request version, it can # be determined if the content of this tile is recent or needs to be # re-computed. # version of the data self.dataVer = 0 # version of self.image # # If self.imgVer < self.dataVer, the image needs to be re-computed # from the new data. self.imgVer = -1 # version of the request that has been generated to update the contents # of self.image # # If self.reqVer == self.dataVer, a request is currently running that will # eventually replace self.image with the new data. self.reqVer = -2 def clear(self): self.image.fill(0) def paint(self, painter): self.lock() painter.drawImage(self._topLeft, self.image) self.unlock() def lock(self): self._mutex.lock() def unlock(self): self._mutex.unlock()
class ImageTile(object): def __init__(self, rect): self._mutex = QMutex() self.image = QImage(rect.width(), rect.height(), QImage.Format_ARGB32_Premultiplied) self.image.fill(0) self._topLeft = rect.topLeft() #Whenever the underlying data changes, the data version is incremented. #By comparing the data version to the image and request version, it can #be determined if the content of this tile is recent or needs to be #re-computed. #version of the data self.dataVer = 0 #version of self.image # #If self.imgVer < self.dataVer, the image needs to be re-computed #from the new data. self.imgVer = -1 #version of the request that has been generated to update the contents #of self.image # #If self.reqVer == self.dataVer, a request is currently running that will #eventually replace self.image with the new data. self.reqVer = -2 def clear(self): self.image.fill(0) def paint(self, painter): self.lock() painter.drawImage(self._topLeft, self.image) self.unlock() def lock(self): self._mutex.lock() def unlock(self): self._mutex.unlock()
class DatabaseThread(QThread): def __init__(self, framework, Data, parent=None): QThread.__init__(self, parent) self.framework = framework self.Data = Data self.qlock = QMutex() QObject.connect(self, SIGNAL('quit()'), self.quitHandler) QObject.connect(self, SIGNAL('started()'), self.startedHandler) def run(self): QObject.connect(self, SIGNAL('doConnectDb()'), self.connectDbHandler, Qt.DirectConnection) self.exec_() def close(self): self.qlock.lock() try: self.Data.close() finally: self.qlock.unlock() def quitHandler(self): self.Data.close() self.exit(0) def startedHandler(self): pass def connectDb(self, filename, callback): self.filename = filename self.callbackObj = callback QTimer.singleShot(50, self, SIGNAL('doConnectDb()')) def connectDbHandler(self): self.qlock.lock() try: self.Data.connect(self.filename) finally: self.qlock.unlock() self.callbackObj.emit(SIGNAL('connectDbFinished()'))
class AnimationThread(QThread): def __init__(self): QThread.__init__(self) self.is_running = True self.mutex = QMutex() def stop(self): self.is_running = False def get_state(self): return self.is_running def change_state(self): self.is_running = not self.is_running def run(self): while self.is_running: self.mutex.lock( ) # locking may not be necessary self.emit(SIGNAL("window_update_request" )) self.mutex.unlock() sleep( 0.005 )
class _LoggingMutexQt(_LoggingMutex): def __init__(self, name): super(_LoggingMutexQt, self).__init__(name) from PyQt4.QtCore import QMutex self.mutex = QMutex() def currentThreadID(self): from PyQt4.QtCore import QThread return QThread.currentThreadId() def _acquire(self): self.mutex.lock() def _release(self): self.mutex.unlock() def enterMutex(self): self.mutex.lock() def exitMutex(self, *_args, **_kwargs): self.mutex.unlock()
class communications: def __init__(self): self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.s.connect((TCP_IP, TCP_PORT)) self.running = QMutex() self.send_command = '.' def command(self, comm): self.running.lock() self.send_command = comm self.running.unlock() def get_data(self): self.running.lock() self.s.send(self.send_command.encode()) values = pickle.loads(self.s.recv(BUFFER_SIZE)) self.send_command = '.' self.running.unlock() return values def get_parameters(self): self.running.lock() self.s.send(b"?") params = pickle.loads(self.s.recv(BUFFER_SIZE)) self.running.unlock() return params def __del__(self): self.s.close()
class DatabaseThread(QThread): def __init__(self, framework, Data, parent = None): QThread.__init__(self, parent) self.framework = framework self.Data = Data self.qlock = QMutex() QObject.connect(self, SIGNAL('quit()'), self.quitHandler) QObject.connect(self, SIGNAL('started()'), self.startedHandler) def run(self): QObject.connect(self, SIGNAL('doConnectDb()'), self.connectDbHandler, Qt.DirectConnection) self.exec_() def close(self): self.qlock.lock() try: self.Data.close() finally: self.qlock.unlock() def quitHandler(self): self.Data.close() self.exit(0) def startedHandler(self): pass def connectDb(self, filename, callback): self.filename = filename self.callbackObj = callback QTimer.singleShot(50, self, SIGNAL('doConnectDb()')) def connectDbHandler(self): self.qlock.lock() try: self.Data.connect(self.filename) finally: self.qlock.unlock() self.callbackObj.emit(SIGNAL('connectDbFinished()'))
class SignalThread(QThread): def __init__(self, parent=None): QThread.__init__(self, parent) self.waitcond = QWaitCondition() self.mutex = QMutex() self.isstopped = False def trigger(self): """lock first to make sure the QThread is actually waiting for a signal""" self.mutex.lock() self.waitcond.wakeOne() self.mutex.unlock() def stopThread(self): self.mutex.lock() self.isstopped = True self.waitcond.wakeOne() self.mutex.unlock() def run(self): self.mutex.lock() while not self.isstopped: # just wait, and trigger every time we receive a signal self.waitcond.wait(self.mutex) if not self.isstopped: self.emit(SIGNAL("triggerSignal()")) self.mutex.unlock()
class SignalThread(QThread): def __init__(self, parent = None): QThread.__init__(self, parent) self.waitcond = QWaitCondition() self.mutex = QMutex() self.isstopped = False def trigger(self): """lock first to make sure the QThread is actually waiting for a signal""" self.mutex.lock() self.waitcond.wakeOne() self.mutex.unlock() def stopThread(self): self.mutex.lock() self.isstopped = True self.waitcond.wakeOne() self.mutex.unlock() def run(self): self.mutex.lock() while not self.isstopped: # just wait, and trigger every time we receive a signal self.waitcond.wait(self.mutex) if not self.isstopped: self.emit(SIGNAL("triggerSignal()")) self.mutex.unlock()
class Imap(QRunnable): def __init__(self, label, parent=None): QRunnable.__init__(self) self.imap = None self.label = label self.parent = parent self.mutex = QMutex() self.login() def run(self): for imap_id in self.get_unread(): thread_id = self.get_thread_id(imap_id) self.mutex.lock() self.parent.idmap[thread_id] = imap_id self.mutex.unlock() def connect(self): self.imap = imaplib.IMAP4_SSL(IMAP_SERVER, IMAP_PORT) def login(self): try: if not self.imap: self.connect() auth = self.imap.login(self.parent.user, self.parent.passwd) self.logged_in = (auth and auth[0] == "OK") except imaplib.IMAP4.error: pass def logout(self): pass def get_unread(self): self.imap.select(self.label.replace("-", "/")) result, unseen = self.imap.search(None, "(UNSEEN)") return unseen[0].split() def get_thread_id(self, id): result, data = self.imap.fetch(id, "(X-GM-THRID BODY.PEEK[HEADER])") return data[0][0].split(" ")[2]
class WorkerThread(QThread): do_something = pyqtSignal() def __init__(self, name="", parent=None): super().__init__(parent) self.sleep_interval = 900 self.mutex = QMutex() self.name = name self.isStopped = False def run(self): while not self.isStopped: # while True: self.mutex.lock() self.do_something.emit() self.mutex.unlock() self.msleep(self.sleep_interval) self.finished.emit() print(self.name + " finished.") def stop(self): with QMutexLocker(self.mutex): self.isStopped = True
class VCSPluginThread( QThread ): " Wrapper for the plugin thread " def __init__( self, plugin, parent = None ): QThread.__init__( self, parent ) self.__plugin = plugin self.__requestQueue = deque() self.__stopRequest = False self.__lock = QMutex() self.__condition = QWaitCondition() return def run( self ): " Thread loop " while not self.__stopRequest: self.__lock.lock() while self.__requestQueue: path, flag = self.__requestQueue.pop() self.__lock.unlock() time.sleep( 0.01 ) self.__processRequest( path, flag ) if self.__stopRequest: break self.__lock.lock() if self.__stopRequest: self.__lock.unlock() break self.__condition.wait( self.__lock ) self.__lock.unlock() return def __processRequest( self, path, flag ): " Processes a single request. It must be exception safe. " try: statuses = self.__plugin.getObject().getStatus( path, flag ) for status in statuses: if len( status ) == 3: self.emit( SIGNAL( "VCSStatus" ), path + status[ 0 ], status[ 1 ], status[ 2 ] ) else: self.emit( SIGNAL( "VCSStatus" ), path, IND_VCS_ERROR, "The " + self.__plugin.getName() + " plugin " "does not follow the getStatus() interface " "agreement" ) except Exception, exc: self.emit( SIGNAL( "VCSStatus" ), path, IND_VCS_ERROR, "Exception in " + self.__plugin.getName() + " plugin while retrieving VCS status: " + str( exc ) ) except:
class ThreadSerial(QThread): dataUpdate = pyqtSignal(str) def __init__(self, envs): super(ThreadSerial, self).__init__() self.serial = envs.get('SERIAL') self.mutex = QMutex() self.exit = False def run(self): debug("[+] Listening {}".format(self.serial.port)) while not self.exit: if self.mutex.tryLock(100): try: data = self.serial.read_all() finally: self.mutex.unlock() if data: self.dataUpdate.emit(data) time.sleep(0.001) def pause(self, timeout=1000): return self.mutex.tryLock(timeout) def resume(self): self.mutex.unlock() def read_no_wait(self): data = self.serial.read_all() self.dataUpdate.emit(data) return data def write_no_wait(self, data): self.serial.write(unicode(data).encode('utf-8')) def write(self, data): if self.mutex.tryLock(100): try: self.serial.write(unicode(data).encode('utf-8')) finally: self.mutex.unlock() def terminate(self, timeout=1000): debug("[-] Terminate {}".format(self.serial.port)) self.exit = True if not self.wait(timeout): QThread.terminate(self) self.serial.close()
class TypeWorker(QThread): def __init__(self, *args): QThread.__init__(self) self.typeQueue = Queue() self.regImage = re.compile("(JPEG|JPG|jpg|jpeg|GIF|gif|bmp|BMP|png|PNG|pbm|PBM|pgm|PGM|ppm|PPM|xpm|XPM|xbm|XBM|TIFF|tiff).*", re.IGNORECASE) self.typeQueue = [] self.setUniq = set() self.qmutex = QMutex() self.qsem = QSemaphore() def enqueue(self, parent, index, node): self.qmutex.lock() if long(node.this) not in self.setUniq: self.typeQueue.insert(0, (parent, index, node)) self.setUniq.add(long(node.this)) self.qsem.release() self.qmutex.unlock() def clear(self): self.qmutex.lock() self.typeQueue = [] self.setUniq.clear() self.qsem.acquire(self.qsem.available()) self.qmutex.unlock() def get(self): self.qsem.acquire() self.qmutex.lock() res = self.typeQueue.pop() self.setUniq.remove(long(res[2].this)) self.qmutex.unlock() return res def isImage(self, ftype): res = self.regImage.search(ftype) return res def run(self): count = 0 while True: (parent, index, node) = self.get() if node.size(): ftype = str(node.dataType()) if parent.imagesthumbnails and self.isImage(ftype): thumb = ImageThumb() img = thumb.getImage(ftype, node, index) if img: parent.emit(SIGNAL('dataImage'), index, node, img)
class ExtractThread(QThread): def __init__(self, files, needPrj): QThread.__init__(self, QThread.currentThread()) self.inFiles = files self.needPrj = needPrj self.mutex = QMutex() self.stopMe = 0 def run(self): self.mutex.lock() self.stopMe = 0 self.mutex.unlock() interrupted = False for f in self.inFiles: extractProjection(f, self.needPrj) self.emit(SIGNAL("fileProcessed()")) self.mutex.lock() s = self.stopMe self.mutex.unlock() if s == 1: interrupted = True break if not interrupted: self.emit(SIGNAL("processFinished()")) else: self.emit(SIGNAL("processIterrupted()")) def stop(self): self.mutex.lock() self.stopMe = 1 self.mutex.unlock() QThread.wait(self)
class SpatialIdxThread( QThread ): def __init__( self, layers, isFiles ): QThread.__init__( self, QThread.currentThread() ) self.layers = layers self.isFiles = isFiles self.mutex = QMutex() self.stopMe = 0 self.errors = [] def run( self ): self.mutex.lock() self.stopMe = 0 self.mutex.unlock() interrupted = False if self.isFiles: for layer in self.layers: vl = QgsVectorLayer( layer, "tmp", "ogr" ) provider = vl.dataProvider() if provider.capabilities() & QgsVectorDataProvider.CreateSpatialIndex: if not provider.createSpatialIndex(): self.errors.append( layer ) else: self.errors.append( layer ) self.emit( SIGNAL( "layerProcessed()" ) ) self.mutex.lock() s = self.stopMe self.mutex.unlock() if s == 1: interrupted = True break else: for layer in self.layers: vl = ftools_utils.getVectorLayerByName( layer ) provider = vl.dataProvider() if provider.capabilities() & QgsVectorDataProvider.CreateSpatialIndex: if not provider.createSpatialIndex(): self.errors.append( layer ) else: self.errors.append( layer ) self.emit( SIGNAL( "layerProcessed()" ) ) self.mutex.lock() s = self.stopMe self.mutex.unlock() if s == 1: interrupted = True break if not interrupted: self.emit( SIGNAL( "processFinished( PyQt_PyObject )" ), self.errors ) else: self.emit( SIGNAL( "processInterrupted()" ) ) def stop( self ): self.mutex.lock() self.stopMe = 1 self.mutex.unlock() QThread.wait( self )
class AnalyzerThread(QThread): def __init__(self, framework, parent=None): QThread.__init__(self, parent) self.framework = framework self.callback_object = None self.qtimer = QTimer() self.qlock = QMutex() QObject.connect(self, SIGNAL('quit()'), self.handle_quit) QObject.connect(self, SIGNAL('started()'), self.handle_started) self.Data = None self.read_cursor = None self.cursor = None def db_attach(self): self.Data = self.framework.getDB() self.read_cursor = self.Data.allocate_thread_cursor() self.cursor = self.Data.allocate_thread_cursor() def db_detach(self): self.close_cursor() self.Data = None def close_cursor(self): if self.read_cursor: self.read_cursor.close() self.Data.release_thread_cursor(self.read_cursor) self.read_cursor = None if self.cursor: self.cursor.close() self.Data.release_thread_cursor(self.cursor) self.cursor = None def run(self): QObject.connect(self, SIGNAL('do_runAnalysis()'), self.handle_runAnalysis, Qt.DirectConnection) self.exec_() def runAnalysis(self, callback_object): self.callback_object = callback_object QTimer.singleShot(50, self, SIGNAL('do_runAnalysis()')) def handle_quit(self): self.framework.debug_log('AnalyzerThread quit...') self.close_cursor() self.exit(0) def handle_started(self): self.framework.debug_log('AnalyzerThread started...') self.framework.subscribe_database_events(self.db_attach, self.db_detach) def handle_runAnalysis(self): fullanalysistext = '' if not self.qlock.tryLock(): self.framework.debug_log('failed to acquire lock for analyzers') else: try: fullanalysistext = self.analyze_content() except Exception as e: self.framework.report_exception(e) finally: self.qlock.unlock() if self.callback_object: self.callback_object.emit(SIGNAL('runAnalysisFinished(QString)'), str(fullanalysistext)) def result_type_to_string(self, resulttype): """Reads type and package information from a result, and returns it as a string""" resultclass = resulttype.__class__ return "".join((resultclass.__module__, ".", resultclass.__name__)) def analyze_content(self): """ Perform analysis on the captured content""" #TODO: NEW DB THREAD TO HOLD RESPONSES, ANOTHER FOR WRITING RESULTS scopeController = self.framework.getScopeController() response = self.Data.read_all_responses(self.read_cursor) response_IDs = [] for row in response: dbrow = [m or '' for m in row] Id = dbrow[ResponsesTable.ID] url = dbrow[ResponsesTable.URL] if scopeController.isUrlInScope(url, url): response_IDs.append(Id) #Instantiate all found analyzers analyzerobjects = AnalyzerList(self.framework) analyzerobjects.instantiate_analyzers() analysisrunid = self.Data.analysis_start(self.cursor) #TODO - Consider threading from here down for x in analyzerobjects: #dbconfig=self.Data.get_config_value(self.read_cursor, 'ANALYSIS', str(x.__class__)) #print "dbconfig=%s"%dbconfig print("class=%s" % x.__class__) #x.setConfiguration(dbconfig) x.preanalysis() x.initResultsData() resultinstance = x.getResults() x.analyzerinstanceid = self.Data.analysis_add_analyzer_instance( self.cursor, analysisrunid, str(x.__class__).translate('<>'), x.friendlyname, x.desc, self.result_type_to_string(resultinstance)) fullanalysistext = StringIO() for Id in response_IDs: transaction = self.framework.get_request_response(Id) for analyzer in analyzerobjects: try: analyzer.analyzeTransaction(transaction, analyzer.getResults()) tempanalysisresults = analyzer.getResults() #If there were results for this page, add them to the DB if transaction.Id in tempanalysisresults.pages: pageresultset = self.Data.analysis_add_resultset( self.cursor, analyzer.analyzerinstanceid, transaction.Id, False, transaction.responseUrl, self.result_type_to_string( tempanalysisresults.pages[transaction.Id])) for result in tempanalysisresults.pages[ transaction.Id].results: self.Data.analysis_add_singleresult( self.cursor, pageresultset, result.severity, result.certainty, result.type, result.desc, #TODO: Consider db structure to handle data field str(result.data), result.span, self.result_type_to_string(result)) for key, value in list(tempanalysisresults.pages[ transaction.Id].stats.items()): self.Data.analysis_add_stat( self.cursor, pageresultset, key, value) except Exception as e: # TODO: add real debugging support self.framework.debug_log('Transaction ID: ' + str(transaction.Id)) self.framework.report_exception(e) #Post Analysis for analyzer in analyzerobjects: results = analyzer.getResults() analyzer.postanalysis(results) for context in list(results.overall.keys()): overallresultset = self.Data.analysis_add_resultset( self.cursor, analyzer.analyzerinstanceid, None, True, context, self.result_type_to_string(results.overall[context])) for result in results.overall[context].results: self.Data.analysis_add_singleresult( self.cursor, overallresultset, result.severity, result.certainty, result.type, result.desc, #TODO: Consider db structure to handle data field str(result.data), result.span, self.result_type_to_string(result)) #print "WRITING:",self.result_type_to_string(result) for key, value in list(results.overall[context].stats.items()): self.Data.analysis_add_stat(self.cursor, overallresultset, key, value) self.Data.commit() #Output results to analysis tab #for analyzer in analyzerobjects: #fullanalysistext.write(analyzer.getResults().toHTML()) return ''
class MessageSender: """ static fileds: context: to keep track of msg req<->resp mechanism msg context is attached. Thanks to this value it is possible to find right response for given request. There are context ids reserved for particular messages handling: 0: free text which should be displayed in console window 1: digidiag frame data Message sender cant assign reserved_context id when message is created and sent. """ reserved_context = ( 0, 1, ) context = Uint16( len(reserved_context)) #start with value greater than reserved_context lock = False class ID: """ Message IDs reperesenting given procedure in EMU BT board """ txt_message = 0 write_to_page = 1 rxflush = 2 bootloader = 11 disable_btlrd = 12 reset = 14 run_main_app_btl = 254 @classmethod def translate_id(cls, m_id): try: return [ p[0] for p in MessageSender.ID.__dict__.items() if p[1] == m_id ][0] except IndexError: return None def __init__(self, tx_interface): self.mutex = QMutex() self.__transmit = tx_interface def __send_m(self, msg, m_id): """ Send createad message. Avoid using recovered context_ids """ context = MessageSender.context MessageSender.context += 1 while MessageSender.context in MessageSender.reserved_context: MessageSender.context += 1 translated_m_id = MessageSender.ID.translate_id(m_id) m_logger.debug("Sent message with context: {}, id: {}({})".format( context, translated_m_id, m_id)) m_logger.debug(msg[11:30]) self.__transmit(msg) return context def peek_context(self): return MessageSender.context + 1 def send(self, m_id, body=b'NULL'): """ Polymorphic method for send """ return self.__send(m_id, body) def send_raw_msg(self, body): """ Polymorphic method for send """ return self.__send(m_id=None, body=body) def __send(self, m_id=None, body='NULL'): msg = create_message( msg_id=m_id, body=body, context=MessageSender.context) if m_id is not None else body self.mutex.lock() context = self.__send_m(msg, m_id) self.mutex.unlock() m_logger.debug("message sending unlocked") return context
class SiteMapThread(QThread): def __init__(self, framework, treeViewModel, parent=None): QThread.__init__(self, parent) self.framework = framework self.treeViewModel = treeViewModel self.qtimer = QTimer() self.qlock = QMutex() self.fillAll = False QObject.connect(self, SIGNAL('quit()'), self.quitHandler) QObject.connect(self, SIGNAL('started()'), self.startedHandler) self.re_set_cookie = re.compile(r'^Set-Cookie2?:\s*(.+)$', re.I | re.M) self.re_host_name = re.compile(r'^Host:\s*(.+?)$', re.I | re.M) self.lastId = 0 self.Data = None self.cursor = None def db_attach(self): self.Data = self.framework.getDB() self.cursor = self.Data.allocate_thread_cursor() self.populateSiteMap(True) def db_detach(self): self.close_cursor() self.Data = None def close_cursor(self): if self.cursor and self.Data: self.cursor.close() self.Data.release_thread_cursor(self.cursor) self.cursor = None def run(self): QObject.connect(self, SIGNAL('updateSiteMap()'), self.doUpdateSiteMap, Qt.DirectConnection) self.framework.subscribe_response_data_added(self.doUpdateSiteMap) self.exec_() def quitHandler(self): self.framework.debug_log('SiteMapThread quit...') self.close_cursor() self.exit(0) def startedHandler(self): self.framework.debug_log('SiteMapThread started...') self.framework.subscribe_database_events(self.db_attach, self.db_detach) def populateSiteMap(self, fillAll): self.fillAll = fillAll QTimer.singleShot(50, self, SIGNAL('updateSiteMap()')) def doUpdateSiteMap(self): if not self.qlock.tryLock(): return try: if self.fillAll: self.fillAll = False self.treeViewModel.clearModel() self.lastId = 0 rows = self.Data.get_sitemap_info(self.cursor, self.lastId) global_cookie_jar = self.framework.get_global_cookie_jar() count = 0 for row in rows: count += 1 if 0 == (count % 100): self.yieldCurrentThread() rowItems = [m or '' for m in list(row)] Id = str(rowItems[0]) try: self.lastId = int(Id) except ValueError: pass # XXX: review all for bytes usage if isinstance(rowItems[1], bytes): url = str(rowItems[1], 'utf-8', 'ignore') else: url = str(rowItems[1]) status = str(rowItems[2]) response_headers = str(rowItems[3]) request_headers = str(rowItems[4]) # TODO: make configurable if status in ('400', '404', '500', '501'): continue # TODO: m = self.re_set_cookie.search(response_headers) if m: setCookies = m.group(1) cookieList = QNetworkCookie.parseCookies(setCookies) global_cookie_jar.setCookiesFromUrl( cookieList, QUrl.fromEncoded(url)) parsed = urlparse.urlsplit(url) hostname = '' if not parsed.hostname: m = self.re_host_name.search(request_headers) if m: hostname = m.group(1).rstrip() else: hostname = parsed.hostname hostname = hostname.lower() hostloc = urlparse.urlunsplit( (parsed.scheme, parsed.netloc, '/', '', '')) rootNode = self.treeViewModel.findOrAddNode(hostname) hostLocNode = rootNode.findOrAddNode(self.treeViewModel, hostloc) pathval = parsed.path # add directories parentNode = hostLocNode parentNode.setResponseId(None, hostloc) lastSlash = 0 slash = 0 while True: slash = pathval.find('/', slash + 1) if slash < 0: break dirname = pathval[lastSlash + 1:slash + 1] parentNode = parentNode.findOrAddNode( self.treeViewModel, dirname) parentNode.setResponseId( None, urlparse.urlunsplit((parsed.scheme, parsed.netloc, pathval[0:slash + 1], '', ''))) lastSlash = slash # add file element if lastSlash + 1 < len(pathval): filename = pathval[lastSlash + 1:] parentNode = parentNode.findOrAddNode( self.treeViewModel, filename) parentNode.setResponseId( None, urlparse.urlunsplit( (parsed.scheme, parsed.netloc, pathval, '', ''))) # add query if parsed.query: parentNode = parentNode.findOrAddNode( self.treeViewModel, '?' + parsed.query) # store the latest Id # TODO: should determine best candidate to display parentNode.setResponseId(Id, url) finally: self.qlock.unlock()
class SplitThread(QThread): def __init__(self, layer, splitField, encoding, outDir): QThread.__init__(self, QThread.currentThread()) self.layer = layer self.field = splitField self.encoding = encoding self.outDir = outDir self.mutex = QMutex() self.stopMe = 0 self.errors = [] def run(self): self.mutex.lock() self.stopMe = 0 self.mutex.unlock() interrupted = False outPath = self.outDir if outPath.find("\\") != -1: outPath.replace("\\", "/") if not outPath.endswith("/"): outPath = outPath + "/" provider = self.layer.dataProvider() index = provider.fieldNameIndex(self.field) unique = ftools_utils.getUniqueValues(provider, int(index)) baseName = unicode(outPath + self.layer.name() + "_" + self.field + "_") fieldList = ftools_utils.getFieldList(self.layer) sRs = provider.crs() geom = self.layer.wkbType() inFeat = QgsFeature() self.emit(SIGNAL("rangeCalculated(PyQt_PyObject)"), len(unique)) for i in unique: check = QFile(baseName + "_" + unicode(i).strip() + ".shp") fName = check.fileName() if check.exists(): if not QgsVectorFileWriter.deleteShapeFile(fName): self.errors.append(fName) continue writer = QgsVectorFileWriter(fName, self.encoding, fieldList, geom, sRs) fit = provider.getFeatures() while fit.nextFeature(inFeat): atMap = inFeat.attributes() if atMap[index] == i: writer.addFeature(inFeat) del writer self.emit(SIGNAL("valueProcessed()")) self.mutex.lock() s = self.stopMe self.mutex.unlock() if s == 1: interrupted = True break if not interrupted: self.emit(SIGNAL("processFinished( PyQt_PyObject )"), self.errors) else: self.emit(SIGNAL("processInterrupted()")) def stop(self): self.mutex.lock() self.stopMe = 1 self.mutex.unlock() QThread.wait(self)
class Walker(QThread): finished = Signal(bool, int) indexed = Signal(QString, int) COMMON_WORDS_THRESHOLD = 250 MIN_WORD_LEN = 3 MAX_WORD_LEN = 25 INVALID_FIRST_OR_LAST = frozenset("0123456789_") STRIPHTML_RE = re.compile(r"<[^>]*?>", re.IGNORECASE|re.MULTILINE) ENTITY_RE = re.compile(r"&(\w+?);|&#(\d+?);") SPLIT_RE = re.compile(r"\W+", re.IGNORECASE|re.MULTILINE) def __init__(self, index, lock, files, filenamesForWords, commonWords, parent=None): super(Walker, self).__init__(parent) self.index = index self.lock = lock self.files = files self.filenamesForWords = filenamesForWords self.commonWords = commonWords self.stopped = False self.mutex = QMutex() self.completed = False def stop(self): try: self.mutex.lock() self.stopped = True finally: self.mutex.unlock() def isStopped(self): try: self.mutex.lock() return self.stopped finally: self.mutex.unlock() def run(self): self.processFiles() self.stop() self.finished.emit(self.completed, self.index) def processFiles(self): for fname in self.files: if self.isStopped(): return self.processOneFile(fname) self.completed = True def processOneFile(self, fname): words = set() text = self.readFile(fname) if text is None or self.isStopped(): return text = self.STRIPHTML_RE.sub("", text) text = self.ENTITY_RE.sub(unichrFromEntity, text) text = text.lower() for word in self.SPLIT_RE.split(text): if (self.MIN_WORD_LEN <= len(word) <= self.MAX_WORD_LEN and word[0] not in self.INVALID_FIRST_OR_LAST and word[-1] not in self.INVALID_FIRST_OR_LAST): try: self.lock.lockForRead() new = word not in self.commonWords finally: self.lock.unlock() if new: words.add(word) if self.isStopped(): return for word in words: try: self.lock.lockForWrite() files = self.filenamesForWords[word] if len(files) > self.COMMON_WORDS_THRESHOLD: del self.filenamesForWords[word] self.commonWords.add(word) else: files.add(unicode(fname)) finally: self.lock.unlock() self.indexed.emit(fname, self.index) def readFile(self, fname): try: with codecs.open(fname, "r", "UTF8", "ignore") as file: return file.read() except (IOError, OSError), err: sys.stderr.write("Error: {0}\n".format(err))
class WorkerThread(QThread): ''' Schedule things to do Wait all time or launch/wake other threads to recompute max amount of node to display with or without zoom, refresh/redraw paint area, etc. . ''' def __init__(self, parent = None): super(WorkerThread, self).__init__(parent) self.mutex = QMutex() self.condition = QWaitCondition() self.restart = False self.abort = False self.timeline = parent self.countThread = parent.countThread self.populateThread = parent.populateThread self.maxOccThread = parent.maxOccThread def __del__(self): self.mutex.lock() self.abort = True self.condition.wakeOne() self.mutex.unlock() self.wait() def render(self): locker = QMutexLocker(self.mutex) if not self.isRunning() and not self.countThread.isRunning() and not self.populateThread.isRunning() and not self.maxOccThread.isRunning(): self.start(QThread.LowPriority) else: self.restart = True self.condition.wakeOne() def run(self): while True: self.mutex.lock() # Fetch value from timeline nodeCount = self.timeline.nodeCount dataListsCreated = self.timeline.dataListsCreated if dataListsCreated: self.timeline.findMaxValue() xHop = self.timeline.xHop maxOcc = self.timeline.maxOcc self.mutex.unlock() if not nodeCount and not self.restart and not self.countThread.isRunning(): self.countThread.start() elif nodeCount and not dataListsCreated and not self.restart and not self.populateThread.isRunning(): self.populateThread.start() elif nodeCount and dataListsCreated and xHop and not maxOcc and not self.restart and not self.maxOccThread.isRunning(): self.maxOccThread.start() elif nodeCount and dataListsCreated and xHop and maxOcc and not self.restart and not self.maxOccThread.isRunning(): self.emit(SIGNAL('refresh'), True) if self.abort: return self.mutex.lock() if not self.restart: self.condition.wait(self.mutex) self.restart = False self.mutex.unlock()
class _OperationsSpool(QObject): """ A spool of operations of a certain type. Can use many threads. """ def __init__(self, parent): """ Pprepare spools and threads Must be overriden to set : - type : one of "content", "action", "authentication", "network" - max_threads : max number of threads (default 1) """ super(_OperationsSpool, self).__init__(parent) self.max_threads = 1 self.threads = [] # list of threads running operations self.mutex = QMutex() self.spool = deque() # list of remaining operations self.running = [] # list of running operations self.done = [] # list of finished operations self.errors = [] # list of failed operations self.waiting = deque() # list of operations not yet added to the spool def count_all(self): """ Return a hash with count of operations for each list in this spool """ result = {} for type in ('spool', 'running', 'done', 'errors', 'waiting', ): result[type] = len(getattr(self, type)) return result def count_running(self): """ Count "alive" operations : spool + running + waiting """ by_type = self.count_all() return by_type['spool'] + by_type['running'] + by_type['waiting'] def add(self, operation): """ Add an operation to the waiting spool """ self.waiting.append(operation) self.check_waiting_list() def check_waiting_list(self): """ This method will check conditions about the operation before to put it in the spool queue (max_same, kill_same, opposites...) Thi method is threadsafe (mutex.lock is here to ensure that only one waiting operation is checked at a time) """ self.mutex.lock() try: # get operation operation = self.waiting.popleft() # check operations specificities can_add, to_remove = self._test_opposite(operation) for op in to_remove: op.status = 'rejected' self.spool.remove(op) if can_add: can_add, to_remove = self._test_same(operation) for op in to_remove: op.status = 'rejected' self.spool.remove(op) # finally add the operation to the spool only if we can add # the operation if can_add: self.spool.append(operation) self.ping_threads() else: operation.status = 'rejected' finally: self.mutex.unlock() def _test_opposite(self, operation): """ If the operation has an opposite name, this method check opposites in the spool. If an opposite is found and the kill_opposite flag is True, then the opposite operation is removed from the spool, but if the flag is False, then the current operation is cancelled. """ to_remove = [] can_add = True if operation.opposite: for op in self.spool: if op.is_opposite_of(operation): if operation.kill_opposite: to_remove.append(op) else: can_add = False return can_add, to_remove def _test_same(self, operation): """ If the operation has a positive max_same, this method will check for same operations in the spool and if at least one, return False (and an empty list of operations to remove). Else if we have a kill_same, we check for same operations in the spool and return True (operation still valid) and the list of removals """ to_remove = [] can_add = True # check max_same if operation.max_same: count_same = 0 for op in self.spool: if op.is_same_as(operation): count_same += 1 if count_same >= operation.max_same: can_add = False break # check max_same elif operation.kill_same: to_remove = [] for op in self.spool: if op.is_same_as(operation): to_remove.append(op) return can_add, to_remove def next(self): """ Return the next operation to run, remove it from the spool queue, and move it to the running one. Will be called by the next thread which has finished working on its previous operation. This method is threadsafe (mutex.lock is here to ensure only one thread at a time can claim a operation) """ self.mutex.lock() try: operation = self.spool.popleft() self.running.append(operation) return operation except: return None finally: self.mutex.unlock() def ask_for_network(self): """ We ask for network """ self.parent().ask_for_network() def ask_for_authentication(self): """ We ask for a new authentication """ self.parent().ask_for_authentication() def is_ready(self): """ Check if the spool is ready to work (we are both connected and authenticated) """ return self.parent().is_ready() def ping_threads(self): """ Ping all threads to let them check if new operations are available in the spool. A new thread is created if needed (and possible) """ one_available = False for thread in self.threads: if not thread.isRunning(): one_available = True thread.ping() if not one_available and len(self.threads) < self.max_threads: thread = OperationsThread(spool=self) QObject.connect(thread, SIGNALS['operation_started'], self.operation_started, Qt.QueuedConnection) QObject.connect(thread, SIGNALS['operation_ended'], self.operation_ended, Qt.QueuedConnection) self.threads.append(thread) thread.ping() def operation_started(self, operation_id): """ Forward the "operation_started" from the spool to the operations manager """ log("[op-start] %s\n" % Operation.get_by_id(operation_id)) operation = Operation.get_by_id(operation_id) self.parent().emit(SIGNALS['%s_started' % operation.name], operation_id) self.parent().emit(SIGNALS['operation_started'], operation_id) def operation_ended(self, operation_id): """ Forward the "operation_ended" from the spool to the operations manager """ # get the corresponding operation operation = Operation.get_by_id(operation_id) # update the spool self.running.remove(operation) if operation.status == 'done': self.done.append(operation) else: self.errors.append(operation) # alert that an operation has finished log("[op-end] %s\n" % operation) self.parent().emit(SIGNALS['operation_ended'], operation_id) # then alert for this specific operation status = 'done' operation = Operation.get_by_id(operation_id) if operation.status != 'done': status = 'error' self.parent().emit(SIGNALS['%s_%s' % (operation.name, status)], operation_id)
class PlottingThread(QThread): def __init__(self, parent): QThread.__init__(self) self.result = None self.parent = parent self._stopped = False self.mutex = QMutex() self.filePrefix = None self.fileFormat = None self.wallColoring = None self.cellColoring = None self.pointColoring = None self.extraDrawing = [] self.pointSize = None self.pointLineColor = None self.pointLineThickness = None self.ellipsisDraw = None self.overSampling = None self.wallThickness = None self.bgColor = None self.loading = False self._crop = QRect(0,0,1,1) self._pix = None self._end_image_plot = False self._loading_arguments = {} self.retryObject = None def end_step(self): return len(self.result)+1 def stop(self, value = True): self.mutex.lock() self._stopped = value self.mutex.unlock() def stopped(self): self.mutex.lock() val = self._stopped self.mutex.unlock() return val def nextImage(self): QCoreApplication.postEvent(self.parent, NextImageEvent()) def abort(self, reason, **others): e = AbortPlottingEvent(reason) if others: e.others = others QCoreApplication.postEvent(self.parent, e) def finished(self): if self.loading: QCoreApplication.postEvent(self.parent, FinishLoadingEvent()) self.loading = False else: QCoreApplication.postEvent(self.parent, FinishPlottingEvent()) def image_ready(self): QCoreApplication.postEvent(self.parent, ImageReadyPlottingEvent()) def update_nb_images(self, nb): QCoreApplication.postEvent(self.parent, UpdateNbImageEvent(nb)) @property def crop_left(self): return self._crop.left() @crop_left.setter def crop_left(self, value): self._crop.moveLeft(int(value)) @property def crop_top(self): return self._crop.top() @crop_top.setter def crop_top(self, value): self._crop.moveTop(int(value)) @property def crop_width(self): return self._crop.width() @crop_width.setter def crop_width(self, value): self._crop.setWidth(int(value)) @property def crop_height(self): return self._crop.height() @crop_height.setter def crop_height(self, value): self._crop.setHeight(int(value)) def reset_crop(self): self._crop = QRect(QPoint(0,0), self.img_size) @property def crop(self): return QRect(self._crop) @crop.deleter def crop(self): self.reset_crop() @property def end_image_plot(self): ''' If true, plot the growth data on the end image rather than the start image of the growth calculation. ''' return self._end_image_plot @end_image_plot.setter def end_image_plot(self, value): self._end_image_plot = bool(value) @property def pix(self): '''Thread-safe image storage.''' self.mutex.lock() pix = self._pix self.mutex.unlock() return pix @pix.setter def pix(self, value): self.mutex.lock() self._pix = value self.mutex.unlock() def render_valid(self): if self.result is None: log_debug("result is None") return False if self.parent is None: log_debug("parent is None") return False if self.ellipsisDraw is None: log_debug("ellipsisDraw is None") return False if self.cellColoring is None: log_debug("cellColoring is None") return False if self.wallColoring is None: log_debug("wallColoring is None") return False if self.pointColoring is None: log_debug("pointColoring is None") return False if self.pointSize is None: log_debug("pointSize is None") return False if self.pointLineThickness is None: log_debug("pointSize is None") return False if self.pointLineColor is None: log_debug("pointSize is None") return False if self.wallThickness is None: log_debug("wallThickness is None") return False if self.overSampling is None: log_debug("overSampling is None") return False if self.bgColor is None: log_debug("bgColor is None") return False return True def valid(self): if self.filePrefix is None: log_debug("filePrefix is None") return False if not self.filePrefix: log_debug("filePrefix is Empty") return False if self.fileFormat is None: log_debug("fileFormat is None") return False return self.render_valid() def drawImage(self, imageid): cache = image_cache.cache cellColoring = self.cellColoring wallColoring = self.wallColoring pointColoring = self.pointColoring ellipsisDraw = self.ellipsisDraw overSampling = self.overSampling extraDrawing = self.extraDrawing bgColor = self.bgColor.rgb() result = self.result if self.result_type == "Data": data = result img_name = result.images_name[imageid] else: data = result.data img_name = result.images[imageid] #scale = data.images_scale[img_name] min_scale = data.minScale() img = cache.image(data.image_path(img_name)) img_data = data[img_name] size = self._crop.size() pix = QImage(size*overSampling, QImage.Format_ARGB32) pix.fill(bgColor) painter = QPainter() if not painter.begin(pix): self.abort("Cannot create painter on QImage") return None, None, None painter.setRenderHints(QPainter.SmoothPixmapTransform, True) painter.setRenderHints(QPainter.Antialiasing, True) if overSampling > 1: painter.scale(overSampling, overSampling) painter.translate(-self._crop.topLeft()) painter.save() painter.translate(self.translate) log_debug("Translating: %gx%g" % (self.translate.x(), self.translate.y()) ) painter.scale(1/min_scale, 1/min_scale) painter.save() matrix = img_data.matrix() painter.setWorldTransform(matrix, True) painter.drawImage(QPoint(0,0), img) painter.restore() #pt_matrix = QTransform() #pt_matrix.scale(1/min_scale, 1/min_scale) #painter.setTransform(pt_matrix, True) cellColoring.startImage(painter, imageid) wallColoring.startImage(painter, imageid) for ed in extraDrawing: ed.startImage(painter, imageid) if self.result_type == "Growth": cells = result.cells[imageid] walls = result.walls[imageid] else: cells = img_data.cells walls = set() for cid in img_data.cells: pts = [ pt for pt in data.cells[cid] if pt in img_data ] if len(pts) > 1: for i in range(len(pts)): walls.add(data.wallId(pts[i-1], pts[i])) # Now, draw the cells and the ellipsis for cid in cells: painter.setPen(Qt.NoPen) color = cellColoring(imageid, cid) painter.setBrush(color) pts = data.cellAtTime(cid, img_data.index) if pts: pts.append(pts[0]) ppts = [] for p1,p2 in zip(pts[:-1], pts[1:]): ppts.append(img_data[p1]) ppts.extend(img_data.walls[p1,p2]) ppts.append(ppts[0]) poly = QPolygonF(ppts) painter.drawPolygon(poly) # And draw the walls wallThickness = self.wallThickness*min_scale for wid in walls: color = wallColoring(imageid, wid) if color.alpha() > 0: pen = QPen(color) pen.setWidthF(wallThickness) painter.setPen(pen) pts = [img_data[wid[0]]] + img_data.walls[wid[0], wid[1]] + [img_data[wid[1]]] #painter.drawLine(img_data[wid[0]], img_data[wid[1]]) painter.drawPolyline(*pts) # Then, draw the points pointSize = self.pointSize*min_scale pointLineColor = self.pointLineColor pointLineThickness = self.pointLineThickness*min_scale log_debug("pointSize = %g" % pointSize) for pid in img_data: color = pointColoring(imageid, pid) if color.alpha() > 0: pen = QPen(pointLineColor) pen.setWidthF(pointLineThickness) brush = QBrush(color) painter.setPen(pen) painter.setBrush(brush) pos = img_data[pid] rect = QRectF(pos.x()-pointSize, pos.y()-pointSize, 2*pointSize, 2*pointSize) painter.drawEllipse(rect) if ellipsisDraw.plot: for cid in cells: pts = data.cellAtTime(cid, img_data.index) if pts: pts.append(pts[0]) ppts = [] for p1,p2 in zip(pts[:-1], pts[1:]): ppts.append(img_data[p1]) ppts.extend(img_data.walls[p1,p2]) ppts.append(ppts[0]) #poly = QPolygonF(ppts) #painter.drawPolygon(poly) ellipsisDraw(painter, imageid, cid, ppts, min_scale) # At last, draw the extra data for ed in extraDrawing: ed(painter, imageid) tr = painter.worldTransform() painter.restore() pic_w = wallColoring.finalizeImage(painter, imageid, tr, self.crop) pic_c = cellColoring.finalizeImage(painter, imageid, tr, self.crop) for ed in extraDrawing: ed.finalizeImage(painter, imageid, tr, self.crop) painter.end() return pix, pic_w, pic_c def start(self): if self.isRunning(): assert not self.rendering_all, "Cannot run twice the rendering of all images with the same object." return if parameters.instance.use_thread: log_debug("Starting rendering thread.") QThread.start(self) return False else: self.run() return True def render_all(self): self.rendering_all = True return self.start() def render_single(self, img_id, retry=False): if retry: while self.isRunning(): self.wait(10000) elif self.isRunning(): return self.rendering_all = False self.current_image = img_id return self.start() def load(self, filename): self.loading = True self.result = filename return self.start() def run(self): if self.loading: self.run_loader() elif self.rendering_all: self.run_full() else: self.run_single() def run_single(self): img = self.current_image self.cellColoring.init() self.wallColoring.init() self.pointColoring.init() log_debug("Rendering image %d" % img) self.pix, self.pic_w, self.pic_c = self.drawImage(img) if self.pic_w is not None: log_debug("Has wall image") if self.pic_c is not None: log_debug("Has cell image") if self.pix is not None: log_debug("Pix correctly rendered") log_debug("Rendered image %d = %s" % (img, self.pix)) self.image_ready() def reload(self): if self.retryObject is None: return self._loading_arguments.update(self.retryObject.method_args) self.load(self.retryObject.filename) def run_loader(self): filename = self.result try: self.retryObject = None # First, prepare the data by getting the images and computing how big they # should be f = open(filename) first_line = f.readline() f.close() if first_line.startswith("TRKR_VERSION"): result = Result(None) result.load(self.result, **self._loading_arguments) result_type = "Growth" else: result = TrackingData() result.load(self.result, **self._loading_arguments) result_type = "Data" self.result = result self.result_type = result_type if result_type == "Data": data = result images = data.images_name if data.cells: self.has_cells = True self.has_walls = True else: self.has_cells = False self.has_walls = False self.has_points = bool(data.cell_points) else: data = result.data images = result.images self.has_cells = False self.has_walls = False self.has_points = False self.images = images cache = image_cache.cache self.update_nb_images(len(result)) bbox = QRectF() ms = data.minScale() for i in range(len(result)): img_name = images[i] img_data = data[img_name] img = cache.image(data.image_path(img_name)) matrix = QTransform() matrix = img_data.matrix() sc = QTransform() sc.scale(1.0/ms, 1.0/ms) matrix *= sc r = QRectF(img.rect()) rbox = matrix.map(QPolygonF(r)).boundingRect() bbox |= rbox log_debug("Image '%s':\n\tSize = %gx%g\n\tTransformed = %gx%g %+g %+g\n\tGlobal bbox = %gx%g %+g %+g\n" % (img_name, r.width(), r.height(), rbox.width(), rbox.height(), rbox.left(), rbox.top(), bbox.width(), bbox.height(), bbox.left(), bbox.top())) log_debug("Matrix:\n%g\t%g\t%g\n%g\t%g\t%g\n" % (matrix.m11(), matrix.m12(), matrix.dx(), matrix.m21(), matrix.m22(), matrix.dy())) if result_type == "Growth": if result.cells[i]: self.has_cells = True if result.walls[i]: self.has_walls = True self.has_points = bool(result.data.cell_points) self.nextImage() translate = bbox.topLeft() translate *= -1 self.translate = translate size = bbox.size().toSize() self.img_size = size self._crop = QRect(QPoint(0,0), size) self.finished() self._loading_arguments = {} # All done, we don't need that anymore except RetryTrackingDataException as ex: ex.filename = filename self.retryObject = ex self.finished() return except Exception as ex: _, _, exceptionTraceback = sys.exc_info() self.abort(ex, traceback=exceptionTraceback) raise def run_full(self): if not self.valid(): self.abort("Object was not correctly initialized") return self.stop(False) painter = None try: result = self.result self.update_nb_images(len(result)) # if self.result_type == "Data": # data = result # images = result.images_name # else: # data = result.data # images = result.images # cache = image_cache.cache cellColoring = self.cellColoring wallColoring = self.wallColoring pointColoring = self.pointColoring file_format = self.fileFormat file_pattern = "%s%%0%dd.%s" % (self.filePrefix, len(str(len(result))), file_format) wall_file_pattern = "%s%%0%dd_wall.%s" % (self.filePrefix, len(str(len(result))), file_format) cell_file_pattern = "%s%%0%dd_cell.%s" % (self.filePrefix, len(str(len(result))), file_format) cellColoring.init() wallColoring.init() pointColoring.init() self.nextImage() for i in range(len(result)): if self.stopped(): self.abort("User interruption") return pix, pic_w, pic_c = self.drawImage(i) pix.save(file_pattern % (i+1), file_format) if pic_w is not None: self.saveExtra(pic_w, wall_file_pattern % (i+1), file_format) if pic_c is not None: self.saveExtra(pic_c, cell_file_pattern % (i+1), file_format) self.nextImage() self.finished() except Exception as ex: if painter is not None: painter.end() _, _, exceptionTraceback = sys.exc_info() self.abort(ex, traceback=exceptionTraceback) raise def saveExtra(self, picture, file_name, file_format): rect = picture.boundingRect() pix = QImage(rect.size(), QImage.Format_ARGB32) pix.fill(QColor(0, 0, 0, 0).rgba()) paint = QPainter() paint.begin(pix) paint.drawPicture(rect.topLeft()*-1, picture) paint.end() pix.save(file_name, file_format)
class Rexarm(): """! @brief This class describes a rexarm. The configuration of the rexarm is loaded from a .csv file as described in the Rexarm.initialize. """ # Class constants # Convert config file type strings to string to type functions STR_TO_TYPE = { 'bool': lambda s: s.lower() in s.lower() in ['true', '1', 't', 'y', 'yes'], 'str': str, 'float': float, 'int': int } def __init__(self): """! @brief Constructs a new instance. Starts the rexarm run thread but does not initialise the Joints. Call Rexarm.initialize to initialise the Joints. """ # Thread locks self._cmd_mutex = QMutex() self._serial_mutex = QMutex() # Serial bus self.dxlbus = None # Gripper self.gripper = None self.gripper_state = True # State self.estop = False self.initialized = False # Update rexarm every update_period seconds self.update_period = 0.01 # Cmds self.new_speed_cmds = False self.speed_cmds = None self.new_torque_limit_cmds = False self.torque_limit_cmds = None self.new_position_cmds = False self.position_cmds = None # Feedback self.position_fb = None self.speed_fb = None self.load_fb = None self.temp_fb = None # Joints self._joints = [] self.open_angle = 0 self.close_angle = -60 # Data collection flag self.collect_flag = False # Start run thread self.run_thread = RexarmThread(self) self.run_thread.start() def initialize(self, config_file=script_path+'/config/rexarm_config.csv'): """! @brief Initializes the rexarm from given configuration file. Initializes the Joints and serial port @param config_file The configuration file see Rexarm._config_joints function for more details on the config file. @return True is succes False otherwise """ self.initialized = False # Wait for other threads to finish with the rexarm instead of locking every single call time.sleep(1) # Get a serial port interface self._serial_mutex.lock() if self.dxlbus is not None: self.dxlbus.close() self.dxlbus = DXL_BUS(DEVICENAME, BAUDRATE) self.port_num = self.dxlbus.port() self._serial_mutex.unlock() """ Read in rexarm configs from config file """ if not self._config_joints(config_file): return self.initialized """ Commanded Values """ self.num_joints = len(self._joints) self.position = [0.0] * self.num_joints # degrees self.speed = [1.0] * self.num_joints # 0 to 1 self.max_torque = [1.0] * self.num_joints # 0 to 1 """ Feedback Values """ self.position_fb = [0.0] * self.num_joints # degrees self.speed_fb = [0.0] * self.num_joints # 0 to 1 self.load_fb = [0.0] * self.num_joints # -1 to 1 self.temp_fb = [0.0] * self.num_joints # Celsius self.move_fb = [0] * self.num_joints """ Gripper Vaules """ self.gripper = None for joint in self._joints: if joint.is_gripper: self.gripper = joint if self.gripper is not None: self.gripper.torque_limit = 1.0 self.gripper.speed = 0.8 # Reset estop and initialized self.estop = False self.initialized = True return self.initialized def _config_joints(self, config_file): """! @brief Configure the Joints and initialize them TODO: Find the physical properties of the Rexarm (such as angle limits and DH parameters) and write a config file for the Rexarm. The config file should be formated as a csv file with the following properties: * The first row lists the exact names of the member variables of the Joint class * The second row list the types of the associated member variables, Rexarm.STR_TO_TYPE dict defines the avaliable types * Each following row defines thoes member variables for a joint in the rexarm * There should be no spaces after a comma You may add any new member variables as you wish to the config file in any order. By default they will be added as a readonly property of the Joint class. If special behaviors are desired for a new member variable then they must be added to the Joint.__init__ parameter list and handled in that function. @param config_file The configuration file @return True if all joints initialize correctly, False if any joint fails """ self._joints = [] with open(config_file) as configs: reader = csv.reader(configs) title_row = next(reader) type_strs = next(reader) types = [Rexarm.STR_TO_TYPE[s] for s in type_strs] for row in reader: values = {name: dtype(value) for name, value, dtype in zip(title_row, row, types)} joint = Joint(self._serial_mutex, self.port_num, **values) self._joints.append(joint) if not joint.initialize(): return False return True def _ensure_initialized(func): """! @brief Decorator to skip the function if the rexarm is not initialized. @param func The function to wrap @return The wraped function """ def func_out(self, *args, **kwargs): if self.initialized: return func(self, *args, **kwargs) else: print('WARNING: Trying to use the Rexarm before initialized') return func_out @_ensure_initialized def open_gripper(self): """! @brief TODO: Tell the gripper to open. """ new_position = self.position_fb new_position[5] = self.open_angle self.set_positions(new_position) @_ensure_initialized def open_gripper_blocking(self): """! @brief TODO: Open a gripper and block until it is. """ pass def is_gripper_open(self): """! @brief TODO: Determines if gripper open. @return True if gripper open, False otherwise. """ if abs(self.position_fb[5] - self.open_angle) < 0.2: return True else: return False @_ensure_initialized def close_gripper(self): """! @brief TODO Closes a gripper. """ new_position = self.position_fb new_position[5] = self.close_angle self.set_positions(new_position) @_ensure_initialized def close_gripper_blocking(self): """! @brief TODO Closes a gripper and block until it is. """ pass def is_gripper_close(self): """! @brief TODO Determines if gripper close. @return True if gripper close, False otherwise. """ pass def toggle_gripper(self): if self.is_gripper_open(): self.open_gripper() else: self.close_gripper() @_ensure_initialized def toggle_gripper(self): """! @brief TODO Toggle the gripper between open and close """ pass @_ensure_initialized def toggle_gripper_blocking(self): """! @brief TODO Toggle the gripper between open and close and block until done """ pass def set_positions(self, joint_angles): """! @brief Sets the positions. @param joint_angles The joint angles """ self._cmd_mutex.lock() self.position_cmds = joint_angles self.new_position_cmds = True self._cmd_mutex.unlock() def set_speeds_normalized_all(self, speed): """! @brief Sets the speeds normalized all. @param speed The speed """ self._cmd_mutex.lock() self.speed_cmds = [speed] * len(self._joints) self.new_speed_cmds = True self._cmd_mutex.unlock() def set_speeds_normalized(self, speeds): """! @brief Sets the speeds normalized. @param speeds The speeds """ self._cmd_mutex.lock() self.speed_cmds = speeds self.new_speed_cmds = True self._cmd_mutex.unlock() def set_speeds(self, speeds): """! @brief Sets the speeds. @param speeds The speeds """ self._cmd_mutex.lock() self.speed_cmds = [None] * len(self._joints) for i in range(len(self._joints)): cmd = abs(speeds[i] / self._joints[i].max_speed) if (cmd < 3.0 / 1023.0): cmd = 3.0 / 1023.0 self.speed_cmds[i] = cmd self.new_speed_cmds = True self._cmd_mutex.unlock() def set_torque_limits(self, torques): """! @brief Sets the torque limits. Can't be used when estoped. @param torques The torques """ if self.estop: print('WARNING: Torques are zero when estoped') return self._cmd_mutex.lock() self.torque_limit_cmds = torques self.new_torque_limit_cmds = True self._cmd_mutex.unlock() @_ensure_initialized def _send_commands(self): """! @brief Sends commands. Send the most recent commands to the joints or set torques to zero if estoped. """ # Copy current commands so other threads need not wait for the serial communications self._cmd_mutex.lock() torque_limit_cmds = None speed_cmds = None position_cmds = None if self.new_torque_limit_cmds: torque_limit_cmds = self.torque_limit_cmds.copy() if self.new_speed_cmds: speed_cmds = self.speed_cmds.copy() if self.new_position_cmds: position_cmds = self.position_cmds.copy() self.new_torque_limit_cmds = False self.new_speed_cmds = False self.new_position_cmds = False self._cmd_mutex.unlock() # Set torques to zero when estoped if self.estop: torque_limit_cmds = [0] * len(self._joints) for joint in self._joints: joint.torque_limit = 0 return # Write the commands for i, joint in enumerate(self._joints): if position_cmds is not None: joint.position = position_cmds[i] # Gripper torque and speed handled seperatly if joint.is_gripper: continue if torque_limit_cmds is not None: joint.torque_limit = torque_limit_cmds[i] if speed_cmds is not None: joint.speed = speed_cmds[i] def disable_torque(self): """! @brief Disables the torque and estops. """ self.estop = True def get_positions(self): """! @brief Gets the positions. @return The positions. """ return self.position_fb def get_speeds(self): """! @brief Gets the speeds. @return The speeds. """ return self.speed_fb def get_loads(self): """! @brief Gets the loads. @return The loads. """ return self.load_fb def get_temps(self): """! @brief Gets the temps. @return The temps. """ return self.temp_fb def get_moving_status(self): """! @brief Gets the moving status. @return The moving status. """ return self.move_fb def get_errors(self): """! @brief Gets the serial errors. @return The serial errors. """ return [joint.serial_errors for joint in self._joints] @_ensure_initialized def _get_feedback(self): """! @brief Updates all the feedback variables of the joints. """ self.position_fb = [joint.position_fb for joint in self._joints] self.speed_fb = [joint.speed_fb for joint in self._joints] self.load_fb = [joint.load_fb for joint in self._joints] self.temp_fb = [joint.temp_fb for joint in self._joints] self.move_fb = [joint.move_fb for joint in self._joints] @_ensure_initialized def get_wrist_pose(self): """! @brief TODO Get the wrist pose. @return The wrist pose as [x, y, z, phi]. """ return [0, 0, 0, 0] def get_dh_parameters(self): """! @brief Gets the dh parameters. @return The dh parameters. """ return [[joint.dh_a, joint.dh_alpha, joint.dh_d, joint.dh_theta] for joint in self._joints] @_ensure_initialized def run(self): """! @brief Update the servos feedback and commands Run in a seperate thread see RexarmThread. """ # Update feedback self._get_feedback() if self.collect_flag: with open("rexarm_pos_data.csv", 'a') as inFile: inFile.write(str(time.time()) + ",") for pos in self.position_fb: inFile.write(str(pos) + ",") inFile.write("\n") # Send new commands self._send_commands()
class PointsInPolygonThread(QThread): def __init__(self, inPoly, inPoints, fieldName, outPath, encoding, attributeList, statisticSelector): QThread.__init__(self, QThread.currentThread()) self.mutex = QMutex() self.stopMe = 0 self.interrupted = False self.layerPoly = inPoly self.layerPoints = inPoints self.fieldName = fieldName self.outPath = outPath self.encoding = encoding self.attributeList = attributeList self.statistics = statisticSelector.currentText() def run(self): self.mutex.lock() self.stopMe = 0 self.mutex.unlock() interrupted = False polyProvider = self.layerPoly.dataProvider() pointProvider = self.layerPoints.dataProvider() fieldList = ftools_utils.getFieldList(self.layerPoly) index = polyProvider.fieldNameIndex(unicode(self.fieldName)) if index == -1: index = polyProvider.fields().count() fieldList.append( QgsField(unicode(self.fieldName), QVariant.Int, "int", 10, 0, self.tr("point count field"))) # Add the selected vector fields to the output polygon vector layer selectedItems = self.attributeList.selectedItems() for item in selectedItems: global typeDouble columnName = unicode(item.text() + "_" + self.statistics) index = polyProvider.fieldNameIndex(unicode(columnName)) if index == -1: if item.type( ) == typeDouble or self.statistics == "mean" or self.statistics == "stddev": fieldList.append( QgsField(columnName, QVariant.Double, "double", 24, 15, "Value")) else: fieldList.append( QgsField(columnName, QVariant.Int, "int", 10, 0, "Value")) sRs = polyProvider.crs() if QFile(self.outPath).exists(): if not QgsVectorFileWriter.deleteShapeFile(self.outPath): return writer = QgsVectorFileWriter(self.outPath, self.encoding, fieldList, polyProvider.geometryType(), sRs) spatialIndex = ftools_utils.createIndex(pointProvider) self.emit(SIGNAL("rangeChanged(int)"), polyProvider.featureCount()) polyFeat = QgsFeature() pntFeat = QgsFeature() outFeat = QgsFeature() inGeom = QgsGeometry() polyFit = polyProvider.getFeatures() while polyFit.nextFeature(polyFeat): inGeom = polyFeat.geometry() atMap = polyFeat.attributes() outFeat.setAttributes(atMap) outFeat.setGeometry(inGeom) count = 0 pointList = [] hasIntersection = True pointList = spatialIndex.intersects(inGeom.boundingBox()) if len(pointList) > 0: hasIntersection = True else: hasIntersection = False if hasIntersection: valueList = {} for item in selectedItems: valueList[item.text()] = [] for p in pointList: pointProvider.getFeatures(QgsFeatureRequest().setFilterFid( p)).nextFeature(pntFeat) tmpGeom = QgsGeometry(pntFeat.geometry()) if inGeom.intersects(tmpGeom): count += 1 for item in selectedItems: valueList[item.text()].append( pntFeat.attribute(item.text())) self.mutex.lock() s = self.stopMe self.mutex.unlock() if s == 1: interrupted = True break atMap.append(count) # Compute the statistical values for selected vector attributes for item in selectedItems: values = valueList[item.text()] # Check if the input contains non-numeric values non_numeric_values = False for value in values: if not isinstance(value, type( float())) and not isinstance( value, type(int())): non_numeric_values = True break # Jump over invalid values if non_numeric_values is True: continue if values and len(values) > 0: if self.statistics == "sum": value = reduce(myAdder, values) elif self.statistics == "mean": value = reduce(myAdder, values) / float( len(values)) elif self.statistics == "min": values.sort() value = values[0] elif self.statistics == "max": values.sort() value = values[-1] elif self.statistics == "stddev": value = two_pass_variance(values) value = math.sqrt(value) atMap.append(value) outFeat.setAttributes(atMap) writer.addFeature(outFeat) self.emit(SIGNAL("updateProgress()")) self.mutex.lock() s = self.stopMe self.mutex.unlock() if s == 1: interrupted = True break del writer if not interrupted: self.emit(SIGNAL("processingFinished()")) else: self.emit(SIGNAL("processingInterrupted()")) def stop(self): self.mutex.lock() self.stopMe = 1 self.mutex.unlock() QThread.wait(self)
class Walker(QThread): COMMON_WORDS_THRESHOLD = 250 MIN_WORD_LEN = 3 MAX_WORD_LEN = 25 INVALID_FIRST_OR_LAST = frozenset("0123456789_") STRIPHTML_RE = re.compile(r"<[^>]*?>", re.IGNORECASE | re.MULTILINE) ENTITY_RE = re.compile(r"&(\w+?);|&#(\d+?);") SPLIT_RE = re.compile(r"\W+", re.IGNORECASE | re.MULTILINE) def __init__(self, index, lock, files, filenamesForWords, commonWords, parent=None): super(Walker, self).__init__(parent) self.index = index self.lock = lock self.files = files self.filenamesForWords = filenamesForWords self.commonWords = commonWords self.stopped = False self.mutex = QMutex() self.completed = False def stop(self): try: self.mutex.lock() self.stopped = True finally: self.mutex.unlock() def isStopped(self): try: self.mutex.lock() return self.stopped finally: self.mutex.unlock() def run(self): self.processFiles() self.stop() self.emit(SIGNAL("finished(bool,int)"), self.completed, self.index) def processFiles(self): def unichrFromEntity(match): text = match.group(match.lastindex) if text.isdigit(): return unichr(int(text)) u = htmlentitydefs.name2codepoint.get(text) return unichr(u) if u is not None else "" for fname in self.files: if self.isStopped(): return words = set() fh = None try: fh = codecs.open(fname, "r", "UTF8", "ignore") text = fh.read() except (IOError, OSError), e: sys.stderr.write("Error: {0}\n".format(e)) continue finally:
class SVDViewer(QWidget): svdSelectEvent = QtCore.pyqtSignal() def __init__(self, array, labels, **options): QWidget.__init__(self) assert isinstance(array, np.ndarray),\ "Got a %s instead of an ndarray" % type(array) # figure out our size self.width = self.size().width() self.height = self.size().height() self.labels = labels self.magnitudes = None # can be assigned by external information self.array = np.asarray(array) self.orig_array = self.array.copy() self.npoints = self.array.shape[0] self.scale = self.calculate_scale() if options.get('jitter', True): self.add_jitter() self.layers = [] self.k = self.array.shape[1] self.projection = Projection(self.k) # create our internal display self.painter = QPainter() self.selected_index = None self.mouseX = 0 self.mouseY = 0 self.buttons = 0 # avoid racing self.paint_lock = QMutex() self.timer = QTimer(self) self.timer_ticks = 0 self.timer.setInterval(30) self.timer.timeout.connect(self.timerEvent) self.timer.start() self.setMouseTracking(True) self.default_colors = self.components_to_colors(self.array)[:] self.update_colors() # initialize the mapping from projection coordinates to screen # coordinates self.reset_view() def age_timer(self): self.timer_ticks += 1 if self.timer_ticks == TIMER_MAX: self.timer.stop() def activate_timer(self): if self.timer_ticks >= TIMER_MAX: self.timer.start() self.timer_ticks = 0 def stop_timer(self): self.timer.stop() def __del__(self): self.stop_timer() def calculate_scale(self): """ Find roughly the median of axis coordinates, determining a reasonable zoom level for the initial view. Make sure it's non-zero. """ coords = [c for c in np.abs(self.array.flatten()) if c > 0] + [1.0] coords.sort() return coords[len(coords)//2] def add_jitter(self): self.jitter = np.exp(np.random.normal(size=self.array.shape) / 50.0) self.array *= self.jitter def insert_layer(self, pos, layertype, *args): """ Add a layer of visualization to this widget, by specifying the subclass of Layer and whatever arguments its constructor takes. """ self.layers.insert(pos, layertype(self, *args)) def add_layer(self, layertype, *args): """ Add a layer of visualization to this widget, by specifying the subclass of Layer and whatever arguments its constructor takes. """ self.layers.append(layertype(self, *args)) def reset_view(self): self.screen_center = np.array([0., 0.]) self.screen_size = np.array([self.scale/10, self.scale/10]) self.projection.reset_projection() self.set_default_axes() self.update_screenpts() self.update() @staticmethod def make_svdview(matrix, svdmatrix, magnitudes=None, canonical=None): widget = SVDViewer(svdmatrix, svdmatrix.row_labels) if magnitudes is None: magnitudes = np.array([np.linalg.norm(vec) for vec in svdmatrix]) widget.magnitudes = magnitudes widget.setup_standard_layers() widget.set_default_axes() if canonical is None: canonical = [] for c in canonical: #svdmatrix[svdmatrix.row_index(c)] *= 4 magnitudes[svdmatrix.row_index(c)] *= 2 widget.insert_layer(1, CanonicalLayer, canonical) widget.insert_layer(2, LinkLayer, matrix) widget.insert_layer(3, NetworkLayer, 6) return widget @staticmethod def make_colors(matrix, svdmatrix): widget = SVDViewer(svdmatrix, svdmatrix.col_labels) widget.setup_standard_layers() from csc.concepttools.colors import text_color colors = [text_color(text) for text in widget.labels] widget.default_colors = np.clip(np.array(colors), 55, 230) widget.update_colors() return widget @staticmethod def make(array, labels): widget = SVDViewer(array, labels) widget.setup_standard_layers() return widget def setup_standard_layers(self): self.add_layer(PixelRenderingLayer) self.add_layer(PointLayer) self.add_layer(LabelLayer, 1000, 2000) self.add_layer(SelectionLayer) self.add_layer(SimilarityLayer) self.add_layer(RotationLayer) self.add_layer(PanZoomLayer) def set_default_axes(self): self.set_axis_to_pc(0, 1) self.set_axis_to_pc(1, 2) def set_default_x_axis(self): self.set_axis_to_pc(0, 1) def set_default_y_axis(self): self.set_axis_to_pc(1, 2) def set_axis_to_pc(self, axis, pc): """ Sets an axis to a particular principal component. """ pcvec = np.zeros((self.array.shape[1],)) pcvec[pc] = 1.0 if axis == 0: self.projection.set_x_axis(pcvec) elif axis == 1: self.projection.set_y_axis(pcvec) self.activate_timer() def set_axis_to_text(self, axis, text): if isinstance(text, QString): text = unicode(text) if not text: return if text in self.labels: index = self.labels.index(text) if axis == 0: self.projection.set_x_axis(self.array[index,:]) elif axis == 1: self.projection.set_y_axis(self.array[index,:]) self.activate_timer() else: print repr(text), "not in label list" def is_point_on_screen(self, coords): return np.all((coords >= np.int32([0, 0])) & (coords < np.int32([self.width, self.height])), axis=-1) def is_on_screen(self, x, y): return x >= 0 and x < self.width and y >= 0 and y < self.height def pixel_size(self): """ What is the diameter of a single pixel at the current zoom level? If the x and y scales end up being different, take the geometric mean. """ xsize = self.screen_size[0] / self.width ysize = self.screen_size[1] / self.height return np.sqrt(xsize*ysize) def components_to_screen(self, coords): return self.projection_to_screen(self.projection.components_to_projection(coords)) def projection_to_screen(self, coords): zoomed = (coords - self.screen_center) / self.screen_size screen = (zoomed + np.array([self.width, self.height])/2) return np.int32(screen * FLIP_Y + [0, self.height]) def screen_to_projection(self, screen_coords): zoomed = ((screen_coords - [0, self.height])*FLIP_Y) - np.array([self.width, self.height])/2 return (zoomed * self.screen_size) + self.screen_center def components_to_colors(self, coords): while coords.shape[1] < 5: coords = np.concatenate([coords, -coords, coords], axis=1) return np.clip(np.int32(coords[...,3:6]*80/self.scale + 160), 50, 230) def update_screenpts(self): self.screenpts = self.components_to_screen(self.array) def update_colors(self): self.colors = self.default_colors def constrain_to_screen(self, points): return np.clip(points, np.int32([1., 1.]), np.int32([self.width-2, self.height-2])).T def distances_from_mouse(self, coords): mouse = np.int32([self.mouseX, self.mouseY]) offsets = self.screenpts - mouse return np.sqrt(np.sum(offsets*offsets, axis=1)) def get_nearest_point(self): return np.argmin(self.distances_from_mouse(self.array)) def select_nearest_point(self): self.selected_index = self.get_nearest_point() self.selectEvent(self.selected_index) def selected_vector(self): return self.array[self.selected_index] def selected_label(self): return self.labels[self.selected_index] def selectEvent(self, index): for layer in self.layers: layer.selectEvent(index) self.svdSelectEvent.emit() def focus_on_point(self, text): index = self.labels.index(text) if index is None: return coords = self.projection.components_to_projection(self.array[index]) if not self.is_point_on_screen(coords): self.screen_center = coords self.selected_index = index self.selectEvent(index) def paintEvent(self, event): if self.paint_lock.tryLock(): QWidget.paintEvent(self, event) self.painter.begin(self) self.painter.setRenderHint(QPainter.Antialiasing, True) self.painter.setRenderHint(QPainter.TextAntialiasing, True) try: for layer in self.layers: layer.draw(self.painter) finally: self.painter.end() self.paint_lock.unlock() def resizeEvent(self, sizeEvent): self.width = sizeEvent.size().width() self.height = sizeEvent.size().height() for layer in self.layers: layer.resize(self.width, self.height) def mouseMoveEvent(self, mouseEvent): point = mouseEvent.pos() self.mouseX = point.x() self.mouseY = point.y() for layer in self.layers: layer.mouseMoveEvent(mouseEvent) if self.leftMouseDown() or self.rightMouseDown(): self.activate_timer() elif self.timer_ticks >= TIMER_MAX: self.update() def timerEvent(self): self.projection.timerEvent() for layer in self.layers: layer.timerEvent() self.update_screenpts() self.update() self.age_timer() def updateMouseButtons(self, event): self.buttons = event.buttons() self.modifiers = event.modifiers() def leftMouseDown(self): ''' Checks if the left mouse button was pressed at the time of the last event. Right mouse clicks can be faked by holding a modifier (Ctrl on Macs, Logo key on Windows). ''' return (self.buttons & Qt.LeftButton) and not (self.modifiers & RIGHT_BUTTON_MODIFIER) def rightMouseDown(self): ''' Checks if the right mouse button was pressed at the time of the last event. Right mouse clicks can be faked by holding a modifier (Ctrl on Macs, Logo key on Windows). ''' return (self.buttons & Qt.RightButton) or (self.buttons & Qt.LeftButton and self.modifiers & RIGHT_BUTTON_MODIFIER) def mousePressEvent(self, mouseEvent): self.updateMouseButtons(mouseEvent) for layer in self.layers: layer.mousePressEvent(mouseEvent) self.activate_timer() def mouseReleaseEvent(self, mouseEvent): self.updateMouseButtons(mouseEvent) for layer in self.layers: layer.mouseReleaseEvent(mouseEvent) self.activate_timer() def wheelEvent(self, mouseEvent): for layer in self.layers: layer.wheelEvent(mouseEvent) self.activate_timer() def dropLabel(self, index, label): self.refreshData(self, index) def refreshData(self): self.update_screenpts() self.default_colors = self.components_to_colors(self.array)[:] self.update_colors() self.update() def get_svg_figure(self): figs = [] for layer in self.layers: fig = layer.drawSVG() if fig is not None: figs.append(fig) return svgfig.Fig(*figs) def write_svg(self, filename=None): if filename is None: filename = 'luminoso.svg' out = open(filename, 'w') out.write("<svg>\n") out.write(self.get_svg_figure().SVG().xml()) out.write("\n</svg>\n") out.close()
class ShapeMergeThread(QThread): def __init__(self, dir, shapes, inputEncoding, outputFileName, outputEncoding): QThread.__init__(self, QThread.currentThread()) self.baseDir = dir self.shapes = shapes self.inputEncoding = inputEncoding self.outputFileName = outputFileName self.outputEncoding = outputEncoding self.mutex = QMutex() self.stopMe = 0 def run(self): self.mutex.lock() self.stopMe = 0 self.mutex.unlock() interrupted = False # create attribute list with uniquie fields # from all selected layers mergedFields = [] self.emit(SIGNAL("rangeChanged( PyQt_PyObject )"), len(self.shapes)) self.emit(SIGNAL("checkStarted()")) shapeIndex = 0 fieldMap = {} for fileName in self.shapes: layerPath = QFileInfo(self.baseDir + "/" + fileName).absoluteFilePath() newLayer = QgsVectorLayer(layerPath, QFileInfo(layerPath).baseName(), "ogr") if not newLayer.isValid(): continue newLayer.setProviderEncoding(self.inputEncoding) vprovider = newLayer.dataProvider() fieldMap[shapeIndex] = {} fieldIndex = 0 for layerField in vprovider.fields(): fieldFound = False for mergedFieldIndex, mergedField in enumerate(mergedFields): if mergedField.name() == layerField.name() and mergedField.type() == layerField.type(): fieldFound = True fieldMap[shapeIndex][fieldIndex] = mergedFieldIndex if mergedField.length() < layerField.length(): # suit the field size to the field of this layer mergedField.setLength(layerField.length()) break if not fieldFound: fieldMap[shapeIndex][fieldIndex] = len(mergedFields) mergedFields.append(layerField) fieldIndex += 1 shapeIndex += 1 self.emit(SIGNAL("featureProcessed()")) self.emit(SIGNAL("checkFinished()")) # get information about shapefiles layerPath = QFileInfo(self.baseDir + "/" + self.shapes[0]).absoluteFilePath() newLayer = QgsVectorLayer(layerPath, QFileInfo(layerPath).baseName(), "ogr") self.crs = newLayer.crs() self.geom = newLayer.wkbType() vprovider = newLayer.dataProvider() fields = QgsFields() for f in mergedFields: fields.append(f) writer = QgsVectorFileWriter( self.outputFileName, self.outputEncoding, fields, self.geom, self.crs) shapeIndex = 0 for fileName in self.shapes: layerPath = QFileInfo(self.baseDir + "/" + fileName).absoluteFilePath() newLayer = QgsVectorLayer(layerPath, QFileInfo(layerPath).baseName(), "ogr") if not newLayer.isValid(): continue newLayer.setProviderEncoding(self.inputEncoding) vprovider = newLayer.dataProvider() nFeat = vprovider.featureCount() self.emit(SIGNAL("rangeChanged( PyQt_PyObject )"), nFeat) self.emit(SIGNAL("fileNameChanged( PyQt_PyObject )"), fileName) inFeat = QgsFeature() outFeat = QgsFeature() inGeom = QgsGeometry() fit = vprovider.getFeatures() while fit.nextFeature(inFeat): mergedAttrs = [""] * len(mergedFields) # fill available attributes with values fieldIndex = 0 for v in inFeat.attributes(): if shapeIndex in fieldMap and fieldIndex in fieldMap[shapeIndex]: mergedAttrs[fieldMap[shapeIndex][fieldIndex]] = v fieldIndex += 1 if inFeat.geometry() is not None: inGeom = QgsGeometry(inFeat.geometry()) outFeat.setGeometry(inGeom) outFeat.setAttributes(mergedAttrs) writer.addFeature(outFeat) self.emit(SIGNAL("featureProcessed()")) self.emit(SIGNAL("shapeProcessed()")) self.mutex.lock() s = self.stopMe self.mutex.unlock() if s == 1: interrupted = True break shapeIndex += 1 del writer if not interrupted: self.emit(SIGNAL("processingFinished()")) else: self.emit(SIGNAL("processingInterrupted()")) def stop(self): self.mutex.lock() self.stopMe = 1 self.mutex.unlock() QThread.wait(self)
class Search_worker(QThread): """ Defines a `QThread <http://doc.qt.nokia.com/qthread.html>`_ subclass used to search for a pattern in a directory files. """ # Custom signals definitions. searchFinished = pyqtSignal(list) """ This signal is emited by the :class:`Search_worker` class when the search is finished. ( pyqtSignal ) :return: Search results. :rtype: list """ def __init__(self, parent, pattern=None, location=None, settings=None): """ Initializes the class. :param parent: Object parent. :type parent: QObject """ LOGGER.debug("> Initializing '{0}()' class.".format(self.__class__.__name__)) QThread.__init__(self, parent) # --- Setting class attributes. --- self.__container = parent self.__pattern = None self.pattern = pattern self.__location = None self.location = location self.__settings = None self.settings = settings self.__searchResults = None self.__interrupt = False self.__lock = QMutex() #****************************************************************************************************************** #*** Attributes properties. #****************************************************************************************************************** @property def container(self): """ Property for **self.__container** attribute. :return: self.__container. :rtype: QObject """ return self.__container @container.setter @foundations.exceptions.handleExceptions(foundations.exceptions.ProgrammingError) def container(self, value): """ Setter for **self.__container** attribute. :param value: Attribute value. :type value: QObject """ raise foundations.exceptions.ProgrammingError( "{0} | '{1}' attribute is read only!".format(self.__class__.__name__, "container")) @container.deleter @foundations.exceptions.handleExceptions(foundations.exceptions.ProgrammingError) def container(self): """ Deleter for **self.__container** attribute. """ raise foundations.exceptions.ProgrammingError( "{0} | '{1}' attribute is not deletable!".format(self.__class__.__name__, "container")) @property def pattern(self): """ Property for **self.__pattern** attribute. :return: self.__pattern. :rtype: unicode """ return self.__pattern @pattern.setter @foundations.exceptions.handleExceptions(AssertionError) def pattern(self, value): """ Setter for **self.__pattern** attribute. :param value: Attribute value. :type value: unicode """ if value is not None: assert type(value) in (unicode, QString), \ "'{0}' attribute: '{1}' type is not 'unicode' or 'QString'!".format("pattern", value) self.__pattern = value @pattern.deleter @foundations.exceptions.handleExceptions(foundations.exceptions.ProgrammingError) def pattern(self): """ Deleter for **self.__pattern** attribute. """ raise foundations.exceptions.ProgrammingError( "{0} | '{1}' attribute is not deletable!".format(self.__class__.__name__, "pattern")) @property def location(self): """ Property for **self.__location** attribute. :return: self.__location. :rtype: Location """ return self.__location @location.setter @foundations.exceptions.handleExceptions(AssertionError) def location(self, value): """ Setter for **self.__location** attribute. :param value: Attribute value. :type value: Location """ if value is not None: assert type(value) is umbra.ui.common.Location, \ "'{0}' attribute: '{1}' type is not 'umbra.ui.common.Location'!".format("location", value) self.__location = value @location.deleter @foundations.exceptions.handleExceptions(foundations.exceptions.ProgrammingError) def location(self): """ Deleter for **self.__location** attribute. """ raise foundations.exceptions.ProgrammingError( "{0} | '{1}' attribute is not deletable!".format(self.__class__.__name__, "location")) @property def settings(self): """ Property for **self.__settings** attribute. :return: self.__settings. :rtype: Location """ return self.__settings @settings.setter @foundations.exceptions.handleExceptions(AssertionError) def settings(self, value): """ Setter for **self.__settings** attribute. :param value: Attribute value. :type value: Location """ if value is not None: assert type(value) is dict, "'{0}' attribute: '{1}' type is not 'dict'!".format("settings", value) self.__settings = foundations.dataStructures.Structure(**{"caseSensitive" : False, "wholeWord" : False, "regularExpressions" : False}) self.__settings.update(value) @settings.deleter @foundations.exceptions.handleExceptions(foundations.exceptions.ProgrammingError) def settings(self): """ Deleter for **self.__settings** attribute. """ raise foundations.exceptions.ProgrammingError( "{0} | '{1}' attribute is not deletable!".format(self.__class__.__name__, "settings")) @property def searchResults(self): """ Property for **self.__searchResults** attribute. :return: self.__searchResults. :rtype: list """ return self.__searchResults @searchResults.setter @foundations.exceptions.handleExceptions(foundations.exceptions.ProgrammingError) def searchResults(self, value): """ Setter for **self.__searchResults** attribute. :param value: Attribute value. :type value: list """ raise foundations.exceptions.ProgrammingError( "{0} | '{1}' attribute is read only!".format(self.__class__.__name__, "searchResults")) @searchResults.deleter @foundations.exceptions.handleExceptions(foundations.exceptions.ProgrammingError) def searchResults(self): """ Deleter for **self.__searchResults** attribute. """ raise foundations.exceptions.ProgrammingError( "{0} | '{1}' attribute is not deletable!".format(self.__class__.__name__, "searchResults")) #****************************************************************************************************************** #*** Class methods. #****************************************************************************************************************** def run(self): """ Reimplements the :meth:`QThread.run` method. """ self.__search() def quit(self): """ Reimplements the :meth:`QThread.quit` method. """ self.__interrupt = True QThread.quit(self) def __search(self): """ Performs the search. """ self.__searchResults = [] editorsFiles = self.__container.defaultTarget in self.__location.targets and \ [editor.file for editor in self.__container.scriptEditor.listEditors()] or [] self.__searchEditorsFiles(editorsFiles) self.__searchFiles(self.__location.files) for directory in self.__location.directories: if self.__interrupt: return filesWalker = foundations.walkers.filesWalker(directory, self.__location.filtersIn, list(itertools.chain(self.__location.filtersOut, self.__location.files, editorsFiles))) self.__searchFiles(filesWalker) not self.__interrupt and self.searchFinished.emit(self.__searchResults) def __searchEditorsFiles(self, files): """ Searches in :class:`umbra.components.factory.scriptEditor.scriptEditor.ScriptEditor` class editors files. :param files: Editor files. :type files: list """ for file in files: if self.__interrupt: return if foundations.io.isReadable(file): if foundations.io.isBinaryFile(file): continue LOGGER.info("{0} | Searching '{1}' file!".format(self.__class__.__name__, file)) editor = self.__container.scriptEditor.getEditor(file) if not editor: continue self.__lock.lock() occurrences = self.__searchDocument(editor.document(), self.__pattern, self.__settings) self.__lock.unlock() occurrences and self.__searchResults.append(SearchResult(file=file, pattern=self.__pattern, settings=self.__settings, occurrences=occurrences)) def __searchFiles(self, files): """ Searches in given files. :param files: Files. :type files: list """ for file in files: if self.__interrupt: return if not foundations.common.pathExists(file): continue if foundations.io.isReadable(file): if foundations.io.isBinaryFile(file): continue LOGGER.info("{0} | Searching '{1}' file!".format(self.__class__.__name__, file)) cacheData = self.__container.filesCache.getContent(file) if not cacheData: reader = foundations.io.File(file) content = reader.read() if content is None: LOGGER.warning("!> Error occured while reading '{0}' file proceeding to next one!".format(file)) continue self.__container.filesCache.addContent(**{file : CacheData(content=content, document=None)}) else: content = cacheData.content occurrences = self.__searchDocument(QTextDocument(QString(content)), self.__pattern, self.__settings) occurrences and self.__searchResults.append(SearchResult(file=file, pattern=self.__pattern, settings=self.__settings, occurrences=occurrences)) def __searchDocument(self, document, pattern, settings): """ Searches for given pattern occurrences in given document using given settings. :param document: Document. :type document: QTextDocument :param pattern: Pattern. :type pattern: unicode :param settings: Search settings. :type settings: Structure :return: Matched occurrences. :rtype: list """ pattern = settings.regularExpressions and QRegExp(pattern) or pattern flags = QTextDocument.FindFlags() if settings.caseSensitive: flags = flags | QTextDocument.FindCaseSensitively if settings.wholeWord: flags = flags | QTextDocument.FindWholeWords occurrences = [] block = document.findBlock(0) cursor = document.find(pattern, block.position(), flags) while block.isValid() and cursor.position() != -1: if self.__interrupt: return blockCursor = QTextCursor(cursor) blockCursor.movePosition(QTextCursor.StartOfLine, QTextCursor.MoveAnchor) blockCursor.movePosition(QTextCursor.EndOfLine, QTextCursor.KeepAnchor) length = cursor.selectionEnd() - cursor.selectionStart() occurrences.append(Occurence(line=cursor.blockNumber(), column=cursor.columnNumber() - length, length=length, position=cursor.position() - length, text=blockCursor.selectedText())) cursor = document.find(pattern, cursor.position(), flags) block = block.next() return occurrences
class SplitThread(QThread): def __init__(self, layer, splitField, encoding, outDir): QThread.__init__(self, QThread.currentThread()) self.layer = layer self.field = splitField self.encoding = encoding self.outDir = outDir self.mutex = QMutex() self.stopMe = 0 self.errors = [] def run(self): self.mutex.lock() self.stopMe = 0 self.mutex.unlock() interrupted = False outPath = self.outDir if outPath.find("\\") != -1: outPath.replace("\\", "/") if not outPath.endswith("/"): outPath = outPath + "/" provider = self.layer.dataProvider() index = provider.fieldNameIndex(self.field) unique = ftools_utils.getUniqueValues(provider, int(index)) baseName = unicode( outPath + self.layer.name() + "_" + self.field + "_" ) fieldList = ftools_utils.getFieldList(self.layer) sRs = provider.crs() geom = self.layer.wkbType() inFeat = QgsFeature() self.emit(SIGNAL("rangeCalculated(PyQt_PyObject)"), len(unique)) for i in unique: check = QFile(baseName + "_" + unicode(i).strip() + ".shp") fName = check.fileName() if check.exists(): if not QgsVectorFileWriter.deleteShapeFile(fName): self.errors.append( fName ) continue writer = QgsVectorFileWriter(fName, self.encoding, fieldList, geom, sRs) fit = provider.getFeatures() while fit.nextFeature(inFeat): atMap = inFeat.attributes() if atMap[index] == i: writer.addFeature(inFeat) del writer self.emit(SIGNAL("valueProcessed()")) self.mutex.lock() s = self.stopMe self.mutex.unlock() if s == 1: interrupted = True break if not interrupted: self.emit(SIGNAL("processFinished( PyQt_PyObject )"), self.errors) else: self.emit(SIGNAL("processInterrupted()")) def stop(self): self.mutex.lock() self.stopMe = 1 self.mutex.unlock() QThread.wait(self)
class BeamshutterWidget(QWidget): currentShutterState = pyqtSignal(str, name='currentShutterState') def __init__(self, parent=None): QWidget.__init__(self, parent) self.ui = uic.loadUi("Beamshutter.ui", self) self.control = QAxContainer.QAxWidget(self) self.control.setControl('{3CE35BF3-1E13-4D2C-8C0B-DEF6314420B3}') self.control.setProperty("HWSerialNum", 85845031) self.control.setGeometry(0, 0, 600, 400) self.mutex = QMutex(mode=QMutex.Recursive) self.shutterReplied = False self.shut = Beamshutter(self) self.worker = WorkerThread() self.ui.mainLayout.addWidget(self.control) self.layout().setAlignment(Qt.AlignCenter) sizePolicy = QSizePolicy(QSizePolicy.Preferred, QSizePolicy.Preferred) sizePolicy.setHeightForWidth(True) self.setSizePolicy(sizePolicy) def heightForWidth(self, width): return width * 0.75 @pyqtSlot() def on_BInitializeShutter_clicked(self): if not self.LEDShutter.isChecked(): self.initialize() else: self.finalize() def initialize(self): with self.check(timeOut=10): self.worker.executeFunction(self.shut.initialize) self.ui.LEDShutter.setChecked(True) def finalize(self): with self.check(): self.worker.executeFunction(self.shut.finalize) self.LEDShutter.setChecked(False) def enable(self): with self.check(): self.worker.executeFunction(self.shut.enable) def disable(self): with self.check(): self.worker.executeFunction(self.shut.disable) def _del(self): if self.ui.LEDShutter.isChecked(): self.finalize() def heightForWidth(self, w): return (w) @contextmanager def check(self, timeOut=2): out = datetime.datetime.now() + datetime.timedelta(seconds=timeOut) self.shutterReplied = shutterReplied = False yield while datetime.datetime.now() < out and shutterReplied == False: self.mutex.lock() shutterReplied = self.shutterReplied self.mutex.unlock() if shutterReplied == False: raise Exception("ThorlabsShutter timed out and might be Crashed")
class StdErrWrapper(QObject): _write = pyqtSignal(type(u'')) _flush = pyqtSignal() def __init__(self, old_stderr): QObject.__init__(self) self._old_stderr = old_stderr self._widget = None self._mutex = QMutex(QMutex.Recursive) def setApplication(self, app): assert (self._widget is None) widget = QPlainTextEdit() widget.setWindowTitle(u"Error Console") widget.resize(486, 300) widget.appendHtml( u'<span style="color: green">' u'An unhandled error occurred.<br>' u'Sorry for the inconvinience.<br>' u'Please copy the following text into a bug report:<br><br>' u'</span>') app.aboutToQuit.connect(self.restoreStdErr) self._write.connect(self._write_handler) self._flush.connect(self._flush_handler) self._widget = widget def _write_handler(self, data): self._mutex.lock() if self._widget: self._widget.show() self._widget.insertPlainText(data) self._mutex.unlock() def _flush_handler(self): self._mutex.lock() if self._widget: self._widget.show() self._mutex.unlock() def restoreStdErr(self): self._mutex.lock() if self._widget: self._widget.close() self._widget = None self._mutex.unlock() @property def encoding(self): return 'utf-8' def write(self, s): self._mutex.lock() if self._widget: if isinstance(s, bytes): s = s.decode('utf-8', 'replace') self._write.emit(s) else: self._old_stderr.write(s) self._mutex.unlock() def flush(self): self._mutex.lock() if self._widget: self._flush.emit() else: self._old_stderr.flush() self._mutex.unlock()
class MessageReceiver: """" Checks given rx buffer if message is present there. struct Tail { uint8_t tail_start = TAIL_START_MARK; // '<' uint16_t id = 0; uint16_t context = 0; uint16_t msg_len = 0; uint16_t body_crc= 0; uint8_t tail_end = TAIL_END_MARK; // '>' } uint16_t tail_crc= 0; """ TAIL_LEN = 10 TAIL_START_MARK = ord('<') TAIL_END_MARK = ord('>') TAIL_CRC_SHIFT_POS = 3 ts = time.time() LOCKED = False def __init__(self, rx_buffer): self.rx_buffer = rx_buffer self.__mean_rx_time = MeanCalculator() self.mutex = QMutex() self.t0 = time.time() def check_tail(self, peek_buff): init_find = peek_buff.find(MessageReceiver.TAIL_START_MARK) peek_buff_len = len(peek_buff) for i in range(peek_buff_len - MessageReceiver.TAIL_LEN + init_find): latest_find = i + init_find try: tail_start_mark_pos = peek_buff[latest_find:].find( MessageReceiver.TAIL_START_MARK) tail_end_mark_pos = tail_start_mark_pos + MessageReceiver.TAIL_LEN - 1 tail_end_mark = peek_buff[latest_find:][tail_end_mark_pos] except IndexError: return False try: if tail_end_mark == MessageReceiver.TAIL_END_MARK: peek_buff = peek_buff[latest_find:] tail = peek_buff[tail_start_mark_pos:tail_end_mark_pos] _full_tail = peek_buff[ tail_start_mark_pos:tail_end_mark_pos + 1] _id = struct.unpack('H', tail[1:3])[0] _context = struct.unpack('H', tail[3:5])[0] _msg_len = struct.unpack('H', tail[5:7])[0] _body_crc = tail[7:9] _tail_crc = peek_buff[tail_end_mark_pos + 1:tail_end_mark_pos + MessageReceiver.TAIL_CRC_SHIFT_POS] tail_integrity = _tail_crc == crc_bytes( _full_tail) # tail integrity check if _id < len(RxMessage.rx_id_tuple) and _msg_len < MAX_PACKET_SIZE and _context < 0xffff \ and self.rx_buffer.available() > _msg_len and tail_integrity: return _id, _context, _msg_len, _body_crc, tail_start_mark_pos + latest_find, tail_end_mark_pos + latest_find except struct.error: return False return False def get_message(self): t0 = time.time() ret_rxmsg = None if self.rx_buffer.available() >= MessageReceiver.TAIL_LEN: self.mutex.lock() peek_buff = self.rx_buffer.peek() check_tail_result = self.check_tail(peek_buff) self.mutex.unlock() if check_tail_result: _id, _context, _msg_len, _crc, tail_start_mark_pos, tail_end_mark_pos = check_tail_result self.mutex.lock() msg_body = self.rx_buffer.read(tail_end_mark_pos + 1 + 2) self.mutex.unlock() msg_body = msg_body[:tail_start_mark_pos] MessageReceiver.ts = time.time() crc_check = RxMessage.RxId.ack if _crc == crc_bytes( msg_body) else RxMessage.RxId.nack rxmsg = RxMessage(msg_id=_id, crc_check=crc_check, length=len(msg_body), context=_context, body=msg_body) m_logger.debug(MSG_RX_DBG_TEMPLATE.format(rxmsg)) self.t0 = time.time() if _crc == crc_bytes(msg_body): self.__mean_rx_time.count(time.time() - t0) ret_rxmsg = rxmsg m_logger.debug("Mean msg extract time: {}".format( self.__mean_rx_time)) m_logger.debug("Period: {}".format(time.time() - self.t0)) return ret_rxmsg
class SiteMapThread(QThread): def __init__(self, framework, treeViewModel, parent=None): QThread.__init__(self, parent) self.framework = framework self.treeViewModel = treeViewModel self.qtimer = QTimer() self.qlock = QMutex() self.fillAll = False QObject.connect(self, SIGNAL("quit()"), self.quitHandler) QObject.connect(self, SIGNAL("started()"), self.startedHandler) self.re_set_cookie = re.compile(r"^Set-Cookie2?:\s*(.+)$", re.I | re.M) self.re_host_name = re.compile(r"^Host:\s*(.+?)$", re.I | re.M) self.lastId = 0 self.Data = None self.cursor = None def db_attach(self): self.Data = self.framework.getDB() self.cursor = self.Data.allocate_thread_cursor() self.populateSiteMap(True) def db_detach(self): self.close_cursor() self.Data = None def close_cursor(self): if self.cursor and self.Data: self.cursor.close() self.Data.release_thread_cursor(self.cursor) self.cursor = None def run(self): QObject.connect(self, SIGNAL("updateSiteMap()"), self.doUpdateSiteMap, Qt.DirectConnection) self.framework.subscribe_response_data_added(self.doUpdateSiteMap) self.exec_() def quitHandler(self): self.framework.debug_log("SiteMapThread quit...") self.close_cursor() self.exit(0) def startedHandler(self): self.framework.debug_log("SiteMapThread started...") self.framework.subscribe_database_events(self.db_attach, self.db_detach) def populateSiteMap(self, fillAll): self.fillAll = fillAll QTimer.singleShot(50, self, SIGNAL("updateSiteMap()")) def doUpdateSiteMap(self): if not self.qlock.tryLock(): return try: if self.fillAll: self.fillAll = False self.treeViewModel.clearModel() self.lastId = 0 rows = self.Data.get_sitemap_info(self.cursor, self.lastId) global_cookie_jar = self.framework.get_global_cookie_jar() count = 0 for row in rows: count += 1 if 0 == (count % 100): self.yieldCurrentThread() rowItems = [m or "" for m in list(row)] Id = str(rowItems[0]) try: self.lastId = int(Id) except ValueError: pass # XXX: review all for bytes usage if isinstance(rowItems[1], bytes): url = str(rowItems[1], "utf-8", "ignore") else: url = str(rowItems[1]) status = str(rowItems[2]) response_headers = str(rowItems[3]) request_headers = str(rowItems[4]) # TODO: make configurable if status in ("400", "404", "500", "501"): continue # TODO: m = self.re_set_cookie.search(response_headers) if m: setCookies = m.group(1) cookieList = QNetworkCookie.parseCookies(setCookies) global_cookie_jar.setCookiesFromUrl(cookieList, QUrl.fromEncoded(url)) parsed = urlparse.urlsplit(url) hostname = "" if not parsed.hostname: m = self.re_host_name.search(request_headers) if m: hostname = m.group(1).rstrip() else: hostname = parsed.hostname hostname = hostname.lower() hostloc = urlparse.urlunsplit((parsed.scheme, parsed.netloc, "/", "", "")) rootNode = self.treeViewModel.findOrAddNode(hostname) hostLocNode = rootNode.findOrAddNode(self.treeViewModel, hostloc) pathval = parsed.path # add directories parentNode = hostLocNode parentNode.setResponseId(None, hostloc) lastSlash = 0 slash = 0 while True: slash = pathval.find("/", slash + 1) if slash < 0: break dirname = pathval[lastSlash + 1 : slash + 1] parentNode = parentNode.findOrAddNode(self.treeViewModel, dirname) parentNode.setResponseId( None, urlparse.urlunsplit((parsed.scheme, parsed.netloc, pathval[0 : slash + 1], "", "")) ) lastSlash = slash # add file element if lastSlash + 1 < len(pathval): filename = pathval[lastSlash + 1 :] parentNode = parentNode.findOrAddNode(self.treeViewModel, filename) parentNode.setResponseId(None, urlparse.urlunsplit((parsed.scheme, parsed.netloc, pathval, "", ""))) # add query if parsed.query: parentNode = parentNode.findOrAddNode(self.treeViewModel, "?" + parsed.query) # store the latest Id # TODO: should determine best candidate to display parentNode.setResponseId(Id, url) finally: self.qlock.unlock()
class Walker(QThread): COMMON_WORDS_THRESHOLD = 250 MIN_WORD_LEN = 3 MAX_WORD_LEN = 25 INVALID_FIRST_OR_LAST = frozenset("0123456789_") STRIPHTML_RE = re.compile(r"<[^>]*?>", re.IGNORECASE | re.MULTILINE) ENTITY_RE = re.compile(r"&(\w+?);|&#(\d+?);") SPLIT_RE = re.compile(r"\W+", re.IGNORECASE | re.MULTILINE) def __init__(self, lock, parent=None): super(Walker, self).__init__(parent) self.lock = lock self.stopped = False self.mutex = QMutex() self.path = None self.completed = False def initialize(self, path, filenamesForWords, commonWords): self.stopped = False self.path = path self.filenamesForWords = filenamesForWords self.commonWords = commonWords self.completed = False def stop(self): try: self.mutex.lock() self.stopped = True finally: self.mutex.unlock() def isStopped(self): try: self.mutex.lock() return self.stopped finally: self.mutex.unlock() def run(self): self.processFiles(self.path) self.stop() self.emit(SIGNAL("finished(bool)"), self.completed) def processFiles(self, path): def unichrFromEntity(match): text = match.group(match.lastindex) if text.isdigit(): return chr(int(text)) u = html.entities.name2codepoint.get(text) return chr(u) if u is not None else "" for root, dirs, files in os.walk(path): if self.isStopped(): return for name in [ name for name in files if name.endswith((".htm", ".html")) ]: fname = os.path.join(root, name) if self.isStopped(): return words = set() fh = None try: fh = codecs.open(fname, "r", "UTF8", "ignore") text = fh.read() except EnvironmentError as e: sys.stderr.write("Error: {0}\n".format(e)) continue finally: if fh is not None: fh.close() if self.isStopped(): return text = self.STRIPHTML_RE.sub("", text) text = self.ENTITY_RE.sub(unichrFromEntity, text) text = text.lower() for word in self.SPLIT_RE.split(text): if (self.MIN_WORD_LEN <= len(word) <= self.MAX_WORD_LEN and word[0] not in self.INVALID_FIRST_OR_LAST and word[-1] not in self.INVALID_FIRST_OR_LAST): try: self.lock.lockForRead() new = word not in self.commonWords finally: self.lock.unlock() if new: words.add(word) if self.isStopped(): return for word in words: try: self.lock.lockForWrite() files = self.filenamesForWords[word] if len(files) > self.COMMON_WORDS_THRESHOLD: del self.filenamesForWords[word] self.commonWords.add(word) else: files.add(str(fname)) finally: self.lock.unlock() self.emit(SIGNAL("indexed(QString)"), fname) self.completed = True
class Walker(QThread): COMMON_WORDS_THRESHOLD = 250 MIN_WORD_LEN = 3 MAX_WORD_LEN = 25 INVALID_FIRST_OR_LAST = frozenset("0123456789_") STRIPHTML_RE = re.compile(r"<[^>]*?>", re.IGNORECASE|re.MULTILINE) ENTITY_RE = re.compile(r"&(\w+?);|&#(\d+?);") SPLIT_RE = re.compile(r"\W+", re.IGNORECASE|re.MULTILINE) def __init__(self, lock, parent=None): super(Walker, self).__init__(parent) self.lock = lock self.stopped = False self.mutex = QMutex() self.path = None self.completed = False def initialize(self, path, filenamesForWords, commonWords): self.stopped = False self.path = path self.filenamesForWords = filenamesForWords self.commonWords = commonWords self.completed = False def stop(self): try: self.mutex.lock() self.stopped = True finally: self.mutex.unlock() def isStopped(self): try: self.mutex.lock() return self.stopped finally: self.mutex.unlock() def run(self): self.processFiles(self.path) self.stop() self.emit(SIGNAL("finished(bool)"), self.completed) def processFiles(self, path): def unichrFromEntity(match): text = match.group(match.lastindex) if text.isdigit(): return unichr(int(text)) u = htmlentitydefs.name2codepoint.get(text) return unichr(u) if u is not None else "" for root, dirs, files in os.walk(path): if self.isStopped(): return for name in [name for name in files if name.endswith((".htm", ".html"))]: fname = os.path.join(root, name) if self.isStopped(): return words = set() fh = None try: fh = codecs.open(fname, "r", "UTF8", "ignore") text = fh.read() except (IOError, OSError), e: sys.stderr.write("Error: {0}\n".format(e)) continue finally: if fh is not None: fh.close()
class VcsStatusMonitorThread(QThread): """ Class implementing the VCS status monitor thread base class. @signal vcsStatusMonitorData(QStringList) emitted to update the VCS status @signal vcsStatusMonitorStatus(QString, QString) emitted to signal the status of the monitoring thread (ok, nok, op) and a status message """ def __init__(self, interval, projectDir, vcs, parent = None): """ Constructor @param interval new interval in seconds (integer) @param projectDir project directory to monitor (string or QString) @param vcs reference to the version control object @param parent reference to the parent object (QObject) """ QThread.__init__(self, parent) self.setObjectName("VcsStatusMonitorThread") self.setTerminationEnabled(True) self.projectDir = QString(projectDir) self.vcs = vcs self.interval = interval self.autoUpdate = False self.statusList = QStringList() self.reportedStates = {} self.shouldUpdate = False self.monitorMutex = QMutex() self.monitorCondition = QWaitCondition() self.__stopIt = False def run(self): """ Protected method implementing the tasks action. """ while not self.__stopIt: # perform the checking task self.statusList.clear() self.emit(SIGNAL("vcsStatusMonitorStatus(QString, QString)"), QString("wait"), self.trUtf8("Waiting for lock")) try: locked = self.vcs.vcsExecutionMutex.tryLock(5000) except TypeError: locked = self.vcs.vcsExecutionMutex.tryLock() if locked: try: self.emit(SIGNAL("vcsStatusMonitorStatus(QString, QString)"), QString("op"), self.trUtf8("Checking repository status")) res, statusMsg = self._performMonitor() finally: self.vcs.vcsExecutionMutex.unlock() if res: status = QString("ok") else: status = QString("nok") self.emit(SIGNAL("vcsStatusMonitorStatus(QString, QString)"), QString("send"), self.trUtf8("Sending data")) self.emit(SIGNAL("vcsStatusMonitorData(QStringList)"), QStringList(self.statusList)) self.emit(SIGNAL("vcsStatusMonitorStatus(QString, QString)"), status, statusMsg) else: self.emit(SIGNAL("vcsStatusMonitorStatus(QString, QString)"), QString("timeout"), self.trUtf8("Timed out waiting for lock")) if self.autoUpdate and self.shouldUpdate: try: self.vcs.vcsUpdate(self.projectDir, True) continue # check again except TypeError: pass # compatibility for older VCS plugins self.shouldUpdate = False # wait until interval has expired checking for a stop condition self.monitorMutex.lock() if not self.__stopIt: self.monitorCondition.wait(self.monitorMutex, self.interval * 1000) self.monitorMutex.unlock() self.exit() def setInterval(self, interval): """ Public method to change the monitor interval. @param interval new interval in seconds (integer) """ locked = self.monitorMutex.tryLock() self.interval = interval self.monitorCondition.wakeAll() if locked: self.monitorMutex.unlock() def getInterval(self): """ Public method to get the monitor interval. @return interval in seconds (integer) """ return self.interval def setAutoUpdate(self, auto): """ Public method to enable the auto update function. @param auto status of the auto update function (boolean) """ self.autoUpdate = auto def getAutoUpdate(self): """ Public method to retrieve the status of the auto update function. @return status of the auto update function (boolean) """ return self.autoUpdate def checkStatus(self): """ Public method to wake up the status monitor thread. """ locked = self.monitorMutex.tryLock() self.monitorCondition.wakeAll() if locked: self.monitorMutex.unlock() def stop(self): """ Public method to stop the monitor thread. """ locked = self.monitorMutex.tryLock() self.__stopIt = True self.monitorCondition.wakeAll() if locked: self.monitorMutex.unlock() def clearCachedState(self, name): """ Public method to clear the cached VCS state of a file/directory. @param name name of the entry to be cleared (QString or string) """ project = e4App().getObject("Project") key = project.getRelativePath(unicode(name)) try: del self.reportedStates[key] except KeyError: pass def _performMonitor(self): """ Protected method implementing the real monitoring action. This method must be overridden and populate the statusList member variable with a list of strings giving the status in the first column and the path relative to the project directory starting with the third column. The allowed status flags are: <ul> <li>"A" path was added but not yet comitted</li> <li>"M" path has local changes</li> <li>"O" path was removed</li> <li>"R" path was deleted and then re-added</li> <li>"U" path needs an update</li> <li>"Z" path contains a conflict</li> <li>" " path is back at normal</li> </ul> @return tuple of flag indicating successful operation (boolean) and a status message in case of non successful operation (QString) """ raise RuntimeError('Not implemented')