def __init__(self, engine, dao, max_file_processors=5): ''' Constructor ''' super(QueueManager, self).__init__() self._dao = dao self._engine = engine self._local_folder_queue = Queue() self._local_file_queue = Queue() self._remote_file_queue = Queue() self._remote_folder_queue = Queue() self._connected = local() self._local_folder_enable = True self._local_file_enable = True self._remote_folder_enable = True self._remote_file_enable = True self._local_folder_thread = None self._local_file_thread = None self._remote_folder_thread = None self._remote_file_thread = None self._error_threshold = ERROR_THRESHOLD self._error_interval = 60 self.set_max_processors(max_file_processors) self._threads_pool = list() self._processors_pool = list() self._get_file_lock = Lock() # Should not operate on thread while we are inspecting them ''' This error required to add a lock for inspecting threads, as the below Traceback shows the processor thread was ended while the method was running Traceback (most recent call last): File "/Users/hudson/tmp/workspace/FT-nuxeo-drive-master-osx/nuxeo-drive-client/nxdrive/engine/watcher/local_watcher.py", line 845, in handle_watchdog_event self.scan_pair(rel_path) File "/Users/hudson/tmp/workspace/FT-nuxeo-drive-master-osx/nuxeo-drive-client/nxdrive/engine/watcher/local_watcher.py", line 271, in scan_pair self._suspend_queue() File "/Users/hudson/tmp/workspace/FT-nuxeo-drive-master-osx/nuxeo-drive-client/nxdrive/engine/watcher/local_watcher.py", line 265, in _suspend_queue for processor in self._engine.get_queue_manager().get_processors_on('/', exact_match=False): File "/Users/hudson/tmp/workspace/FT-nuxeo-drive-master-osx/nuxeo-drive-client/nxdrive/engine/queue_manager.py", line 413, in get_processors_on res.append(self._local_file_thread.worker) AttributeError: 'NoneType' object has no attribute 'worker' ''' self._thread_inspection = Lock() # ERROR HANDLING self._error_lock = Lock() self._on_error_queue = BlacklistQueue(delay=DEFAULT_DELAY) self._error_timer = QTimer() # TODO newErrorGiveUp signal is not connected self._error_timer.timeout.connect(self._on_error_timer) self.newError.connect(self._on_new_error) self.queueProcessing.connect(self.launch_processors) # LAST ACTION self._dao.register_queue_manager(self)
def __init__(self, engine, dao, max_file_processors=(0, 0, 5)): ''' Constructor ''' super(QueueManager, self).__init__() self._dao = dao self._engine = engine self._local_folder_queue = Queue() self._local_file_queue = Queue() self._remote_file_queue = Queue() self._remote_folder_queue = Queue() self._connected = local() self._local_folder_enable = True self._local_file_enable = True self._remote_folder_enable = True self._remote_file_enable = True self._local_folder_thread = None self._local_file_thread = None self._remote_folder_thread = None self._remote_file_thread = None self._error_threshold = ERROR_THRESHOLD self._error_interval = 60 self._max_local_processors = 0 self._max_remote_processors = 0 self._max_generic_processors = 0 self.set_max_processors(max_file_processors) self._threads_pool = list() self._processors_pool = list() self._get_file_lock = Lock() # Should not operate on thread while we are inspecting them ''' This error required to add a lock for inspecting threads, as the below Traceback shows the processor thread was ended while the method was running Traceback (most recent call last): File "/Users/hudson/tmp/workspace/FT-nuxeo-drive-master-osx/nuxeo-drive-client/nxdrive/engine/watcher/local_watcher.py", line 845, in handle_watchdog_event self.scan_pair(rel_path) File "/Users/hudson/tmp/workspace/FT-nuxeo-drive-master-osx/nuxeo-drive-client/nxdrive/engine/watcher/local_watcher.py", line 271, in scan_pair self._suspend_queue() File "/Users/hudson/tmp/workspace/FT-nuxeo-drive-master-osx/nuxeo-drive-client/nxdrive/engine/watcher/local_watcher.py", line 265, in _suspend_queue for processor in self._engine.get_queue_manager().get_processors_on('/', exact_match=False): File "/Users/hudson/tmp/workspace/FT-nuxeo-drive-master-osx/nuxeo-drive-client/nxdrive/engine/queue_manager.py", line 413, in get_processors_on res.append(self._local_file_thread.worker) AttributeError: 'NoneType' object has no attribute 'worker' ''' self._thread_inspection = Lock() # ERROR HANDLING self._error_lock = Lock() self._on_error_queue = BlacklistQueue(delay=DEFAULT_DELAY) self._error_timer = QTimer() # TODO newErrorGiveUp signal is not connected self._error_timer.timeout.connect(self._on_error_timer) self.newError.connect(self._on_new_error) self.queueProcessing.connect(self.launch_processors) # LAST ACTION self._dao.register_queue_manager(self)
class QueueManager(QObject): # Always create thread from the main thread newItem = pyqtSignal(object) newError = pyqtSignal(object) newErrorGiveUp = pyqtSignal(object) queueEmpty = pyqtSignal() queueProcessing = pyqtSignal() queueFinishedProcessing = pyqtSignal() # Only used by Unit Test _disable = False ''' classdocs ''' def __init__(self, engine, dao, max_file_processors=(0, 0, 5)): ''' Constructor ''' super(QueueManager, self).__init__() self._dao = dao self._engine = engine self._local_folder_queue = Queue() self._local_file_queue = Queue() self._remote_file_queue = Queue() self._remote_folder_queue = Queue() self._connected = local() self._local_folder_enable = True self._local_file_enable = True self._remote_folder_enable = True self._remote_file_enable = True self._local_folder_thread = None self._local_file_thread = None self._remote_folder_thread = None self._remote_file_thread = None self._error_threshold = ERROR_THRESHOLD self._error_interval = 60 self._max_local_processors = 0 self._max_remote_processors = 0 self._max_generic_processors = 0 self.set_max_processors(max_file_processors) self._threads_pool = list() self._processors_pool = list() self._get_file_lock = Lock() # Should not operate on thread while we are inspecting them ''' This error required to add a lock for inspecting threads, as the below Traceback shows the processor thread was ended while the method was running Traceback (most recent call last): File "/Users/hudson/tmp/workspace/FT-nuxeo-drive-master-osx/nuxeo-drive-client/nxdrive/engine/watcher/local_watcher.py", line 845, in handle_watchdog_event self.scan_pair(rel_path) File "/Users/hudson/tmp/workspace/FT-nuxeo-drive-master-osx/nuxeo-drive-client/nxdrive/engine/watcher/local_watcher.py", line 271, in scan_pair self._suspend_queue() File "/Users/hudson/tmp/workspace/FT-nuxeo-drive-master-osx/nuxeo-drive-client/nxdrive/engine/watcher/local_watcher.py", line 265, in _suspend_queue for processor in self._engine.get_queue_manager().get_processors_on('/', exact_match=False): File "/Users/hudson/tmp/workspace/FT-nuxeo-drive-master-osx/nuxeo-drive-client/nxdrive/engine/queue_manager.py", line 413, in get_processors_on res.append(self._local_file_thread.worker) AttributeError: 'NoneType' object has no attribute 'worker' ''' self._thread_inspection = Lock() # ERROR HANDLING self._error_lock = Lock() self._on_error_queue = BlacklistQueue(delay=DEFAULT_DELAY) self._error_timer = QTimer() # TODO newErrorGiveUp signal is not connected self._error_timer.timeout.connect(self._on_error_timer) self.newError.connect(self._on_new_error) self.queueProcessing.connect(self.launch_processors) # LAST ACTION self._dao.register_queue_manager(self) def init_processors(self): log.trace("Init processors") self.newItem.connect(self.launch_processors) self.queueProcessing.emit() def shutdown_processors(self): log.trace("Shutdown processors") try: self.newItem.disconnect(self.launch_processors) except TypeError: # TypeError: disconnect() failed between 'newItem' and 'launch_processors' pass def init_queue(self, queue): # Dont need to change modify as State is compatible with QueueItem for item in queue: self.push(item) def _copy_queue(self, queue): result = deepcopy(queue.queue) result.reverse() return result def set_max_processors(self, max_file_processors): max_local_processors = max_file_processors[0] max_remote_processors = max_file_processors[1] max_generic_processors = max_file_processors[2] if max_local_processors < 1: max_local_processors = 1 if max_remote_processors < 1: max_remote_processors = 1 if max_generic_processors < 2: max_generic_processors = 2 self._max_local_processors = max_local_processors - 1 self._max_remote_processors = max_remote_processors - 1 self._max_generic_processors = max_generic_processors - 2 log.trace('number of additional processors: %d local, %d remote, %d generic', self._max_local_processors, self._max_remote_processors, self._max_generic_processors) def resume(self): log.debug("Resuming queue") self.enable_local_file_queue(True, False) self.enable_local_folder_queue(True, False) self.enable_remote_file_queue(True, False) self.enable_remote_folder_queue(True, False) self.queueProcessing.emit() def is_paused(self): return (not self._local_file_enable or not self._local_folder_enable or not self._remote_file_enable or not self._remote_folder_enable) def suspend(self): log.debug("Suspending queue") self.enable_local_file_queue(False) self.enable_local_folder_queue(False) self.enable_remote_file_queue(False) self.enable_remote_folder_queue(False) def restart(self, num_processors=None, wait=False): if num_processors is not None: assert sum(num_processors) <= MAX_NUMBER_PROCESSORS, \ 'total number of additional processors must be %d or less' % MAX_NUMBER_PROCESSORS # stop new items from being processed self.shutdown_processors() # attempt to stop current processors if self._local_file_thread is not None: self._local_file_thread.worker.stop() if self._local_folder_thread is not None: self._local_folder_thread.worker.stop() if self._remote_file_thread is not None: self._remote_file_thread.worker.stop() if self._remote_folder_thread is not None: self._remote_folder_thread.worker.stop() for p in self._processors_pool: p.worker.stop() if wait: while self.is_active(): QCoreApplication.processEvents() sleep(0.1) self.set_max_processors(num_processors) # re-enable new items to trigger processing self.init_processors() # launch processors for new items in the queue, if any self.launch_processors() def enable_local_file_queue(self, value=True, emit=True): self._local_file_enable = value if self._local_file_thread is not None and not value: self._local_file_thread.quit() if value and emit: self.queueProcessing.emit() def enable_local_folder_queue(self, value=True, emit=True): self._local_folder_enable = value if self._local_folder_thread is not None and not value: self._local_folder_thread.quit() if value and emit: self.queueProcessing.emit() def enable_remote_file_queue(self, value=True, emit=True): self._remote_file_enable = value if self._remote_file_thread is not None and not value: self._remote_file_thread.quit() if value and emit: self.queueProcessing.emit() def enable_remote_folder_queue(self, value=True, emit=True): self._remote_folder_enable = value if self._remote_folder_thread is not None and not value: self._remote_folder_thread.quit() if value and emit: self.queueProcessing.emit() def get_local_file_queue(self): return self._copy_queue(self._local_file_queue) def get_remote_file_queue(self): return self._copy_queue(self._remote_file_queue) def get_local_folder_queue(self): return self._copy_queue(self._local_folder_queue) def get_remote_folder_queue(self): return self._copy_queue(self._remote_folder_queue) def push_ref(self, row_id, folderish, pair_state): self.push(QueueItem(row_id, folderish, pair_state)) def push(self, state): if state.pair_state is None: log.trace("Don't push an empty pair_state: %r", state) return log.trace("Pushing %r", state) row_id = state.id if state.pair_state.startswith('locally'): if state.folderish: self._local_folder_queue.put(state) log.trace('Pushed to _local_folder_queue, now of size: %d', self._local_folder_queue.qsize()) else: if "deleted" in state.pair_state: self._engine.cancel_action_on(state.id) self._local_file_queue.put(state) log.trace('Pushed to _local_file_queue, now of size: %d', self._local_file_queue.qsize()) self.newItem.emit(row_id) elif state.pair_state.startswith('remotely'): if state.folderish: self._remote_folder_queue.put(state) log.trace('Pushed to _remote_folder_queue, now of size: %d', self._remote_folder_queue.qsize()) else: if "deleted" in state.pair_state: self._engine.cancel_action_on(state.id) self._remote_file_queue.put(state) log.trace('Pushed to _remote_file_queue, now of size: %d', self._remote_file_queue.qsize()) self.newItem.emit(row_id) else: # deleted and conflicted log.debug("Not processable state: %r", state) @pyqtSlot() def _on_error_timer(self): for item in self._on_error_queue.process_items(): doc_pair = item.get() queueItem = QueueItem(doc_pair.id, doc_pair.folderish, doc_pair.pair_state) log.debug('Retrying blacklisted doc_pair: %r', doc_pair) self.push(queueItem) if self._on_error_queue.is_empty(): self._error_timer.stop() log.debug('blacklist queue timer stopped') def _is_on_error(self, row_id): return self._on_error_queue.exists(row_id) @pyqtSlot() def _on_new_error(self): self._error_timer.start(1000) log.debug('blacklist queue timer started') def get_errors_count(self): return self._on_error_queue.size() def get_error_threshold(self): return self._error_threshold def push_error(self, doc_pair, exception=None): error_count = doc_pair.error_count if (exception is not None and type(exception) == WindowsError and hasattr(exception, 'winerror') and exception.winerror == WINERROR_CODE_PROCESS_CANNOT_ACCESS_FILE): log.debug("Detected WindowsError with code %d: '%s', won't increase next try interval", WINERROR_CODE_PROCESS_CANNOT_ACCESS_FILE, exception.strerror if hasattr(exception, 'strerror') else '') error_count = 1 if error_count > self._error_threshold: self._on_error_queue.remove(doc_pair.id) # TODO this signal is not connected self.newErrorGiveUp.emit(doc_pair.id) log.debug("Giving up on pair : %r", doc_pair) return interval = self._on_error_queue.push(doc_pair.id, doc_pair, count=doc_pair.error_count) log.debug("Blacklisting pair for %ds (error count=%d): %r", interval, doc_pair.error_count, doc_pair) if not self._error_timer.isActive(): self.newError.emit(doc_pair.id) def requeue_errors(self): for doc_pair in self._on_error_queue.items(): doc_pair.error_next_try = 0 def _get_local_folder(self): if self._local_folder_queue.empty(): return None try: state = self._local_folder_queue.get(True, 3) except Empty: return None if state is not None and self._is_on_error(state.id): return self._get_local_folder() return state def _get_local_file(self): if self._local_file_queue.empty(): return None try: state = self._local_file_queue.get(True, 3) except Empty: return None if state is not None and self._is_on_error(state.id): return self._get_local_file() return state def _get_remote_folder(self): if self._remote_folder_queue.empty(): return None try: state = self._remote_folder_queue.get(True, 3) except Empty: return None if state is not None and self._is_on_error(state.id): return self._get_remote_folder() return state def _get_remote_file(self): if self._remote_file_queue.empty(): return None try: state = self._remote_file_queue.get(True, 3) except Empty: return None if state is not None and self._is_on_error(state.id): return self._get_remote_file() return state def _get_file(self): self._get_file_lock.acquire() if self._remote_file_queue.empty() and self._local_file_queue.empty(): self._get_file_lock.release() return None state = None if (self._remote_file_queue.qsize() > self._local_file_queue.qsize()): state = self._get_remote_file() else: state = self._get_local_file() self._get_file_lock.release() if state is not None and self._is_on_error(state.id): return self._get_file() return state @pyqtSlot() def _thread_finished(self): self._thread_inspection.acquire() try: for thread in self._processors_pool: if thread.isFinished(): self._processors_pool.remove(thread) QueueManager.clear_client_transfer_stats(thread.worker.get_thread_id()) if (self._local_folder_thread is not None and self._local_folder_thread.isFinished()): self._local_folder_thread = None if (self._local_file_thread is not None and self._local_file_thread.isFinished()): self._local_file_thread = None if (self._remote_folder_thread is not None and self._remote_folder_thread.isFinished()): self._remote_folder_thread = None if (self._remote_file_thread is not None and self._remote_file_thread.isFinished()): self._remote_file_thread = None if not self._engine.is_paused() and not self._engine.is_stopped(): self.newItem.emit(None) finally: self._thread_inspection.release() def active(self): # Recheck threads self._thread_finished() return self.is_active() def is_active(self): return (self._local_folder_thread is not None or self._local_file_thread is not None or self._remote_file_thread is not None or self._remote_folder_thread is not None or len(self._processors_pool) > 0) def _create_thread(self, item_getter, name=None): processor = self._engine.create_processor(item_getter, name=name) thread = self._engine.create_thread(worker=processor) thread.finished.connect(self._thread_finished) thread.terminated.connect(self._thread_finished) thread.start() return thread def get_metrics(self): metrics = dict() metrics["local_folder_queue"] = self._local_folder_queue.qsize() metrics["local_file_queue"] = self._local_file_queue.qsize() metrics["remote_folder_queue"] = self._remote_folder_queue.qsize() metrics["remote_file_queue"] = self._remote_file_queue.qsize() metrics["remote_file_thread"] = self._remote_file_thread is not None metrics["remote_folder_thread"] = self._remote_folder_thread is not None metrics["local_file_thread"] = self._local_file_thread is not None metrics["local_folder_thread"] = self._local_folder_thread is not None metrics["error_queue"] = self.get_errors_count() metrics["total_queue"] = (metrics["local_folder_queue"] + metrics["local_file_queue"] + metrics["remote_folder_queue"] + metrics["remote_file_queue"]) metrics["additional_processors"] = len(self._processors_pool) return metrics def get_overall_size(self): return (self._local_folder_queue.qsize() + self._local_file_queue.qsize() + self._remote_folder_queue.qsize() + self._remote_file_queue.qsize()) def is_processing_file(self, worker, path, exact_match=False): if not hasattr(worker, "_current_doc_pair"): return False doc_pair = worker._current_doc_pair if (doc_pair is None or doc_pair.local_path is None): return False if exact_match: result = doc_pair.local_path == path else: result = doc_pair.local_path.startswith(path) if result: log.trace("Worker(%r) is processing: %r", worker.get_metrics(), path) return result def interrupt_processors_on(self, path, exact_match=True): for proc in self.get_processors_on(path, exact_match): proc.stop() def get_processors_on(self, path, exact_match=True): self._thread_inspection.acquire() try: res = [] if self._local_folder_thread is not None: if self.is_processing_file(self._local_folder_thread.worker, path, exact_match): res.append(self._local_folder_thread.worker) if self._remote_folder_thread is not None: if self.is_processing_file(self._remote_folder_thread.worker, path, exact_match): res.append(self._remote_folder_thread.worker) if self._local_file_thread is not None: if self.is_processing_file(self._local_file_thread.worker, path, exact_match): res.append(self._local_file_thread.worker) if self._remote_file_thread is not None: if self.is_processing_file(self._remote_file_thread.worker, path, exact_match): res.append(self._remote_file_thread.worker) for thread in self._processors_pool: if self.is_processing_file(thread.worker, path, exact_match): res.append(thread.worker) return res finally: self._thread_inspection.release() def has_file_processors_on(self, path): self._thread_inspection.acquire() try: # First check local and remote file if self._local_file_thread is not None: if self.is_processing_file(self._local_file_thread.worker, path): return True if self._remote_file_thread is not None: if self.is_processing_file(self._remote_file_thread.worker, path): return True for thread in self._processors_pool: if self.is_processing_file(thread.worker, path): return True return False finally: self._thread_inspection.release() @pyqtSlot() def launch_processors(self): if (self._disable or self.is_paused() or (self._local_folder_queue.empty() and self._local_file_queue.empty() and self._remote_folder_queue.empty() and self._remote_file_queue.empty())): self.queueEmpty.emit() if not self.is_active(): self.queueFinishedProcessing.emit() return log.trace("Launching processors") if not (self._local_folder_queue.empty() and self._local_file_queue.empty()): if self._local_folder_thread is None and not self._local_folder_queue.empty() and self._local_folder_enable: log.debug("creating local folder processor") self._local_folder_thread = self._create_thread(self._get_local_folder, name="LocalFolderProcessor") if self._local_file_thread is None and not self._local_file_queue.empty() and self._local_file_enable: log.debug("creating local file processor") self._local_file_thread = self._create_thread(self._get_local_file, name="LocalFileProcessor") if not (self._remote_folder_queue.empty() and self._remote_file_queue.empty()): if self._remote_folder_thread is None and not self._remote_folder_queue.empty() and self._remote_folder_enable: log.debug("creating remote folder processor") self._remote_folder_thread = self._create_thread(self._get_remote_folder, name="RemoteFolderProcessor") if self._remote_file_thread is None and not self._remote_file_queue.empty() and self._remote_file_enable: log.debug("creating remote file processor") self._remote_file_thread = self._create_thread(self._get_remote_file, name="RemoteFileProcessor") if self._remote_file_queue.qsize() + self._local_file_queue.qsize() == 0: return count = 0 # log.trace('processor pool: %s', ','.join([t.worker.get_name() for t in self._processors_pool])) log.trace('max generic processors: %d', self._max_generic_processors) log.trace('max remote processors: %d', self._max_remote_processors) log.trace('max local processors: %d', self._max_local_processors) if not (self._local_file_queue.empty() and self._remote_file_queue.empty()): while len([t for t in self._processors_pool if t.worker.get_name() == "GenericProcessor"]) < self._max_generic_processors: self._processors_pool.append(self._create_thread(self._get_file, name="GenericProcessor")) count += 1 if count > 0: log.trace("created %d additional file processor%s", count, 's' if count > 1 else '') count = 0 if not self._remote_file_queue.empty(): while len([t for t in self._processors_pool if t.worker.get_name() == "RemoteFileProcessor"]) < self._max_remote_processors: self._processors_pool.append(self._create_thread(self._get_remote_file, name="RemoteFileProcessor")) count += 1 if count > 0: log.trace("created %d additional remote file processor%s", count, 's' if count > 1 else '') count = 0 if not self._local_file_queue.empty(): while len([t for t in self._processors_pool if t.worker.get_name() == "LocalFileProcessor"]) < self._max_local_processors: self._processors_pool.append(self._create_thread(self._get_local_file, name="LocalFileProcessor")) count += 1 if count > 0: log.trace("created %d additional local file processor%s", count, 's' if count > 1 else '') @staticmethod def clear_client_transfer_stats(thread_id): if hasattr(BaseAutomationClient, 'download_stats') and BaseAutomationClient.download_stats is not None: BaseAutomationClient.download_stats.clear(thread_id) if hasattr(BaseAutomationClient, 'upload_stats') and BaseAutomationClient.upload_stats is not None: BaseAutomationClient.upload_stats.clear(thread_id) if hasattr(BaseAutomationClient, 'download_token_bucket') and \ BaseAutomationClient.download_token_bucket is not None: BaseAutomationClient.download_token_bucket.clear(thread_id) if hasattr(BaseAutomationClient, 'upload_token_bucket') and \ BaseAutomationClient.upload_token_bucket is not None: BaseAutomationClient.upload_token_bucket.clear(thread_id)
class QueueManager(QObject): # Always create thread from the main thread newItem = pyqtSignal(object) newError = pyqtSignal(object) newErrorGiveUp = pyqtSignal(object) queueEmpty = pyqtSignal() queueProcessing = pyqtSignal() queueFinishedProcessing = pyqtSignal() # Only used by Unit Test _disable = False ''' classdocs ''' def __init__(self, engine, dao, max_file_processors=5): ''' Constructor ''' super(QueueManager, self).__init__() self._dao = dao self._engine = engine self._local_folder_queue = Queue() self._local_file_queue = Queue() self._remote_file_queue = Queue() self._remote_folder_queue = Queue() self._connected = local() self._local_folder_enable = True self._local_file_enable = True self._remote_folder_enable = True self._remote_file_enable = True self._local_folder_thread = None self._local_file_thread = None self._remote_folder_thread = None self._remote_file_thread = None self._error_threshold = ERROR_THRESHOLD self._error_interval = 60 self.set_max_processors(max_file_processors) self._threads_pool = list() self._processors_pool = list() self._get_file_lock = Lock() # Should not operate on thread while we are inspecting them ''' This error required to add a lock for inspecting threads, as the below Traceback shows the processor thread was ended while the method was running Traceback (most recent call last): File "/Users/hudson/tmp/workspace/FT-nuxeo-drive-master-osx/nuxeo-drive-client/nxdrive/engine/watcher/local_watcher.py", line 845, in handle_watchdog_event self.scan_pair(rel_path) File "/Users/hudson/tmp/workspace/FT-nuxeo-drive-master-osx/nuxeo-drive-client/nxdrive/engine/watcher/local_watcher.py", line 271, in scan_pair self._suspend_queue() File "/Users/hudson/tmp/workspace/FT-nuxeo-drive-master-osx/nuxeo-drive-client/nxdrive/engine/watcher/local_watcher.py", line 265, in _suspend_queue for processor in self._engine.get_queue_manager().get_processors_on('/', exact_match=False): File "/Users/hudson/tmp/workspace/FT-nuxeo-drive-master-osx/nuxeo-drive-client/nxdrive/engine/queue_manager.py", line 413, in get_processors_on res.append(self._local_file_thread.worker) AttributeError: 'NoneType' object has no attribute 'worker' ''' self._thread_inspection = Lock() # ERROR HANDLING self._error_lock = Lock() self._on_error_queue = BlacklistQueue(delay=DEFAULT_DELAY) self._error_timer = QTimer() # TODO newErrorGiveUp signal is not connected self._error_timer.timeout.connect(self._on_error_timer) self.newError.connect(self._on_new_error) self.queueProcessing.connect(self.launch_processors) # LAST ACTION self._dao.register_queue_manager(self) def init_processors(self): log.trace("Init processors") self.newItem.connect(self.launch_processors) self.queueProcessing.emit() def shutdown_processors(self): log.trace("Shutdown processors") try: self.newItem.disconnect(self.launch_processors) except TypeError: # TypeError: disconnect() failed between 'newItem' and 'launch_processors' pass def init_queue(self, queue): # Dont need to change modify as State is compatible with QueueItem for item in queue: self.push(item) def _copy_queue(self, queue): result = deepcopy(queue.queue) result.reverse() return result def set_max_processors(self, max_file_processors): if max_file_processors < 2: max_file_processors = 2 self._max_processors = max_file_processors - 2 def resume(self): log.debug("Resuming queue") self.enable_local_file_queue(True, False) self.enable_local_folder_queue(True, False) self.enable_remote_file_queue(True, False) self.enable_remote_folder_queue(True, False) self.queueProcessing.emit() def is_paused(self): return (not self._local_file_enable or not self._local_folder_enable or not self._remote_file_enable or not self._remote_folder_enable) def suspend(self): log.debug("Suspending queue") self.enable_local_file_queue(False) self.enable_local_folder_queue(False) self.enable_remote_file_queue(False) self.enable_remote_folder_queue(False) def enable_local_file_queue(self, value=True, emit=True): self._local_file_enable = value if self._local_file_thread is not None and not value: self._local_file_thread.quit() if value and emit: self.queueProcessing.emit() def enable_local_folder_queue(self, value=True, emit=True): self._local_folder_enable = value if self._local_folder_thread is not None and not value: self._local_folder_thread.quit() if value and emit: self.queueProcessing.emit() def enable_remote_file_queue(self, value=True, emit=True): self._remote_file_enable = value if self._remote_file_thread is not None and not value: self._remote_file_thread.quit() if value and emit: self.queueProcessing.emit() def enable_remote_folder_queue(self, value=True, emit=True): self._remote_folder_enable = value if self._remote_folder_thread is not None and not value: self._remote_folder_thread.quit() if value and emit: self.queueProcessing.emit() def get_local_file_queue(self): return self._copy_queue(self._local_file_queue) def get_remote_file_queue(self): return self._copy_queue(self._remote_file_queue) def get_local_folder_queue(self): return self._copy_queue(self._local_folder_queue) def get_remote_folder_queue(self): return self._copy_queue(self._remote_folder_queue) def push_ref(self, row_id, folderish, pair_state): self.push(QueueItem(row_id, folderish, pair_state)) def push(self, state): if state.pair_state is None: log.trace("Don't push an empty pair_state: %r", state) return log.trace("Pushing %r", state) row_id = state.id if state.pair_state.startswith('locally'): if state.folderish: self._local_folder_queue.put(state) log.trace('Pushed to _local_folder_queue, now of size: %d', self._local_folder_queue.qsize()) else: if "deleted" in state.pair_state: self._engine.cancel_action_on(state.id) self._local_file_queue.put(state) log.trace('Pushed to _local_file_queue, now of size: %d', self._local_file_queue.qsize()) self.newItem.emit(row_id) elif state.pair_state.startswith('remotely'): if state.folderish: self._remote_folder_queue.put(state) log.trace('Pushed to _remote_folder_queue, now of size: %d', self._remote_folder_queue.qsize()) else: if "deleted" in state.pair_state: self._engine.cancel_action_on(state.id) self._remote_file_queue.put(state) log.trace('Pushed to _remote_file_queue, now of size: %d', self._remote_file_queue.qsize()) self.newItem.emit(row_id) else: # deleted and conflicted log.debug("Not processable state: %r", state) @pyqtSlot() def _on_error_timer(self): for item in self._on_error_queue.process_items(): doc_pair = item.get() queueItem = QueueItem(doc_pair.id, doc_pair.folderish, doc_pair.pair_state) log.debug('Retrying blacklisted doc_pair: %r', doc_pair) self.push(queueItem) if self._on_error_queue.is_empty(): self._error_timer.stop() log.debug('blacklist queue timer stopped') def _is_on_error(self, row_id): return self._on_error_queue.exists(row_id) @pyqtSlot() def _on_new_error(self): self._error_timer.start(1000) log.debug('blacklist queue timer started') def get_errors_count(self): return self._on_error_queue.size() def get_error_threshold(self): return self._error_threshold def push_error(self, doc_pair, exception=None): error_count = doc_pair.error_count if (exception is not None and type(exception) == WindowsError and hasattr(exception, 'winerror') and exception.winerror == WINERROR_CODE_PROCESS_CANNOT_ACCESS_FILE): log.debug("Detected WindowsError with code %d: '%s', won't increase next try interval", WINERROR_CODE_PROCESS_CANNOT_ACCESS_FILE, exception.strerror if hasattr(exception, 'strerror') else '') error_count = 1 if error_count > self._error_threshold: self._on_error_queue.remove(doc_pair.id) # TODO this signal is not connected self.newErrorGiveUp.emit(doc_pair.id) log.debug("Giving up on pair : %r", doc_pair) return interval = self._on_error_queue.push(doc_pair.id, doc_pair, count=doc_pair.error_count) log.debug("Blacklisting pair for %ds (error count=%d): %r", interval, doc_pair.error_count, doc_pair) if not self._error_timer.isActive(): self.newError.emit(doc_pair.id) def requeue_errors(self): for doc_pair in self._on_error_queue.items(): doc_pair.error_next_try = 0 def _get_local_folder(self): if self._local_folder_queue.empty(): return None try: state = self._local_folder_queue.get(True, 3) except Empty: return None if state is not None and self._is_on_error(state.id): return self._get_local_folder() return state def _get_local_file(self): if self._local_file_queue.empty(): return None try: state = self._local_file_queue.get(True, 3) except Empty: return None if state is not None and self._is_on_error(state.id): return self._get_local_file() return state def _get_remote_folder(self): if self._remote_folder_queue.empty(): return None try: state = self._remote_folder_queue.get(True, 3) except Empty: return None if state is not None and self._is_on_error(state.id): return self._get_remote_folder() return state def _get_remote_file(self): if self._remote_file_queue.empty(): return None try: state = self._remote_file_queue.get(True, 3) except Empty: return None if state is not None and self._is_on_error(state.id): return self._get_remote_file() return state def _get_file(self): self._get_file_lock.acquire() if self._remote_file_queue.empty() and self._local_file_queue.empty(): self._get_file_lock.release() return None state = None if (self._remote_file_queue.qsize() > self._local_file_queue.qsize()): state = self._get_remote_file() else: state = self._get_local_file() self._get_file_lock.release() if state is not None and self._is_on_error(state.id): return self._get_file() return state @pyqtSlot() def _thread_finished(self): self._thread_inspection.acquire() try: for thread in self._processors_pool: if thread.isFinished(): self._processors_pool.remove(thread) if (self._local_folder_thread is not None and self._local_folder_thread.isFinished()): self._local_folder_thread = None if (self._local_file_thread is not None and self._local_file_thread.isFinished()): self._local_file_thread = None if (self._remote_folder_thread is not None and self._remote_folder_thread.isFinished()): self._remote_folder_thread = None if (self._remote_file_thread is not None and self._remote_file_thread.isFinished()): self._remote_file_thread = None if not self._engine.is_paused() and not self._engine.is_stopped(): self.newItem.emit(None) finally: self._thread_inspection.release() def active(self): # Recheck threads self._thread_finished() return self.is_active() def is_active(self): return (self._local_folder_thread is not None or self._local_file_thread is not None or self._remote_file_thread is not None or self._remote_folder_thread is not None or len(self._processors_pool) > 0) def _create_thread(self, item_getter, name=None): processor = self._engine.create_processor(item_getter, name=name) thread = self._engine.create_thread(worker=processor) thread.finished.connect(self._thread_finished) thread.terminated.connect(self._thread_finished) thread.start() return thread def get_metrics(self): metrics = dict() metrics["local_folder_queue"] = self._local_folder_queue.qsize() metrics["local_file_queue"] = self._local_file_queue.qsize() metrics["remote_folder_queue"] = self._remote_folder_queue.qsize() metrics["remote_file_queue"] = self._remote_file_queue.qsize() metrics["remote_file_thread"] = self._remote_file_thread is not None metrics["remote_folder_thread"] = self._remote_folder_thread is not None metrics["local_file_thread"] = self._local_file_thread is not None metrics["local_folder_thread"] = self._local_folder_thread is not None metrics["error_queue"] = self.get_errors_count() metrics["total_queue"] = (metrics["local_folder_queue"] + metrics["local_file_queue"] + metrics["remote_folder_queue"] + metrics["remote_file_queue"]) metrics["additional_processors"] = len(self._processors_pool) return metrics def get_overall_size(self): return (self._local_folder_queue.qsize() + self._local_file_queue.qsize() + self._remote_folder_queue.qsize() + self._remote_file_queue.qsize()) def is_processing_file(self, worker, path, exact_match=False): if not hasattr(worker, "_current_doc_pair"): return False doc_pair = worker._current_doc_pair if (doc_pair is None or doc_pair.local_path is None): return False if exact_match: result = doc_pair.local_path == path else: result = doc_pair.local_path.startswith(path) if result: log.trace("Worker(%r) is processing: %r", worker.get_metrics(), path) return result def interrupt_processors_on(self, path, exact_match=True): for proc in self.get_processors_on(path, exact_match): proc.stop() def get_processors_on(self, path, exact_match=True): self._thread_inspection.acquire() try: res = [] if self._local_folder_thread is not None: if self.is_processing_file(self._local_folder_thread.worker, path, exact_match): res.append(self._local_folder_thread.worker) if self._remote_folder_thread is not None: if self.is_processing_file(self._remote_folder_thread.worker, path, exact_match): res.append(self._remote_folder_thread.worker) if self._local_file_thread is not None: if self.is_processing_file(self._local_file_thread.worker, path, exact_match): res.append(self._local_file_thread.worker) if self._remote_file_thread is not None: if self.is_processing_file(self._remote_file_thread.worker, path, exact_match): res.append(self._remote_file_thread.worker) for thread in self._processors_pool: if self.is_processing_file(thread.worker, path, exact_match): res.append(thread.worker) return res finally: self._thread_inspection.release() def has_file_processors_on(self, path): self._thread_inspection.acquire() try: # First check local and remote file if self._local_file_thread is not None: if self.is_processing_file(self._local_file_thread.worker, path): return True if self._remote_file_thread is not None: if self.is_processing_file(self._remote_file_thread.worker, path): return True for thread in self._processors_pool: if self.is_processing_file(thread.worker, path): return True return False finally: self._thread_inspection.release() @pyqtSlot() def launch_processors(self): if (self._disable or self.is_paused() or (self._local_folder_queue.empty() and self._local_file_queue.empty() and self._remote_folder_queue.empty() and self._remote_file_queue.empty())): self.queueEmpty.emit() if not self.is_active(): self.queueFinishedProcessing.emit() return log.trace("Launching processors") if self._local_folder_thread is None and not self._local_folder_queue.empty() and self._local_folder_enable: log.debug("creating local folder processor") self._local_folder_thread = self._create_thread(self._get_local_folder, name="LocalFolderProcessor") if self._local_file_thread is None and not self._local_file_queue.empty() and self._local_file_enable: log.debug("creating local file processor") self._local_file_thread = self._create_thread(self._get_local_file, name="LocalFileProcessor") if self._remote_folder_thread is None and not self._remote_folder_queue.empty() and self._remote_folder_enable: log.debug("creating remote folder processor") self._remote_folder_thread = self._create_thread(self._get_remote_folder, name="RemoteFolderProcessor") if self._remote_file_thread is None and not self._remote_file_queue.empty() and self._remote_file_enable: log.debug("creating remote file processor") self._remote_file_thread = self._create_thread(self._get_remote_file, name="RemoteFileProcessor") if self._remote_file_queue.qsize() + self._local_file_queue.qsize() == 0: return while len(self._processors_pool) < self._max_processors: log.debug("creating additional file processor") self._processors_pool.append(self._create_thread(self._get_file, name="GenericProcessor"))