def __init__(self, num_workers, worker_name): threading.Thread.__init__(self, name=worker_name) self.pool = Pool(num_workers=num_workers, name=worker_name) self.in_progress_jobs = Manager().list() self.lock = Manager().Lock()
def __init__(self, parent=None): super(ICEExporter, self).__init__(parent) self.files = [] self.state = self.STOP self.destination_folder = '.' self.t1 = 0 self.t2 = 0 self.fmt = CONSTS.TEXT_FMT self.pool = Pool(self) # dialogs self._folder_dialog = ICEExportFolderDialog(self, parent) self._file_dialog = ICEExportFileDialog(self, parent)
def __init__(self, parent = None): super(ICEExporter,self).__init__(parent) self.files = [] self.state = self.STOP self.destination_folder = '.' self.t1 = 0 self.t2 = 0 self.fmt = CONSTS.TEXT_FMT self.pool = Pool(self) # dialogs self._folder_dialog = ICEExportFolderDialog(self, parent) self._file_dialog = ICEExportFileDialog(self, parent)
class JobManager(threading.Thread): def __init__(self, num_workers, worker_name): threading.Thread.__init__(self, name=worker_name) self.pool = Pool(num_workers=num_workers, name=worker_name) self.in_progress_jobs = Manager().list() self.lock = Manager().Lock() def run(self): while 1: try: time.sleep(10) self.dispatch() except Exception: # Print to debug console instead of to DB. import traceback print(traceback.format_exc()) def dispatch(self): raise NotImplementedError("Children must override dispatch()") def submit_job(self, work_unit): with self.lock: if work_unit.get_unique_key() in self.in_progress_jobs: return False self.in_progress_jobs.append(work_unit.get_unique_key()) # Remember these shared memory references work_unit.in_progress_jobs = self.in_progress_jobs work_unit.lock = self.lock self.pool.submit(work_unit) return True
class ICEExporter(QtCore.QObject): """ Class to manage processes for exporting ICE cache data to ascii. """ # string: export file name cacheExporting = QtCore.pyqtSignal(str) # int: number of files to export beginCacheExporting = QtCore.pyqtSignal(int) endCacheExporting = QtCore.pyqtSignal() # states STOP = 0 RUN = 1 ERROR = 2 def __init__(self, parent=None): super(ICEExporter, self).__init__(parent) self.files = [] self.state = self.STOP self.destination_folder = '.' self.t1 = 0 self.t2 = 0 self.fmt = CONSTS.TEXT_FMT self.pool = Pool(self) # dialogs self._folder_dialog = ICEExportFolderDialog(self, parent) self._file_dialog = ICEExportFileDialog(self, parent) @property def folder_dialog(self): return self._folder_dialog @property def file_dialog(self): return self._file_dialog def cancel(self): self.pool.cancel() def export_folder(self, folder, destination, fmt=CONSTS.TEXT_FMT): """ Export the cache files contained in a folder """ (self.files, startindex, end) = get_files_from_cache_folder(folder) self.destination_folder = destination self.state = self.STOP self.files_processed = 0 self.t1 = 0 self.t2 = 0 self.fmt = fmt self._start() def export_files(self, files, destination, fmt=CONSTS.TEXT_FMT): """ Export a list of cache files """ (self.files, startindex, end) = get_files(files) self.destination_folder = destination self.state = self.STOP self.t1 = 0 self.t2 = 0 self.fmt = fmt self._start() def _start(self): """ Start file export process. """ self.file_block = 1 self.indexset = range(0, len(self.files), self.file_block) # Submit export tasks to process pool cpu_count = 1 if self.parent(): cpu_count = self.parent().prefs.process_count else: import multiprocessing as mp cpu_count = mp.cpu_count() self.pool.init(cpu_count, self._on_process_callback) self.state = self.STOP file_count = len(self.files) self.files_processed = 0 for i in self.indexset: file_list = [] for j in range(self.file_block): if i + j < file_count: file_list.append(self.files[i + j]) self.pool.submit( ExportTask([file_list, self.destination_folder, self.fmt])) def _on_process_callback(self, sender, notif, arg): """ Called when an event occurs from a process """ if notif == Pool.STARTED: #print 'Pool.STARTED' if self.state == self.STOP: self.t1 = time.time() self.beginCacheExporting.emit(len(self.files)) self.state = self.RUN elif notif == Pool.ERROR: print 'process error: %d - %s' % (sender.pid(), arg) self.endCacheExporting.emit() self.state = self.ERROR elif notif == Pool.STATE_CHANGE: pass elif notif == Pool.OUTPUT_MSG: if self.state == self.ERROR: return s_out = bytes.decode(bytes(sender.readAllStandardOutput())) #print 'Pool.OUTPUT_MSG: %s' % s_out self.cacheExporting.emit(s_out) elif notif == Pool.OUTPUT_ERROR_MSG: s_out = bytes.decode(bytes(sender.readAllStandardError())) print 'process error output: %s - %s\nRetry your operation.' % ( (repr(sender)), s_out) elif notif == Pool.FINISHED: #print 'Pool.FINISHED' if self.state == self.ERROR: return self.files_processed += self.file_block if self.files_processed >= len(self.files): self.t2 = time.time() self.endCacheExporting.emit() self.state = self.STOP print 'Export time %0.3f s' % (self.t2 - self.t1)
def init_process_server( self ): self._state = self.STOP self._files = [] self._cache = {} self._pool = Pool(self) self._pool.init(self.parent().prefs.process_count, self._on_process_callback)
class ICECacheLoader(QtCore.QObject): """ Class for loading cache files. Files are loaded through processes managed by the Pool class. The data loaded by processes is sent to ICECacheLoader via PyQt's QLocalSocket and QLocalServer which are basically a named pipe. """ # signal sent for each new loaded cache # int: cache index # tuple: (reader, points, colors, sizes) #cacheLoaded = QtCore.pyqtSignal( int, tuple ) cacheLoaded = QtCore.pyqtSignal( int, str ) beginCacheLoading = QtCore.pyqtSignal() endCacheLoading = QtCore.pyqtSignal() # Loading state STOP = 0 RUN = 1 ERROR = 2 def __init__(self, parent = None): super(ICECacheLoader,self).__init__(parent) self._pool = None self._state = self.STOP self._files = [] self._cache = {} def init_process_server( self ): self._state = self.STOP self._files = [] self._cache = {} self._pool = Pool(self) self._pool.init(self.parent().prefs.process_count, self._on_process_callback) def cancel(self): self._state = self.STOP self.endCacheLoading.emit() if self._pool: self._pool.cancel() print "Operation was cancelled." def points( self, cache_index ): if cache_index in self._cache and self._cache[ cache_index ] != None and POINT_DATA in self._cache[ cache_index ]: return self._cache[ cache_index ][ POINT_DATA ][:] return [] def colors( self, cache_index ): if cache_index in self._cache and self._cache[ cache_index ] != None and COLOR_DATA in self._cache[ cache_index ]: return self._cache[ cache_index ][ COLOR_DATA ][:] return [] def sizes( self, cache_index ): if cache_index in self._cache and self._cache[ cache_index ] != None and SIZE_DATA in self._cache[ cache_index ]: return self._cache[ cache_index ][ SIZE_DATA ][:] return [] def __getitem__( self, arg ): """ return item cache by index """ return self._cache[ arg ] def load_cache_files( self, files, start, end ): """ Start the loading process. """ # initialize the process server first self.init_process_server() self._files = files self.startindex = start self.endindex = end # file_block can be used to assing multiple files per process. # note: Normally this would make poor load-balancing and affect performance load self.file_block = 1 self.indexset = range(0,len(self._files),self.file_block) # Submit all load tasks to the process pool # note: 1 file / task gives very good performance in general. self._state = self.STOP file_count = len(self._files) self._files_processed = 0 index = self.startindex for i in self.indexset: file_list = [] file_index = [] for j in range(self.file_block): if i+j < file_count: file_list.append( self._files[i+j] ) file_index.append( index ) index += 1 self._pool.submit( LoaderTask( [ file_list, file_index ] ) ) def _on_process_callback( self, sender, notif, arg ): """ Called when an event occurs from a process """ if notif == Pool.STARTED: if self._state == self.STOP: self.t1 = time.time() self._state = self.RUN self.beginCacheLoading.emit() return elif notif == Pool.ERROR: self._state = self.ERROR return elif notif == Pool.STATE_CHANGE: return elif notif == Pool.OUTPUT_MSG: if self._state == self.ERROR: return # Process has finished loading the file s_data = bytes.decode( bytes( sender.readAllStandardOutput() ) ) try: data = eval(s_data) # save cache self._cache[ data[0] ] = h5.File( data[1], 'r' ) # notify clients self.cacheLoaded.emit( data[0], data[1] ) except: # problem, just cancel everything self.cancel() raise Pool.Error elif notif == Pool.OUTPUT_ERROR_MSG: self._state = self.ERROR s_out = bytes.decode( bytes( sender.readAllStandardError() ) ) print 'process error output: %s - %s\nRetry your operation.' % ((repr(sender)),s_out) elif notif == Pool.FINISHED: if self._state == self.STOP: return if self._state == self.ERROR: # tell clients the loading process has been cancelled or was terminated prematurely self.cancel() return #print 'process finished: %s\n' % (repr(sender)) self._files_processed += self.file_block if self._files_processed >= len(self._files): self.t2 = time.time() self._state = self.STOP self.endCacheLoading.emit() print 'Processes %d Loading time %0.3f s' % (self._pool.process_count,self.t2-self.t1) return
class ICEExporter(QtCore.QObject): """ Class to manage processes for exporting ICE cache data to ascii. """ # string: export file name cacheExporting = QtCore.pyqtSignal( str ) # int: number of files to export beginCacheExporting = QtCore.pyqtSignal( int ) endCacheExporting = QtCore.pyqtSignal( ) # states STOP = 0 RUN = 1 ERROR = 2 def __init__(self, parent = None): super(ICEExporter,self).__init__(parent) self.files = [] self.state = self.STOP self.destination_folder = '.' self.t1 = 0 self.t2 = 0 self.fmt = CONSTS.TEXT_FMT self.pool = Pool(self) # dialogs self._folder_dialog = ICEExportFolderDialog(self, parent) self._file_dialog = ICEExportFileDialog(self, parent) @property def folder_dialog( self ): return self._folder_dialog @property def file_dialog( self ): return self._file_dialog def cancel(self): self.pool.cancel() def export_folder( self, folder, destination, fmt=CONSTS.TEXT_FMT ): """ Export the cache files contained in a folder """ (self.files,startindex,end) = get_files_from_cache_folder( folder ) self.destination_folder = destination self.state = self.STOP self.files_processed = 0 self.t1 = 0 self.t2 = 0 self.fmt = fmt self._start() def export_files( self, files, destination, fmt=CONSTS.TEXT_FMT): """ Export a list of cache files """ (self.files,startindex,end) = get_files(files) self.destination_folder = destination self.state = self.STOP self.t1 = 0 self.t2 = 0 self.fmt = fmt self._start() def _start( self ): """ Start file export process. """ self.file_block = 1 self.indexset = range( 0, len(self.files), self.file_block ) # Submit export tasks to process pool cpu_count = 1 if self.parent(): cpu_count = self.parent().prefs.process_count else: import multiprocessing as mp cpu_count = mp.cpu_count() self.pool.init( cpu_count, self._on_process_callback ) self.state = self.STOP file_count = len(self.files) self.files_processed = 0 for i in self.indexset: file_list = [] for j in range(self.file_block): if i+j < file_count: file_list.append( self.files[i+j] ) self.pool.submit( ExportTask( [file_list, self.destination_folder, self.fmt] ) ) def _on_process_callback( self, sender, notif, arg ): """ Called when an event occurs from a process """ if notif == Pool.STARTED: #print 'Pool.STARTED' if self.state == self.STOP: self.t1 = time.time() self.beginCacheExporting.emit( len(self.files) ) self.state = self.RUN elif notif == Pool.ERROR: print 'process error: %d - %s' % (sender.pid(), arg) self.endCacheExporting.emit( ) self.state = self.ERROR elif notif == Pool.STATE_CHANGE: pass elif notif == Pool.OUTPUT_MSG: if self.state == self.ERROR: return s_out = bytes.decode( bytes( sender.readAllStandardOutput() ) ) #print 'Pool.OUTPUT_MSG: %s' % s_out self.cacheExporting.emit( s_out ) elif notif == Pool.OUTPUT_ERROR_MSG: s_out = bytes.decode( bytes( sender.readAllStandardError() ) ) print 'process error output: %s - %s\nRetry your operation.' % ((repr(sender)),s_out) elif notif == Pool.FINISHED: #print 'Pool.FINISHED' if self.state == self.ERROR: return self.files_processed += self.file_block if self.files_processed >= len(self.files): self.t2 = time.time() self.endCacheExporting.emit( ) self.state = self.STOP print 'Export time %0.3f s' % (self.t2-self.t1)