def createSubproccesses(self): # Haven't decided yet whether we should force Kwola to always write to disc or spool in memory # using /tmp. The following lines switch between the two approaches # self.batchDirectory = tempfile.mkdtemp(dir=getKwolaUserDataDirectory("batches")) self.batchDirectory = tempfile.mkdtemp() self.subProcessCommandQueues = [] self.subProcessBatchResultQueues = [] self.subProcesses = [] for subprocessIndex in range(self.config['training_batch_prep_subprocesses']): subProcessCommandQueue = multiprocessing.Queue() subProcessBatchResultQueue = multiprocessing.Queue() subProcess = multiprocessing.Process(target=TrainingManager.prepareAndLoadBatchesSubprocess, args=(self.configDir, self.batchDirectory, subProcessCommandQueue, subProcessBatchResultQueue, subprocessIndex, self.applicationId)) subProcess.start() atexit.register(lambda: subProcess.terminate()) self.subProcessCommandQueues.append(subProcessCommandQueue) self.subProcessBatchResultQueues.append(subProcessBatchResultQueue) self.subProcesses.append(subProcess) for queue in self.subProcessBatchResultQueues: readyState = queue.get() if readyState == "error": raise Exception("Error occurred during batch prep sub process initiation.")
def start_queues(self): self.in_queue = billiard.Queue(maxsize=self.config.q_multipler * self.config.n_readers) self.out_queue = billiard.Queue(maxsize=self.config.q_multipler * self.config.n_readers) self.malloc_queue = billiard.Queue(maxsize=self.config.q_multipler * self.config.n_readers) for _ in range(self.config.n_generators): self.multicast_queues.append( billiard.Queue(maxsize=self.config.q_multipler * self.config.n_readers))
def createTestingSubprocesses(self): self.subProcesses = [] for n in range(self.config['testing_subprocess_pool_size']): subProcessCommandQueue = multiprocessing.Queue() subProcessResultQueue = multiprocessing.Queue() preloadTraceFiles = [file for fileList in self.executionSessionTraceLocalPickleFiles for file in fileList] subProcess = multiprocessing.Process(target=TestingStepManager.predictedActionSubProcess, args=(self.configDir, self.shouldBeRandom, subProcessCommandQueue, subProcessResultQueue, preloadTraceFiles)) subProcess.start() atexit.register(lambda: subProcess.terminate()) self.subProcesses.append((subProcessCommandQueue, subProcessResultQueue, subProcess))
def restartOneTestingSubprocess(self): subProcessCommandQueue, subProcessResultQueue, subProcess = self.subProcesses.pop(0) subProcessCommandQueue.put("quit") subProcess.terminate() subProcessCommandQueue = multiprocessing.Queue() subProcessResultQueue = multiprocessing.Queue() preloadTraceFiles = [file for fileList in self.executionSessionTraceLocalPickleFiles for file in fileList] subProcess = multiprocessing.Process(target=TestingStepManager.predictedActionSubProcess, args=(self.configDir, self.shouldBeRandom, subProcessCommandQueue, subProcessResultQueue, preloadTraceFiles)) subProcess.start() atexit.register(lambda: subProcess.terminate()) self.subProcesses.append((subProcessCommandQueue, subProcessResultQueue, subProcess))
def parallel_crawl(product, marketplace): if product.phrases == None: keywords_and_phrases = product.keywords else: keywords_and_phrases = product.keywords + product.phrases # Define an output queue output_queue = mp.Queue() # Setup a list of processes that we want to run processes = [ mp.Process(target=begin_crawl, args=(product, marketplace, keyword, 0, output_queue)) for keyword in keywords_and_phrases ] #intial_time = time() # Run processes for p in processes: p.start() # Exit the completed processes for p in processes: p.join() # Get process results from the output queue results = [output_queue.get() for p in processes] #final_time = time() return dict(pair for d in results for pair in d.items())
def __init__(self, blocking=True, db_path=None, ncpu=1): """ Init function Parameter --------- blocking: bool determines whether join() blocks or not db_path: str the string to a LevelDB for command persistence """ self.__blocking = blocking self.__broker_queue = mp.Queue() self.__job_queue = mp.JoinableQueue() self.__pending_dict = mp.Manager().dict() self.__results_queue = mp.Queue() self.__results_queue_worker = mp.Queue() if db_path is None: tmp_db = NamedTemporaryFile(delete=False, dir=os.getcwd(), suffix=".db") tmp_db.close() self.__is_temp_db = True self.__db_path = tmp_db.name else: self.__is_temp_db = False self.__db_path = db_path self.__broker = _Broker(self.__broker_queue, self.__job_queue, self.__results_queue, self.__results_queue_worker, self.__pending_dict, db_path=self.__db_path) self.__broker.daemon = False self.__broker.start() self.__worker = [] for i in range(ncpu): p = _Worker(self.__broker_queue, self.__job_queue, self.__results_queue_worker) p.daemon = False self.__worker.append(p) p.start()
def __init__(self, config, plugins=None): if plugins is None: self.plugins = [] else: self.plugins = plugins self.plugins = [ JSRewriter(config), HTMLRewriter(config) ] + self.plugins self.config = config self.commandQueue = multiprocessing.Queue() self.resultQueue = multiprocessing.Queue() self.proxyProcess = multiprocessing.Process(target=self.runProxyServerSubprocess, args=(self.config, self.commandQueue, self.resultQueue, pickle.dumps(self.plugins))) self.proxyProcess.start() # Wait for the result indicating that the proxy process is ready self.port = self.resultQueue.get() time.sleep(0.1)
def run(self): tasks = mp.Queue() results = mp.JoinableQueue() interim = [] args = (tasks, results) # n_procs = min(mp.cpu_count(), len(self._videos)) n_procs = mp.cpu_count() all_jobs = [] for video_gt_pair in self._videos: gt = video_gt_pair[0] fp = video_gt_pair[1] for func in self._funcs: func_name = func[0] func_ptr = func[1] base_params = { 'gt_path': gt, 'video_path': fp, 'metric_func': func_ptr, 'init': False } for classifier in self._classifiers: params = base_params.copy() params.update(self._experiment_args) params['classifier'] = classifier(metric=func_name) log.info("Params ({}): {}".format(id(params), params)) all_jobs.append((params, self._experiment)) for job in all_jobs: tasks.put(job) for _ in range(n_procs): p = mp.Process(target=train_classifier, args=args).start() for _ in range(len(all_jobs)): interim.append(results.get()) results.task_done() for _ in range(n_procs): tasks.put(None) results.join() tasks.close() results.close() return interim
def __init__(self, depth, threading): # Logger already setup by config, just get an instance logobj = logging.getLogger('eventgen') from eventgenconfig import EventgenAdapter adapter = EventgenAdapter(logobj, { 'module': 'Queue', 'sample': 'null' }) self.logger = adapter # logger.info("Creating Queue of depth %d, threading %s" % (depth, threading)) if threading == 'thread': self.q = PQueue.Queue(depth) else: self.q = multiprocessing.Queue(depth) self.depth = depth
def get_meta(ds_path): """ This function is a wrapper for the get_gdal metadata because if there is a database diconnection there is no obvious way to clean up and free those resources therefore it is put on a separate process and if it fails it can just be tried again. This is using GDAL 2.2.4 this should be checked again to see if it can be simplified in a later version. :param ds_path: String: Path to dataset :return: Metadata dict driver: Short name of GDAL driver for dataset is_raster: True if dataset is a raster type nodata: NODATA value for all bands if all bands have the same one, otherwise None (raster sets only) """ multiprocess_queue = billiard.Queue() proc = billiard.Process(target=get_gdal_metadata, daemon=True, args=(ds_path, multiprocess_queue,)) proc.start() proc.join() return multiprocess_queue.get()
def run(self): globals()['threadrun'] = self.real_run cProfile.runctx("threadrun()", globals(), locals(), "prof_putter") def real_run(self): while True: l = [] # for i in xrange(20000): # l.append('2014-01-05 23:07:08 WINDBAG Event 1 of 100000') l = [ '2014-01-05 23:07:08 WINDBAG Event 1 of 100000' for i in xrange(20000) ] try: self.q.put(l, block=True, timeout=1.0) except Queue.Full: pass if __name__ == '__main__': print 'running!' q = multiprocessing.Queue(20) g = Getter(q) # g.daemon = True p = Putter(q) # p.daemon = True p.start() g.start()
def execute(self): """See base_runner.execute().""" import platform # In OS X the multiprocessing module is horribly broken, but a fixed # version has been released as the 'billiard' module if platform.system() == 'Darwin': import billiard as multiprocessing multiprocessing.forking_enable(0) else: import multiprocessing from libqtopensesame.misc import process, _ from libopensesame import misc self._workspace_globals = {} if os.name == u'nt' or (sys.platform == u'darwin' \ and not hasattr(sys,"frozen")): # Under Windows and OSX, the multiprocess runner assumes that there # is a file called `opensesame.py` or `opensesame.pyc`. If this file # does not exist, try to copy it from the main script # (`opensesame`). If this fails, provide an informative error # message. os_folder = misc.opensesame_folder() # misc.opensesame_folder() doesn't work for OSX and returns None then, # so determine OpenSesame's rootdir in another way if os_folder is None: os_folder = os.path.dirname( os.path.abspath(sys.modules['__main__'].__file__)) if not os.path.exists(os.path.join(os_folder, u'opensesame.pyc')) \ and not os.path.exists(os.path.join(os_folder, u'opensesame.py')): import shutil try: shutil.copyfile(os.path.join(os_folder, u'opensesame'), os.path.join(os_folder, u'opensesame.py')) except Exception as e: return osexception(_( u'Failed to copy `opensesame` to `opensesame.py`, which is required for the multiprocess runner. Please copy the file manually, or select a different runner under Preferences.' ), exception=e) self.channel = multiprocessing.Queue() try: self.exp_process = process.ExperimentProcess( self.experiment, self.channel) except Exception as e: return osexception(_(u'Failed to initialize experiment process'), exception=e) # Start process! self.exp_process.start() # Wait for experiment to finish. # Listen for incoming messages in the meantime. while self.exp_process.is_alive() or not self.channel.empty(): # We need to process the GUI. To make the GUI feel more responsive # during pauses, we refresh the GUI more often when paused. QtGui.QApplication.processEvents() if self.paused: for i in range(25): time.sleep(.01) QtGui.QApplication.processEvents() # Make sure None is not printed. Ugly hack for a bug in the Queue # class? self.console.suppress_stdout() # Wait for messages. Will throw Exception if no message is received # before timeout. try: msg = self.channel.get(True, 0.05) except: continue # Restore connection to stdout self.console.capture_stdout() if isinstance(msg, basestring): sys.stdout.write(safe_decode(msg, errors=u'ignore')) continue # Capture exceptions if isinstance(msg, Exception): return msg # The workspace globals are sent as a dict. A special __pause__ key # indicates whether the experiment should be paused or resumed. if isinstance(msg, dict): self._workspace_globals = msg if u'__heartbeat__' in msg: self.console.set_workspace_globals(msg) self.main_window.extension_manager.fire(u'heartbeat') elif u'__pause__' in msg: if msg[u'__pause__']: self.pause() else: self.resume() continue # Anything that is not a string, not an Exception, and not None is # unexpected return osexception( u"Illegal message type received from child process: %s (%s)" \ % (msg, type(msg))) # Return None if experiment finished without problems return None
def execute(self): """See base_runner.execute().""" import platform # In OS X the multiprocessing module is horribly broken, but a fixed # version has been released as the 'billiard' module if platform.system() == 'Darwin': import billiard as multiprocessing multiprocessing.forking_enable(0) else: import multiprocessing from libqtopensesame.misc import process, _ from libopensesame import misc, debug from StringIO import StringIO if os.name == u'nt': # Under Windows, the multiprocess runner assumes that there is a # file called `opensesame.py` or `opensesame.pyc`. If this file does # not exist, try to copy it from the main script (`opensesame`). If # this fails, provide an informative error message. os_folder = misc.opensesame_folder() if not os.path.exists(os.path.join(os_folder, u'opensesame.pyc')) \ and not os.path.exists(os.path.join(os_folder, u'opensesame.py')): import shutil try: shutil.copyfile(os.path.join(os_folder, u'opensesame'), \ os.path.join(os_folder, u'opensesame.py')) except Exception as e: return osexception( \ _(u'Failed to copy `opensesame` to `opensesame.py`, which is required for the multiprocess runner. Please copy the file manually, or select a different runner under Preferences.'), exception=e) self.channel = multiprocessing.Queue() self.exp_process = process.ExperimentProcess(self.experiment, \ self.channel) # Start process! self.exp_process.start() # Variables used for ugly hack to suppress 'None' print by Queue.get() _stdout = sys.stdout _pit = StringIO() # Wait for experiment to finish. # Listen for incoming messages in the meantime. while self.exp_process.is_alive() or not self.channel.empty(): QtGui.QApplication.processEvents() # Make sure None is not printed. Ugly hack for a bug in the Queue # class? sys.stdout = _pit # Wait for messages. Will throw Exception if no message is received # before timeout. try: msg = self.channel.get(True, 0.05) except: msg = None # Restore connection to stdout sys.stdout = _stdout # For standard print statements if isinstance(msg, basestring): sys.stdout.write(msg) # Errors arrive as a tuple with (Error object, traceback) elif isinstance(msg, Exception): return msg # Anything that is not a string, not an Exception, and not None is # unexpected elif msg != None: return osexception( \ u"Illegal message type received from child process: %s (%s)" \ % (msg, type(msg))) # Return None if experiment finished without problems return None
def fill_template( template_name, context, output_format='odt', options=None, separate_process=True, ): """Fill a document with data and convert it to the requested format. Returns an absolute path to the generated file. Supported output format: Text documents: doc, docx, fodt, html, odt, ott, pdf, txt, xhtml, png Spreadsheets: csv, fods, html, ods, ots, pdf, xhtml, xls, xlsx, png Presentations: fodp, html, odg, odp, otp, pdf, potm, pot, pptx, pps, ppt, svg, swf, xhtml, png Drawings: fodg, html, odg, pdf, svg, swf, xhtml, png More on filter options, https://wiki.openoffice.org/wiki/Documentation/DevGuide/Spreadsheets/Filter_Options # noqa: E501 :param template_name: the path to template, in OpenDocument format :param context: the context to be used to inject content :param output_format: the output format :param options: value of filterOptions in libreofficekit :param separate_process: allow LO to :return: """ if not isinstance(context, Context): context = Context(context) context['output_format'] = output_format source_file = find_template_file(template_name) source_extension = os.path.splitext(source_file)[1] source = zipfile.ZipFile(source_file, 'r') dest_file = NamedTemporaryFile(delete=False, suffix=source_extension) dest = zipfile.ZipFile(dest_file, 'w') manifest_data = '' for name in source.namelist(): data = source.read(name) if name.endswith('.xml'): data = smart_str(data) if any(name.endswith(file) for file in ('content.xml', 'styles.xml')): template = Template(fix_inline_tags(data)) data = template.render(context) elif name == 'META-INF/manifest.xml': manifest_data = data[:-20] # Cut off the closing </manifest> tag continue # We will append it at the very end dest.writestr(name, smart_bytes(data)) for _, image in context.dicts[0].get(IMAGES_CONTEXT_KEY, {}).items(): filename = os.path.basename(image.name) ext = os.path.splitext(filename)[1][1:] manifest_data += ( '<manifest:file-entry ' 'manifest:media-type="image/%(ext)s" ' 'manifest:full-path="Pictures/%(filename)s"/>\n') % locals() image.open() dest.writestr('Pictures/%s' % filename, image.read()) image.close() manifest_data += '</manifest:manifest>' dest.writestr('META-INF/manifest.xml', manifest_data) source.close() dest.close() if source_extension[1:] != output_format: if separate_process: results = multiprocessing.Queue() converter = multiprocessing.Process( target=_convert_file, args=(str(dest_file.name), output_format, results, options), ) converter.start() return results.get() else: return _convert_file( filename=str(dest_file.name), format=output_format, options=options, ) else: return dest_file.name
def execute(self): """See base_runner.execute().""" import platform if platform.system() == 'Darwin' and \ sys.version_info < (3,4): # In OS X the multiprocessing module is horribly broken, # for python 2.7 but a fixed version has been released # as the 'billiard' module import billiard as multiprocessing else: import multiprocessing from libqtopensesame.misc import process, _ self._workspace_globals = {} self.channel = multiprocessing.Queue() try: self.exp_process = process.ExperimentProcess( self.experiment, self.channel) except Exception as e: return osexception(_(u'Failed to initialize experiment process'), exception=e) # Start process! self.exp_process.start() # Wait for experiment to finish. # Listen for incoming messages in the meantime. finished = False while self.exp_process.is_alive() or not self.channel.empty(): # We need to process the GUI. To make the GUI feel more responsive # during pauses, we refresh the GUI more often when paused. QtWidgets.QApplication.processEvents() if self.paused: for i in range(25): time.sleep(.01) QtWidgets.QApplication.processEvents() # Make sure None is not printed. Ugly hack for a bug in the Queue # class? self.console.suppress_stdout() # Wait for messages. Will throw Exception if no message is received # before timeout. try: msg = self.channel.get(True, 0.05) except: continue # Restore connection to stdout self.console.capture_stdout() if isinstance(msg, basestring): sys.stdout.write(safe_decode(msg, errors=u'ignore')) continue # Capture exceptions if isinstance(msg, Exception): return msg # The workspace globals are sent as a dict. A special __pause__ key # indicates whether the experiment should be paused or resumed. if isinstance(msg, dict): self._workspace_globals = msg if u'__heartbeat__' in msg: self.console.set_workspace_globals(msg) self.main_window.extension_manager.fire(u'heartbeat') elif u'__pause__' in msg: if msg[u'__pause__']: self.pause() else: self.resume() elif u'__finished__' in msg: finished = True continue # Anything that is not a string, not an Exception, and not None is # unexpected return osexception( u"Illegal message type received from child process: %s (%s)" \ % (msg, type(msg))) if not finished: return osexception( u'Python seems to have crashed. This should not ' u'happen. If Python crashes often, please report it on the ' u'OpenSesame forum.')
def run_exp_for_all_classifiers(save_dir=DIR_CLASSIFIERS, parallel=True): """ Runs all the saved classifiers that are located in save_dir. parallel, if True, will use the multiprocessing module to run multiple experiments at the same time. At present, however, this is broken due to the way in which Python processes match up to C-lib extensions. In this case, OpenCV just kinda dies when processing is attempted in this manner. Currently investigating a fix -- until then, just run linear or via threads. """ classifiers = EXPClassifierHandler.get_all_saved_classifiers( DIR_CLASSIFIERS) classifiers = [x for x in classifiers if not x.endswith(".csv")] if len(classifiers) == 0: log.info("No more experiments to run, exiting.") return if parallel: videos_to_classifiers = {} for c in classifiers: clf = load_saved_classifier(save_dir + c) file_name = clf.video_path.split("/")[-1] if file_name not in videos_to_classifiers: videos_to_classifiers[file_name] = [] clfid = (clf.identifier, c) videos_to_classifiers[file_name].append(clfid) # So now we've mapped video_file: [classifiers], multiproc by k tasks = mp.Queue() results = mp.JoinableQueue() interim = [] args = (tasks, results, save_dir) n_procs = min(mp.cpu_count(), len(videos_to_classifiers.keys())) for k in videos_to_classifiers.keys(): these_classifiers = videos_to_classifiers[k] tasks.put(these_classifiers) delegator = EXPClassifierHandler.run_exp_from_mp_queue for _ in range(n_procs): p = mp.Process(target=delegator, args=args).start() for _ in range(len(videos_to_classifiers.keys())): interim.append(results.get()) results.task_done() for _ in range(n_procs): tasks.put(None) results.join() tasks.close() results.close() else: for c in classifiers: EXPClassifierHandler.run_exp_for_classifier(c, save_dir) # Maybe by the time we get here more will be waiting... keep going EXPClassifierHandler.run_exp_for_all_classifiers(save_dir, parallel)