def tag(self): iqueue = parallel.manager.Queue(self.settings.QUEUE_SIZE) process = self._start_streaming(iqueue) count = parallel.run(do, aggregate, iqueue, self.num_processes) process.join() return count
def test_run_without_return(self): iqueue = parallel.manager.Queue() process = multiprocessing.Process( target=stream, args=(list(range(self.count)), iqueue, 2) ) process.start() expected = [(i + 10) for i in range(self.count)] aggregate = aggregate_without_return actual = parallel.run(do, aggregate, iqueue, 2) self.assertIsNone(actual) process.join()
def load(self): """ Grabs all of the bugs from within the specified range of years, parses through them, cleans them up, then saves them. Returns the total number of bugs loaded. """ count = 0 iqueue = parallel.manager.Queue(self.settings.QUEUE_SIZE) process = self._start_streaming(iqueue) count = parallel.run(do, aggregate, iqueue, self.num_processes) process.join() return count
def compute(review_ids, idf, num_procs, key='lemma'): if idf is None or type(idf) is not dict: raise ValueError('Argument IDF must be a dictionary!') global IDF, KEY IDF = idf KEY = key iqueue = parallel.manager.Queue() proc = multiprocessing.Process(target=stream, args=(review_ids, iqueue, num_procs)) proc.start() tfidfs = parallel.run(do, aggregate, iqueue, num_procs) proc.join() return tfidfs
def load(self): """ Grabs all of the reviews created within the specified range of years, parses them, cleans them up, and saves them. Returns the total number of loaded reviews. """ count = 0 iqueue = parallel.manager.Queue(self.settings.QUEUE_SIZE) process = self._start_streaming(iqueue) count = parallel.run(do, aggregate, iqueue, self.num_processes) process.join() self._cluster() return count