def main(): stage = mpipe.OrderedStage(echo) pipe = mpipe.Pipeline(stage) for val in (0, 1, 2, 3): pipe.put(val) pipe.put(None) # Stop the pipeline.
def main(): stage = mpipe.UnorderedStage(for_loop, 2) pipe = mpipe.Pipeline(stage) for foobar in range(5): pipe.put(int(sys.argv[1]) if len(sys.argv) >= 2 else 10) pipe.put(None)
def main(): stage1 = mpipe.Stage(Incrementor, 3) stage2 = mpipe.Stage(Doubler, 3) stage1.link(stage2) pipe = mpipe.Pipeline(stage1) for number in range(10): pipe.put(number) pipe.put(None) for result in pipe.results(): print(result)
def main(watched_path, mountpoint, logfile="test_log.csv"): oh = OutputHandler.OutputHandler(logfile) def signal_handler(sig, frame): # TODO maybe bug - needs two Ctrl+C to exit oh.close() signal.signal(signal.SIGINT, signal_handler) stage1 = mpipe.UnorderedStage(scanner.scan_file, size=3, max_backlog=3) stage2 = mpipe.OrderedStage(oh.write, size=1) pipeline = mpipe.Pipeline(stage1.link(stage2)) watcher = Watcher(watched_path, pipeline) watcher.main([sys.argv[0], mountpoint], foreground=True)
# Parse files in a pipeline. def parseFile(fname): """Parse the XML file looking for fully demangled class names, and communicate the result.""" names = list() doc = xml.dom.minidom.parse(fname) classes = doc.getElementsByTagName('Class') for entry in classes: name = entry.getAttribute('demangled') NSPACE = 'Wm5::' if name[:len(NSPACE)] != NSPACE: continue names.append(name) return names pipe = mpipe.Pipeline(mpipe.UnorderedStage(parseFile, num_cpus)) for fname in fnames: pipe.put(fname) pipe.put(None) # Report on progress in realtime. total_names = dict() done_count = 0 for result in pipe.results(): for name in result: total_names[name] = None done_count += 1 percent = float(done_count) / len(fnames) * 100 sys.stdout.write('\r' + '%d of %d (%.1f%%)'%(done_count, len(fnames), percent)) sys.stdout.flush()
# filter_detector = mpipe.FilterStage( detector_stages, max_tasks=1, cache_results=True, ) postproc = mpipe.Stage(Postprocessor) filter_viewer = mpipe.FilterStage( (mpipe.Stage(Viewer), ), max_tasks=2, drop_results=True, ) filter_detector.link(postproc) postproc.link(filter_viewer) pipe_iproc = mpipe.Pipeline(filter_detector) # Create an auxiliary process (modeled as a one-task pipeline) # that simply pulls results from the image processing pipeline, # and deallocates associated shared memory after allowing # the designated amount of time to pass. def deallocate(tdelta): for tstamp in pipe_iproc.results(): elapsed = datetime.datetime.now() - tstamp if tdelta - elapsed > datetime.timedelta(): time.sleep(tdelta.total_seconds()) del images[tstamp] pipe_dealloc = mpipe.Pipeline(mpipe.UnorderedStage(deallocate))
import mpipe def echo(value): print(value) stage = mpipe.OrderedStage(echo) pipe = mpipe.Pipeline(stage) for val in (0, 1, 2, 3): pipe.put(val) pipe.put(None) # Stop the pipeline.
'multiwork', 'filter', ) # Export Dia diagrams. saved = os.getcwd() os.chdir('source') def runDia(diagram): """Generate the diagrams using Dia.""" ifname = '{}.dia'.format(diagram) ofname = '{}.png'.format(diagram) cmd = 'dia -t png-libart -e {} {}'.format(ofname, ifname) print(' {}'.format(cmd)) subprocess.call(cmd, shell=True) return True pipe = mpipe.Pipeline(mpipe.UnorderedStage(runDia, len(diagrams))) for diagram in diagrams: pipe.put(diagram) pipe.put(None) for result in pipe.results(): pass os.chdir(saved) # Copy the .py examples from test/ to source/ directory # so that they can be picked up by the Sphinx build. codes = ( 'tiny.py', 'helloworld.py', 'chain.py', 'pipeout.py', 'fork.py',
num_cpus = multiprocessing.cpu_count() print('Running %d commands on %d CPUs' % (len(commands), num_cpus)) # Run commands in a pipeline. def runCommand(command): """Run the given command in a subprocess shell.""" result = subprocess.call(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) return result pipe = mpipe.Pipeline(mpipe.UnorderedStage(runCommand, num_cpus)) for command in commands: if OPTS['verbose']: print(command) pipe.put(command) pipe.put(None) # Report on progress in realtime. num_succeeded = 0 for result in pipe.results(): num_succeeded += int(not result) percent = float(num_succeeded) / len(commands) * 100 sys.stdout.write('\r' + '%d of %d (%.1f%%)' % (num_succeeded, len(commands), percent)) sys.stdout.flush()
if __name__ == "__main__": for f in range(2): frame = camera.capture(encoding="raw") ls[f] = np.ctypeslib.as_array(frame.buffer_ptr[0].data,shape=(800,1280)) del frame i = 2 cont = True while cont: stage1 = mpipe.OrderedStage(cap, 1) stage2 = mpipe.OrderedStage(processing, 1) pipe = mpipe.Pipeline(stage1.link(stage2)) pipe.put(i,camera,ls) if(i==100): i=0 else: i+=1 for result in pipe.results(): counter = result if counter >pix_threshold: cont = False for i in range(100): im = Image.fromarray(ls[i]) im = im.convert("L")
cv2.imshow('diff average 4', common[tstamp]['image_diff']) cv2.waitKey(1) # Allow HighGUI to process event. return tstamp # Assemble the pipeline. stage1 = mpipe.Stage(Step1) stage2 = mpipe.FilterStage( (mpipe.OrderedStage(step2),), max_tasks=2, # Allow maximum 2 tasks in the viewer stage. drop_results=True, ) stage1.link(stage2) pipe = mpipe.Pipeline( mpipe.FilterStage( (stage1,), max_tasks=3, # Allow maximum 3 tasks in pipeline. drop_results=True, ) ) # Create an auxiliary process (modeled as a one-task pipeline) # that simply pulls results from the image processing pipeline, # and deallocates associated shared memory after allowing # the designated amount of time to pass. def deallocate(age): for tstamp in pipe.results(): delta = datetime.datetime.now() - tstamp duration = datetime.timedelta(seconds=age) - delta if duration > datetime.timedelta(): time.sleep(duration.total_seconds()) del common[tstamp]
def buildTrainingSet(DF, inArray, outArray, storeSourceImg=True): outTsv = outArray.replace('.h5', '.tsv') modelFile = sys.argv[1] #nChannels = len(gradientFunctions) nChannels = 1 if storeSourceImg: nChannels += 1 CAM_SHAPE = (140, 140, 140, nChannels) class cubeSource(mpipe.OrderedWorker): def doInit(self): self.sparseImages = SparseImageSource(inArray) def doTask(self, row): cubes, positions = self.sparseImages.getCubesAndPositions(row, posType='pos') self.putResult((row, cubes, positions)) print 'returning image' class camImgMaker(mpipe.OrderedWorker): def doInit(self): from keras.models import load_model, Sequential from gradCam import register_gradient, modify_backprop, compile_saliency_function, normalize from gradCam import target_category_loss, target_category_loss_output_shape, buildGradientFunction from gradCam import buildSoftmaxGradientFunction from gradCam import batchImages, gradCamsFromList, saliencyMapsFromList, makeCamImageFromCubesFaster from gradCam import makeCamImgFromImage, makeResizedSourceImage self.model = load_model(modelFile) # single output neuron cams noduleGrad, cubeCamSize = buildGradientFunction(self.model) # diamGrad, cubeCamSize = buildGradientFunction(model, output='diam') self.gradientFunctions = [noduleGrad] nChannels = len(self.gradientFunctions) global nChannels self.cubeCamSize = cubeCamSize # softmax output models # noduleGrad, cubeCamSize = buildSoftmaxGradientFunction(model, 0) def doTask(self, task): row, cubes, positions = task from gradCam import makeCamImageFromCubesFaster camImage = makeCamImageFromCubesFaster(cubes, positions, self.gradientFunctions, self.cubeCamSize, storeSourceImg=storeSourceImg) print 'returning cubes and positions' return row, camImage class storeCams(mpipe.OrderedWorker): def doInit(self): DBo = tables.open_file(outArray, mode='w') filters = tables.Filters(complevel=1, complib='blosc:snappy') # 7.7sec / 1.2 GB (14 sec 1015MB if precision is reduced) 140s 3.7GB # filters = None self.cams = DBo.create_earray(DBo.root, 'cams', atom=tables.Float32Atom(shape=CAM_SHAPE), shape=(0,), expectedrows=len(DF), filters=filters) self.camImageDF = pandas.DataFrame() def doTask(self, task): row, camImage = task if camImage.mean() == 0: print 'THIS IMAGE IS BAD ========================' print camImage.shape print 'nodule image mean %s min %s max %s : ' % ( camImage[:,:,:,0].mean(), camImage[:,:,:,0].min(), camImage[:,:,:,0].max()) print 'diam image mean %s min %s max %s : ' % ( camImage[:,:,:,1].mean(), camImage[:,:,:,1].min(), camImage[:,:,:,1].max()) #print 'source image mean %s min %s max %s : ' % ( # camImage[:,:,:,2].mean(), camImage[:,:,:,2].min(), camImage[:,:,:,2].max()) cam = forceImageIntoShape(camImage, CAM_SHAPE) #cam = resize(camImage, CAM_SHAPE) #crop = boundingBox(camImage, channel=0) print cam.shape self.cams.append([cam]) self.camImageDF = self.camImageDF.append(row) self.camImageDF.to_csv(outTsv, sep='\t') print 'starting workers' stage1 = mpipe.Stage(cubeSource, 1, ) stage2 = mpipe.Stage(camImgMaker, 1) stage3 = mpipe.Stage(storeCams, 1, disable_result=True) stage1.link(stage2.link(stage3)) pipe = mpipe.Pipeline(stage1) for index, row in tqdm(DF.iterrows(), total=len(DF)): print 'putting row', index pipe.put(row)
def step2(tstamp): """Display the image, stamped with framerate.""" fps_text = '{:.2f}, {:.2f}, {:.2f} fps'.format(*framerate.tick()) util.writeOSD(common[tstamp]['image_diff'], (fps_text, )) cv2.imshow('diff average 3', common[tstamp]['image_diff']) cv2.waitKey(1) # Allow HighGUI to process event. return tstamp # Assemble the pipeline. stage1 = mpipe.Stage(Step1) stage2 = mpipe.OrderedStage(step2) stage1.link(stage2) pipe = mpipe.Pipeline(stage1) # Create an auxiliary process (modeled as a one-task pipeline) # that simply pulls results from the image processing pipeline, # and deallocates associated shared memory after allowing # the designated amount of time to pass. def deallocate(age): for tstamp in pipe.results(): delta = datetime.datetime.now() - tstamp duration = datetime.timedelta(seconds=age) - delta if duration > datetime.timedelta(): time.sleep(duration.total_seconds()) del common[tstamp]