def process_reply(self, frames): parts = len(frames) # for each batch of images, we save an entry ... # how to handle multiple batches? will need a batch to eventid map # when eventid changes, then save for i in range(0, parts, 3): name = frames[i].decode("ascii") metamsg = frames[i + 1] x_enc = frames[i + 2] meta = decode_larcv1_metamsg(metamsg) arr = msgpack.unpackb(x_enc, object_hook=m.decode) print "SimpleLArCV1Client[{}] received array name=\"{}\" shape={} meta={}".format( self._identity, name, arr.shape, meta.dump().strip()) nbatches = arr.shape[0] for ib in range(nbatches): rse = self.batch2rse[ib] if self.current_rse is not None and rse != self.current_rse: # save output entry self.io_out.set_id(rse[0], rse[1], rse[2]) print "SimpleLArCV1Client[{}] saving entry {}".format( self._identity, rse) self.io_out.save_entry() self.io_out.clear_entry() self.current_rse = rse output_ev_container = self.io_out.get_data( larcv.kProductImage2D, str(name)) for i in range(arr.shape[1]): meta = decode_larcv1_metamsg(metamsg, i) img = larcv.as_image2d_meta( np.transpose(arr[ib, i, :], (1, 0)), meta) output_ev_container.Append(img) # save output entry self.io_out.set_id(rse[0], rse[1], rse[2]) self.io_out.save_entry() self.io_out.clear_entry() self.current_rse = rse
def decode_larcv2_evimage2d(io, producername, imgdata_np, imgmeta_np): # make evcontainer evout = io.get_data("image2d", producername) # make meta nimgs = imgdata_np.shape[1] for i in xrange(nimgs): nrows = Long(imgmeta_np[0, i, 0, 1]) ncols = Long(imgmeta_np[0, i, 0, 0]) planeid = Long(imgmeta_np[0, i, 0, 6]) lcvmeta = larcv.ImageMeta(imgmeta_np[0, i, 0, 2], imgmeta_np[0, i, 0, 3], imgmeta_np[0, i, 0, 4], imgmeta_np[0, i, 0, 5], nrows, ncols, planeid) # convert image outarr = np.flip(imgdata_np[0, i, :, :].transpose((1, 0)), 0) lcvimg = larcv.as_image2d_meta(outarr, lcvmeta) evout.append(lcvimg) return
for row in xrange(src_np.shape[1]): flow_np[:, row] += colidx[:] flow_np[src_np < 10] = 0 flow_np[flow_np < 0] = 0 flow_np[flow_np >= src_np.shape[0]] = 0 flowed_np = np.zeros(src_np.shape, dtype=np.float32) pixlist = np.argwhere(flow_np != 0) print("pixlist: ", pixlist.shape) for idx in xrange(pixlist.shape[0]): row = pixlist[idx, 1] src_col = pixlist[idx, 0] tar_col = flow_np[src_col, row] #print(row,src_col,"to",tar_col) flowed_np[tar_col, row] = src_np[src_col, row] flowed_lcv = larcv.as_image2d_meta(flowed_np, tarimg.meta()) hflowed = larcv.as_th2d(flowed_lcv, "hflowed_{}".format(iset)) hflowed.GetZaxis().SetRangeUser(0, 100) hflowed.SetTitle("Flowed Pixels from {};wire;tick".format(flowdir)) c.cd(1) hsrc.Draw("COLZ") c.cd(2) hflowed.Draw("COLZ") c.cd(3) htar.Draw("COLZ") c.Update() c.Draw()
# get probabilities from 0-1 labels_np = 10**labels_np for ib in range(batch_size): if ientry >= nevts: # skip last portion of last batch break inputmeta.read_entry(ientry) ev_meta = inputmeta.get_data("image2d", "wire") outmeta = ev_meta.image2d_array()[2].meta() # save output of network as images img_slice0 = labels_np[ib, 0, :, :] if SAVE_NOFILL: nofill_lcv = larcv.as_image2d_meta(img_slice0, outmeta) ev_nofill_out = outputdata.get_data("image2d", "nofill") ev_nofill_out.append(nofill_lcv) img_slice1 = labels_np[ib, 1, :, :] if SAVE_FILL: fill_lcv = larcv.as_image2d_meta(img_slice1, outmeta) ev_fill_out = outputdata.get_data("image2d", "fill") ev_fill_out.append(fill_lcv) #save inputs to network for reference wire_slice = wire_t[ib, 0, :, :] wire_out = larcv.as_image2d_meta(wire_slice, outmeta) ev_out_wire.append(wire_out) weight_slice = weight_t[ib, 0, :, :]
# setting uninteresting pixels to zero is important for good file size thresh_slice = (source_np[ib, 0, :, :] < threshold) #adcgoodch_slice = source_np[ib,0,:,status_batch[ib]==4].transpose((1,0)) # SKIP CHSTATUS MASK FOR NOW #nmasked = 0 #for n,goodch in enumerate( np.nditer( status_batch[ib] ) ): # if goodch==4: # nmasked += (flow_slice[:,n]<threshold).sum() # flow_slice[:,n][ source_np[ib,0,:,n]<threshold ] = 0 # zero regions in good channel list and below threshold #print "nmasked estimate: ",nmasked #print "flow_slice non-zero (post-mask): ",(flow_slice!=0).sum() flow_lcv = larcv.as_image2d_meta( np.transpose(flow_slice, (1, 0)), image_meta[ib]) if stitch: # load cropped info into stitcher # ------------------------------- stitcher.insertFlowSubimage(flow_lcv, target_meta[ib]) else: # save cropped images #-------------------- # prediction images evoutpred = outputdata.get_data( larcv.kProductImage2D, "larflow_%s" % (FLOWDIR.lower())) evoutpred.Append(flow_lcv) # input cropped source and target image if save_cropped_adc:
# ------------- # Store results tcopy = time.time() for ib in xrange(min(batch_size, len(image_meta[p]))): isgood = True for p in xrange(3): if image_meta[p][ib] is None: isgood = False if not isgood: continue # convert data to larcv ssnet_lcv = {} ssnet_lcv["track"] = [ larcv.as_image2d_meta(result_np[p][ib, 1, :], image_meta[p][ib]) for p in xrange(3) ] ssnet_lcv["shower"] = [ larcv.as_image2d_meta(result_np[p][ib, 2, :], image_meta[p][ib]) for p in xrange(3) ] ssnet_lcv["endpt"] = [ larcv.as_image2d_meta(result_np[p][ib, 3, :], image_meta[p][ib]) for p in xrange(3) ] # if stiching, store into stitch if stitch: outmeta = out_v[p].meta() # stitch meta for p in xrange(3): stitcher_track.insertFlowSubimage(
def process_precropped_reply(self, frames): # message parts consist of ssnet output # one part contains a batch for one plane # we must collect data for all three planes for an event, before writing it to disk # by construction, one batch is for one event # one message contains one batch # this makes it a lot easier to understand # someone smarter can make general code treply = time.time() plane_img_v_dict = {} parts = len(frames) for i in range(0, parts, 3): name = frames[i].decode("ascii") metamsg = frames[i + 1] x_comp = frames[i + 2] x_enc = zlib.decompress(x_comp) if name not in plane_img_v_dict: plane_img_v_dict[name] = [ std.vector("larcv::Image2D")() for x in range(self.NPLANES) ] meta = decode_larcv1_metamsg(metamsg) arr = msgpack.unpackb(x_enc, object_hook=m.decode) nbatches = arr.shape[0] print "CaffeLArCV1Client[{}] received array name=\"{}\" shape={} meta={} batchsize={}".format( self._identity, name, arr.shape, meta.dump().strip(), nbatches) for ib in range(nbatches): # set the RSE rse = self.batch2rse[ib] self.current_rse = rse #img = larcv.as_image2d_meta( np.transpose( arr[ib,0,:], (1,0) ), meta ) # not needed? img = larcv.as_image2d_meta(arr[ib, 0, :], meta) print "fill ", name, " meta=", meta.dump().strip() plane_img_v_dict[name][meta.plane()].push_back(img) # make output event containers print "CaffeLArCV1Client[{}] storing images".format(self._identity) for name, plane_img_v in plane_img_v_dict.items(): print "name: ", name, plane_img_v for img_v in plane_img_v: print img_v print img_v.size() planeid = img_v.front().meta().plane() print "CaffeLArCV1Client[{}] storing name={} plane={}".format( self._identity, name, planeid) outname = "%s_plane%d" % (str(name.decode("ascii")), planeid) print "Filling event container: ", outname output_ev_container = self.io_out.get_data( larcv.kProductImage2D, outname) for iimg in range(img_v.size()): output_ev_container.Append(img_v[iimg]) # save output entry self.io_out.set_id(rse[0], rse[1], rse[2]) self.io_out.save_entry() self.io_out.clear_entry() self.current_rse = rse treply = time.time() - treply self._ttracker["savereply::total"] += treply
for out, img2d in zip(out_v, crop_img2d_v): # threshold scores for better compression showerslice = out[:, 1, :, :].reshape(512, 512) trackslice = out[:, 2, :, :].reshape(512, 512) bgslice = out[:, 0, :, :].reshape(512, 512) meta = img2d.meta() # back to image2d shrmeta = larcv.ImageMeta(meta.width(), meta.height(), meta.rows(), meta.cols(), meta.min_x(), meta.min_y(), 0) trkmeta = larcv.ImageMeta(meta.width(), meta.height(), meta.rows(), meta.cols(), meta.min_x(), meta.min_y(), 1) bgmeta = larcv.ImageMeta(meta.width(), meta.height(), meta.rows(), meta.cols(), meta.min_x(), meta.min_y(), 2) showercrop = larcv.as_image2d_meta(showerslice, shrmeta) shower_img.overlay(showercrop, larcv.Image2D.kOverWrite) trackcrop = larcv.as_image2d_meta(trackslice, trkmeta) track_img.overlay(trackcrop, larcv.Image2D.kOverWrite) bgcrop = larcv.as_image2d_meta(bgslice, bgmeta) bg_img.overlay(bgcrop, larcv.Image2D.kOverWrite) h2d = larcv.as_th2d(shower_img, "shower_img") c.Clear() h2d.Draw("colz") c.Update() print "enter to go to next image" raw_input() ev_ssnetout = outio.get_data(larcv.kProductImage2D, "ssnet_plane2") ev_ssnetout.Append(shower_img)
for ib in range(batch_size): if ientry >= nevts: # skip last portion of last batch break # evtinfo = data["event_ids"][ib] #Change event_ids to just id? #outmeta = data["source_test"][ib].meta() inputmeta.read_entry(ientry) ev_meta = inputmeta.get_data("image2d", "adc") # if ev_meta.run()!=evtinfo.run() or ev_meta.subrun()!=evtinfo.subrun() or ev_meta.event()!=evtinfo.event(): # raise RuntimeError("(run,subrun,event) for evtinfo and ev_meta do not match!") outmeta = ev_meta.image2d_array()[2].meta() img_slice0 = labels_np[ib, 0, :, :] background_lcv = larcv.as_image2d_meta(img_slice0, outmeta) ev_out = outputdata.get_data("image2d", "background") ev_out.append(background_lcv) img_slice1 = labels_np[ib, 1, :, :] track_lcv = larcv.as_image2d_meta(img_slice1, outmeta) ev_out = outputdata.get_data("image2d", "track") ev_out.append(track_lcv) img_slice2 = labels_np[ib, 2, :, :] shower_lcv = larcv.as_image2d_meta(img_slice2, outmeta) ev_out = outputdata.get_data("image2d", "shower") ev_out.append(shower_lcv) img_slice3 = labels_np[ib, 3, :, :] track_end_lcv = larcv.as_image2d_meta(img_slice3, outmeta)
"imgdump/img_%d_%d_%d_target.png" % (ientry, iset + 1, ib), crop_np[ib, target_plane, :, :]) # store region of image #image_meta.append( larcv.ImageMeta( bb_v[2], 512, 832 ) ) #target_meta.append( larcv.ImageMeta( bb_v[target_plane], 512, 832 ) ) image_meta.append(bb_v[2]) target_meta.append(bb_v[target_plane]) adc_metas.append(bb_v) # if not stiching, save crops if not stitch: flowcrops = {"flow": [], "visi": [], "adc": []} if save_cropped_adc: for p in xrange(0, 3): adc_lcv = larcv.as_image2d_meta( crop_np[ib, p, :].transpose((1, 0)), bb_v[p]) flowcrops["adc"].append(adc_lcv) #if target_plane==0: # adc_lcv0 = larcv.as_image2d_meta( target_np[ib,0,:].transpose((1,0)), bb_v[0] ) # flowcrops["adc"].append( adc_lcv0 ) # adc_lcv1 = larcv.as_image2d_meta( np.flip(img_np[1,0,bounds[1][0]:bounds[1][2],bounds[1][1]:bounds[1][3]],0).transpose((1,0)), bb_v[1] ) # flowcrops["adc"].append( adc_lcv1 ) #elif target_plane==1: # adc_lcv0 = larcv.as_image2d_meta( np.flip(img_np[0,0,bounds[0][0]:bounds[0][2],bounds[0][1]:bounds[0][3]],0).transpose((1,0)), bb_v[0] ) # flowcrops["adc"].append( adc_lcv0 ) # adc_lcv1 = larcv.as_image2d_meta( target_np[ib,0,:].transpose((1,0)), bb_v[1] ) # flowcrops["adc"].append( adc_lcv1 ) #adc_lcv2 = larcv.as_image2d_meta( source_np[ib,0,:].transpose((1,0)), bb_v[2] ) #flowcrops["adc"].append( adc_lcv2 ) if ismc:
t_forward = time.time() output_np = [ predict_fn({"uplane": input_np[p]})["pred"] for p in range(NPLANES) ] timer["forward"] += time.time() - t_forward # write to disk t_disk = time.time() event_ssnet_containers = [ out.get_data(larcv.kProductImage2D, "ssnet_plane%d" % (p)) for p in range(NPLANES) ] for p in range(NPLANES): for c in range(NCLASSES): event_ssnet_containers[p].Append( larcv.as_image2d_meta(output_np[p][0, :, :, c], img_v[p].meta())) out.set_id(event_image_container.run(), event_image_container.subrun(), event_image_container.event()) out.save_entry() timer["writeout"] += time.time() - t_disk timer["totentry"] += time.time() - t_entry print "End of entry loop" print "Finalize output" out.finalize() timer["total"] = time.time() - timer["total"] print "Timing for different steps" print "--------------------------" for k, i in timer.items():
trun = time.time() pred_flow, pred_visi = model.forward( source_t, target_t ) trun = time.time()-trun timing["+++run_model"] += trun if verbose: print "time to run model: ",trun," secs" # turn pred_flow back into larcv tcopy = time.time() flow_np = pred_flow.detach().cpu().numpy().astype(np.float32) outmeta = out_v[0].meta() for ib in range(min(batch_size,len(image_meta))): if image_meta[ib] is None: continue img_slice = flow_np[ib,0,:] flow_lcv = larcv.as_image2d_meta( img_slice, image_meta[ib] ) stitcher.insertFlowSubimage( flow_lcv, target_meta[ib] ) tcopy = time.time()-tcopy timing["+++copy_to_output"] += tcopy if verbose: print "time to copy results back into full image: ",tcopy," secs" # end of while loop if verbose: print "Processed all the images" tout = time.time() outputdata.read_entry(ientry) stitcher.process( outputdata ) outputdata.save_entry()
print("Retrieving larflow truth...") ev_larflow = io.get_data(larcv.kProductImage2D, "larflow") flow_v = ev_larflow.Image2DArray() if args.has_wirecell: # make wirecell masked image print("making wirecell masked image") start_wcmask = time.time() ev_wcthrumu = io.get_data(larcv.kProductImage2D, "thrumu") ev_wcwire = io.get_data(larcv.kProductImage2D, "wirewc") for p in xrange(adc_v.size()): adc = larcv.Image2D(adc_v[p]) # a copy np_adc = larcv.as_ndarray(adc) np_wc = larcv.as_ndarray(ev_wcthrumu.Image2DArray()[p]) np_adc[np_wc > 0.0] = 0.0 masked = larcv.as_image2d_meta(np_adc, adc.meta()) ev_wcwire.Append(masked) adc_v = ev_wcwire.Image2DArray() end_wcmask = time.time() print("time to mask: ", end_wcmask - start_wcmask, " secs") t_badch = time.time() badch_v = badchmaker.makeBadChImage(4, 3, 2400, 6 * 1008, 3456, 6, 1, ev_badch) print("Number of badcv images: ", badch_v.size()) gapch_v = badchmaker.findMissingBadChs(adc_v, badch_v, 10.0, 100) for p in xrange(badch_v.size()): for c in xrange(badch_v[p].meta().cols()): if (gapch_v[p].pixel(0, c) > 0): badch_v[p].paint_col(c, 255) dt_badch = time.time() - t_badch
for p in range(0,NPLANES): nets[p].forward() timer["forward"] += time.time()-t_forward # retrive the data t_out = time.time() for p in range(0,NPLANES): output_np[p][0,:,:,:] = nets[p].blobs['softmax'].data[:] timer["copyout"] = time.time()-t_out # write to disk t_disk = time.time() event_ssnet_containers = [ out.get_data( larcv.kProductImage2D, "ssnet_plane%d"%(p) ) for p in range(NPLANES) ] for p in range(NPLANES): for c in range(NCLASSES): event_ssnet_containers[p].Append( larcv.as_image2d_meta( output_np[p][0,c,:], img_v[p].meta() ) ) out.set_id( event_image_container.run(), event_image_container.subrun(), event_image_container.event() ) out.save_entry() timer["writeout"] += time.time()-t_disk timer["totentry"] += time.time()-t_entry print "End of entry loop" print "Finalize output" out.finalize() timer["total"] = time.time()-timer["total"] print "Timing for different steps" print "--------------------------" for k,i in timer.items(): print k,": ",i," secs (%.2f sec/event)"%(i/float(nprocess_events))
# get predictions from gpu flow_np = pred_labels.detach().cpu().numpy().astype(np.float32) for ib in range(batch_size): if ientry >= nevts: # skip last portion of last batch break evtinfo = data["_rse_"][ib, :] meta_v = inputdata.getmeta(treename) ev_out = outputdata.get_data(larcv.kProductImage2D, "uburn_plane%d" % (plane)) nclasses = flow_np.shape[1] for c in range(nclasses): img_slice = data[(larcv.kProductImage2D, treename)][:, plane, :, :] flow_lcv = larcv.as_image2d_meta(flow_np[ib, c, :, :], meta_v[plane]) ev_out.Append(flow_lcv) outputdata.set_id(evtinfo[0], evtinfo[1], evtinfo[2]) outputdata.save_entry() ientry += 1 tsave = time.time() - tsave timing["++save_output"] += tsave # end of batch tbatch = time.time() - tbatch if verbose: print "time for batch: ", tbatch, "secs" timing["+batch"] += tbatch # save results
meta = larcv.ImageMeta(srcmeta[b, 2, 0, 0], srcmeta[b, 2, 0, 1], srcmeta[b, 2, 0, 2], srcmeta[b, 2, 0, 3], IMAGE_HEIGHT, IMAGE_WIDTH, 2) srcmetas.append(meta) meta = larcv.ImageMeta(srcmeta[b, 0, 0, 0], srcmeta[b, 0, 0, 1], srcmeta[b, 0, 0, 2], srcmeta[b, 0, 0, 3], IMAGE_HEIGHT, IMAGE_WIDTH, 2) targetumetas.append(meta) meta = larcv.ImageMeta(srcmeta[b, 1, 0, 0], srcmeta[b, 1, 0, 1], srcmeta[b, 1, 0, 2], srcmeta[b, 1, 0, 3], IMAGE_HEIGHT, IMAGE_WIDTH, 2) targetvmetas.append(meta) # check image2d input img_adc = larcv.as_image2d_meta( source[0, 0, :, :].detach().cpu().numpy().transpose((1, 0)), srcmetas[0]) hadc = larcv.as_th2d(img_adc, "hadc_input") c.cd(1) hadc.SetTitle("source y") hadc.Draw("COLZ") c.Update() #c.SaveAs("hadc_input_%d.png"%(ientry)) #cv.imwrite( "cvadc_input_%d.png"%(ientry), source[0,0,:,:].detach().cpu().numpy() ) img_targetu = larcv.as_image2d_meta( target1[0, 0, :, :].detach().cpu().numpy().transpose((1, 0)), targetumetas[0]) htargetu = larcv.as_th2d(img_targetu, "htargetu_input") c.cd(2)
def make_reply(self,request,nreplies): """ we load each image and pass it through the net. we process all images before sending complete reply. the attribute self._still_processing_msg is used to tell us if we are still in the middle of a reply. """ self._log.debug("make_reply:: received message with {} parts".format(len(request))) if not self.is_model_loaded(): self._log.debug("model not loaded for some reason. loading.") try: import torch except: raise RuntimeError("could not load pytorch!") try: from ROOT import std except: raise RuntimeError("could not load ROOT.std") # message pattern: [image_bson,image_bson,...] nmsgs = len(request) nbatches = nmsgs/self.batch_size if not self._still_processing_msg: self._next_msg_id = 0 # turn message pieces into numpy arrays imgset_v = {} img2dset_v = {} rseid_v = {} sizes = [] # tuples with (rows,cols,nfeatures,npoints) frames_used = [] totalpts = 0 for imsg in xrange(self._next_msg_id,nmsgs): try: compressed_data = str(request[imsg]) data = zlib.decompress(compressed_data) c_run = c_int() c_subrun = c_int() c_event = c_int() c_id = c_int() imgdata = larcv.json.sparseimg_from_bson_pybytes(data, c_run, c_subrun, c_event, c_id ) except Exception as e: self._log.error("Image Data in message part {}".format(imsg) +" could not be converted: {}".format(e)) continue self._log.debug("Image[{}] converted: nfeatures={} npts={}"\ .format(imsg,imgdata.nfeatures(), imgdata.pixellist().size()/(imgdata.nfeatures()+2))) #self._log.debug("Image[{}] meta: {}".format(imsg,imgdata.meta_v().front().dump())) # convert back to image2d imgid = c_id.value if imgid not in imgset_v: imgset_v[imgid] = [] img2dset_v[imgid] = [] rseid_v[imgid]=(c_run.value,c_subrun.value,c_event.value,imgid) img2d_v = imgdata.as_Image2D() #print(img2d_v.front().meta().dump()) imgset_v[imgid].append( img2d_v.front() ) img2dset_v[imgid].append(img2d_v) # run the network and produce replies # we run in pairs of (src,target) crops. responsibility of Client # to get this correct keys = imgset_v.keys() keys.sort() nsets = len(keys) current_cols = 0 current_rows = 0 src_np = None tar_np = None ibatch = 0 iset = 0 flow_v = {} set_v = [] meta_v = {} while iset<nsets: setid = keys[iset] if len(imgset_v[setid])!=2: # set is not complete iset += 1 continue if imgset_v[setid][0].meta().plane()==2: src = imgset_v[setid][0] tar = imgset_v[setid][1] else: src = imgset_v[setid][1] tar = imgset_v[setid][0] imgset_v[setid] = [] imgset_v[setid].append(src) imgset_v[setid].append(tar) # if first set of images, create numpy array if src_np is None: imgnptype = np.float32 if self._use_half: imgnptype = np.float16 #print("src_np: {}".format((self.batch_size,1,src.meta().rows(),src.meta().cols()))) src_np = np.zeros( (self.batch_size,1,src.meta().rows(),src.meta().cols()), dtype=imgnptype ) tar_np = np.zeros( (self.batch_size,1,tar.meta().rows(),tar.meta().cols()), dtype=imgnptype ) set_v = [] meta_v = {} # check that same size as previous images samesize = True if src_np.shape[2]!=src.meta().rows() or src_np.shape[3]!=src.meta().cols(): samesize = False # if same size and we have not filled the batch yet, add to batch array if samesize and ibatch<self.batch_size: src_np[ibatch,0,:,:] = np.transpose( larcv.as_ndarray(src), (1,0) ) tar_np[ibatch,0,:,:] = np.transpose( larcv.as_ndarray(tar), (1,0) ) meta_v[setid] = src.meta() set_v.append(setid) iset += 1 ibatch += 1 if not samesize or ibatch==self.batch_size or iset==nsets: # convert to torch and run the batch through the network src_t = torch.from_numpy(src_np).to(self.device) tar_t = torch.from_numpy(tar_np).to(self.device) with torch.set_grad_enabled(False): flow_t, visi_t = self.model( src_t, tar_t ) # repress flow_t for values below threshold flow_t[ torch.lt(src_t,10.0) ] = 0.0 # convert back to image2d. only use those with setid flow_np = flow_t.detach().cpu().numpy().astype(np.float32) for ib,sid in enumerate(set_v): # convert back to image2d flow_v[sid] = larcv.as_image2d_meta( np.transpose(flow_np[ib,0,:,:], (1,0)), meta_v[sid] ) # reset batch variables set_v = [] ibatch = 0 src_np = None tar_np = None # turn image2d into sparseimage and ship back to client reply = [] isfinal = True nfilled = 0 for setid in keys: flow = flow_v[setid] flowpix = larcv.as_pixelarray_with_selection( flow, imgset_v[setid][0], 10.0 ) # make the sparseimage object outmeta_v = std.vector("larcv::ImageMeta")() outmeta_v.push_back( imgset_v[setid][0].meta() ) sparseimg = larcv.sparseimg_from_ndarray( flowpix, outmeta_v, larcv.msg.kNORMAL ) # convert to bson string rseid = rseid_v[setid] bson = larcv.json.as_bson_pybytes( sparseimg, rseid[0], rseid[1], rseid[2], rseid[3] ) # compress compressed = zlib.compress(bson) # add to reply message list reply.append(compressed) self._log.debug("formed reply with {} frames. isfinal={}" .format(len(reply),isfinal)) return reply,isfinal
def main(MODEL=MODEL): global best_prec1 # training parameters lr = 2.0e-5 momentum = 0.9 weight_decay = 1.0e-3 batchsize_valid = 2 # Create model -- instantiate on the GPU if MODEL == 1: if GPUMODE: model = ASPP_ResNet(inplanes=16, in_channels=1, num_classes=3, showsizes=False) else: model = ASPP_ResNet(inplanes=16, in_channels=1, num_classes=3) elif MODEL == 2: if GPUMODE: model = UResNet(inplanes=20, input_channels=1, num_classes=3, showsizes=False) else: model = UResNet(inplanes=20, input_channels=1, num_classes=3) optimizer = torch.optim.SGD(model.parameters(), lr, momentum=momentum, weight_decay=weight_decay) cudnn.benchmark = True # Load checkpoint and state dictionary # Map the checkpoint file to the CPU -- removes GPU mapping map_location = {"cuda:0": "cpu", "cuda:1": "cpu"} checkpoint = torch.load(CHECKPOINT_FILE, map_location=map_location) # Debugging block: # print "Checkpoint file mapped to CPU." # print "Press return to load best prediction tensor." # raw_input() best_prec1 = checkpoint["best_prec1"] print "state_dict size: ", len(checkpoint["state_dict"]) print " " for p, t in checkpoint["state_dict"].items(): print p, t.size() # Debugging block: # print " " # print "Best prediction tensor loaded." # print "Press return to load state dictionary." # raw_input() # Map checkpoint file to the desired GPU model.load_state_dict(checkpoint["state_dict"]) model = model.cuda(GPUID) print " " print "State dictionary mapped to GPU: ", GPUID if MODEL == 1: modelString = "ASPP_ResNet" elif MODEL == 2: modelString = "caffe_uresnet" print "Press return to deploy:", modelString print "From checkpoint:", CHECKPOINT_FILE raw_input() # switch to evaluate mode model.eval() # LOAD THE DATASET validcfg = """ThreadDatumFillerValid: { Verbosity: 2 EnableFilter: false RandomAccess: true UseThread: false #InputFiles: ["/mnt/raid0/taritree/ssnet_training_data/train02.root"] InputFiles: ["/media/hdd1/larbys/ssnet_dllee_trainingdata/val.root"] ProcessType: ["SegFiller"] ProcessName: ["SegFiller"] IOManager: { Verbosity: 2 IOMode: 0 ReadOnlyTypes: [0,0,0] ReadOnlyNames: ["wire","segment","ts_keyspweight"] } ProcessList: { SegFiller: { # DatumFillerBase configuration Verbosity: 2 ImageProducer: "wire" LabelProducer: "segment" WeightProducer: "ts_keyspweight" # SegFiller configuration Channels: [2] SegChannel: 2 EnableMirror: false EnableCrop: false ClassTypeList: [0,1,2] ClassTypeDef: [0,0,0,2,2,2,1,1,1,1] } } } """ with open("segfiller_valid.cfg", 'w') as fvalid: print >> fvalid, validcfg iovalid = LArCV1Dataset("ThreadDatumFillerValid", "segfiller_valid.cfg") iovalid.init() iovalid.getbatch(batchsize_valid) NENTRIES = iovalid.io.get_n_entries() NENTRIES = 10 #debug print "Number of entries in input: ", NENTRIES ientry = 0 nbatches = NENTRIES / batchsize_valid if NENTRIES % batchsize_valid != 0: nbatches += 1 for ibatch in range(nbatches): data = iovalid.getbatch(batchsize_valid) # convert to pytorch Variable (with automatic gradient calc.) if GPUMODE: images_var = torch.autograd.Variable(data.images.cuda(GPUID)) labels_var = torch.autograd.Variable(data.labels.cuda(GPUID), requires_grad=False) weight_var = torch.autograd.Variable(data.weight.cuda(GPUID), requires_grad=False) else: images_var = torch.autograd.Variable(data.images) labels_var = torch.autograd.Variable(data.labels, requires_grad=False) weight_var = torch.autograd.Variable(data.weight, requires_grad=False) # compute output output = model(images_var) ev_out_wire = outputdata.get_data(larcv.kProductImage2D, "wire") wire_t = images_var.data.cpu().numpy() weight_t = weight_var.data.cpu().numpy() # get predictions from gpu (turns validation routine into images) labels_np = output.data.cpu().numpy().astype(np.float32) labels_np = 10**labels_np for ib in range(batchsize_valid): if ientry >= NENTRIES: break inputmeta.read_entry(ientry) ev_meta = inputmeta.get_data(larcv.kProductImage2D, "wire") outmeta = ev_meta.Image2DArray()[2].meta() img_slice0 = labels_np[ib, 0, :, :] nofill_lcv = larcv.as_image2d_meta(img_slice0, outmeta) ev_out = outputdata.get_data(larcv.kProductImage2D, "class0") ev_out.Append(nofill_lcv) img_slice1 = labels_np[ib, 1, :, :] fill_lcv = larcv.as_image2d_meta(img_slice1, outmeta) ev_out = outputdata.get_data(larcv.kProductImage2D, "class1") ev_out.Append(fill_lcv) wire_slice = wire_t[ib, 0, :, :] wire_out = larcv.as_image2d_meta(wire_slice, outmeta) ev_out_wire.Append(wire_out) #weight_slice=weight_t[ib,0,:,:] #weights_out = larcv.as_image2d_meta(weight_slice,outmeta) #ev_out_weights.Append( weights_out ) outputdata.set_id(1, 1, ibatch * batchsize_valid + ib) outputdata.save_entry() ientry += 1 # save results outputdata.finalize()
def make_reply(self, request, nreplies): """we load each image and pass it through the net. we run one batch before sending off partial reply. the attribute self._still_processing_msg is used to tell us if we are still in the middle of a reply. """ #print("DummyPyWorker. Sending client message back") self._log.debug("received message with {} parts".format(len(request))) if not self.is_model_loaded(): self._log.debug("model not loaded for some reason. loading.") try: import torch except: raise RuntimeError("could not load pytorch!") # message pattern: [image_bson,image_bson,...] nmsgs = len(request) nbatches = nmsgs / self.batch_size if not self._still_processing_msg: self._next_msg_id = 0 # turn message pieces into numpy arrays img2d_v = [] sizes = [] frames_used = [] rseid_v = [] for imsg in xrange(self._next_msg_id, nmsgs): try: compressed_data = str(request[imsg]) data = zlib.decompress(compressed_data) c_run = c_int() c_subrun = c_int() c_event = c_int() c_id = c_int() img2d = larcv.json.image2d_from_pystring( data, c_run, c_subrun, c_event, c_id) except: self._log.error("Image Data in message part {}\ could not be converted".format(imsg)) continue self._log.debug("Image[{}] converted: {}"\ .format(imsg,img2d.meta().dump())) # check if correct plane! if img2d.meta().plane() != self.plane: self._log.debug("Image[{}] is the wrong plane!".format(imsg)) continue # check that same size as previous images imgsize = (int(img2d.meta().cols()), int(img2d.meta().rows())) if len(sizes) == 0: sizes.append(imgsize) elif len(sizes) > 0 and imgsize not in sizes: self._log.debug("Next image a different size. \ we do not continue batch.") self._next_msg_id = imsg break img2d_v.append(img2d) frames_used.append(imsg) rseid_v.append( (c_run.value, c_subrun.value, c_event.value, c_id.value)) if len(img2d_v) >= self.batch_size: self._next_msg_id = imsg + 1 break # convert the images into numpy arrays nimgs = len(img2d_v) self._log.debug( "converted msgs into batch of {} images. frames={}".format( nimgs, frames_used)) np_dtype = np.float32 img_batch_np = np.zeros((nimgs, 1, sizes[0][1], sizes[0][0]), dtype=np_dtype) for iimg, img2d in enumerate(img2d_v): meta = img2d.meta() img2d_np = larcv.as_ndarray( img2d )\ .reshape( (1,1,meta.cols(),meta.rows())) img2d_np = np.transpose(img2d_np, (0, 1, 3, 2)) img_batch_np[iimg, :] = img2d_np # print("shape of image: ",img2d_np.shape) # now make into torch tensor img2d_batch_t = torch.from_numpy(img_batch_np).to(self.device) # out_batch_np = img2d_batch_t.detach().cpu().numpy() # out_batch_np=np.transpose(out_batch_np,(0,1,3,2)) print("shape of image: ", img2d_batch_t.shape) with torch.set_grad_enabled(False): out_batch_np = self.model.forward( img2d_batch_t).detach().cpu().numpy() out_batch_np = np.transpose(out_batch_np, (0, 1, 3, 2)) # compression techniques ## 1) threshold values to zero ## 2) suppress output for non-adc values ## 3) use half # suppress small values out_batch_np[out_batch_np < 1.0e-3] = 0.0 # threshold # for ich in xrange(out_batch_np.shape[1]): # out_batch_np[:,ich,:,:][ img_batch_np[:,0,:,:]<10.0 ] = 0.0 # convert back to full precision, if we used half-precision in the net self._log.debug( "passed images through net. output batch shape={}".format( out_batch_np.shape)) # convert from numpy array batch back to image2d and messages reply = [] for iimg in xrange(out_batch_np.shape[0]): img2d = img2d_v[iimg] rseid = rseid_v[iimg] meta = img2d.meta() out_np = out_batch_np[iimg, 0, :, :] # print("out_np",type(out_np)) # print("meta",type(meta)) out_img2d = larcv.as_image2d_meta(out_np, meta) bson = larcv.json.as_pystring(out_img2d, rseid[0], rseid[1], rseid[2], rseid[3]) compressed = zlib.compress(bson) reply.append(compressed) if self._next_msg_id >= nmsgs: isfinal = True self._still_processing_msg = False else: isfinal = False self._still_processing_msg = True self._log.debug("formed reply with {} frames. isfinal={}".format( len(reply), isfinal)) return reply, isfinal
def generate_reply(self): """ we run the network """ reply = [] totmsgsize = 0.0 totcompsize = 0.0 for key, shape in self.shape_dict.items(): name = key[0] planeid = key[1] dummy = np.zeros(shape, dtype=np.float32) meta = self.meta_dict[key] img = self.image_dict[key] rse = self.rse_dict[key] # rse = "(0,0,0)" # for debug: purposely send the wrong rse back msg_batchsize = shape[0] # prepare numpy array for output # note, we through away the background scores to save egress data # we save results in half-precision. since numbers between [0,1] precision still good to 2^-11 at worse if not self.reply_in_float16: ssnetout = np.zeros( (shape[0], self.NCLASSES - 1, shape[2], shape[3]), dtype=np.float16) else: ssnetout = np.zeros( (shape[0], self.NCLASSES - 1, shape[2], shape[3]), dtype=np.float32) blobshape = (self.BATCHSIZE, 1, shape[2], shape[3]) # run the net for the plane self.nets[planeid].blobs['data'].reshape(*blobshape) # process the images for ibatch in range(0, msg_batchsize, self.BATCHSIZE): imgslice = img[ibatch * self.BATCHSIZE:(ibatch + 1) * self.BATCHSIZE, :] self.nets[planeid].blobs['data'].data[...] = imgslice self.nets[planeid].forward() if (ibatch + 1) * self.BATCHSIZE > msg_batchsize: remaining = msg_batchsize % self.BATCHSIZE start = ibatch * self.BATCHSIZE end = ibatch * self.BATCHSIZE + remaining if self.reply_in_float16: ssnetout[start:end, :] = self.nets[planeid].blobs[ 'softmax'].data[0:remaining, :].astype(np.float16) else: ssnetout[start:end, :] = self.nets[planeid].blobs[ 'softmax'].data[0:remaining, :] else: start = ibatch * self.BATCHSIZE end = (ibatch + 1) * self.BATCHSIZE if self.reply_in_float16: ssnetout[start:end, :] = self.nets[planeid].blobs[ 'softmax'].data[0:self.BATCHSIZE, 1:, :].astype(np.float16) else: ssnetout[start:end, :] = self.nets[planeid].blobs[ 'softmax'].data[0:self.BATCHSIZE, 1:, :] # we threshold score images so compression performs better outslice = ssnetout[start:end, :] for c in range(outslice.shape[1]): chslice = outslice[:, c, :].reshape( (1, 1, imgslice.shape[2], imgslice.shape[3])) chslice[imgslice < 10.0] = 0 # encode if self.decoder == "msgpack": x_enc = msgpack.packb(ssnetout, default=m.encode) x_comp = zlib.compress(x_enc, self.compression_level) # for debug: inspect compression gains (usually reduction to 1% or lower of original size) if self.print_msg_size: encframe = zmq.Frame(x_enc) comframe = zmq.Frame(x_comp) totmsgsize += len(encframe.bytes) totcompsize += len(comframe.bytes) reply.append(name.encode('utf-8')) reply.append(meta.dump().strip() + ":" + rse) reply.append(x_comp) # make the return message print "CaffeLArCV1Worker[{}] preparing (msgpack) reply for name=\"{}\" shape={} meta={} rse={}".format( self._identity, name, ssnetout.shape, meta.dump().strip(), rse) elif self.decoder == "tmessage": # we have to serialize both channels # make track image imgtrack = larcv.as_image2d_meta(ssnet[0, 0, :, :], meta) imgshower = larcv.as_image2d_meta(ssnet[0, 1, :, :], meta) tmsgtrack = rt.TMessage() tmsgtrack.WriteObject(imgtrack) msgtrack = tmsgtrack.Buffer() tmsgshower = rt.TMessage() tmsgshower.WriteObject(imgshower) msgshower = tmsgshower.Buffer() reply.append((name + "_track").encode('utf-8')) reply.append(meta.dump().strip()) reply.append(msgtrack) reply.append((name + "_shower").encode('utf-8')) reply.append(meta.dump().strip()) reply.append(msgshower) if self.print_msg_size: print "CaffeLArCV1Worker[{}] finished reply for name=\"{}\". size of array portion={} MB (uncompressed {} MB)".format( self._identity, name, totcompsize / 1.0e6, totmsgsize / 1.0e6) return reply