def main(unused_argv): request = inference_flags.request_from_flags() if not gfile.Exists(request.segmentation_output_dir): gfile.MakeDirs(request.segmentation_output_dir) bbox = bounding_box_pb2.BoundingBox() text_format.Parse(FLAGS.bounding_box, bbox) runner = inference.Runner() runner.start(request, with_membrane=FLAGS.with_membrane) print('>>>>>>>>>>>>>>>>> FAKE RUN') runner.run((bbox.start.z, bbox.start.y, bbox.start.x), (bbox.size.z, bbox.size.y, bbox.size.x), with_membrane=FLAGS.with_membrane, fake=True) print('>>>>>>>>>>>>>>>>> REAL RUN') runner.run((bbox.start.z, bbox.start.y, bbox.start.x), (bbox.size.z, bbox.size.y, bbox.size.x), with_membrane=FLAGS.with_membrane) counter_path = os.path.join(request.segmentation_output_dir, 'counters.txt') if not gfile.Exists(counter_path): runner.counters.dump(counter_path)
def main(unused_argv): print('log_level', FLAGS.verbose) if FLAGS.verbose: logger = logging.getLogger() logger.setLevel(logging.INFO) else: logger = logging.getLogger() logger.setLevel(logging.WARNING) start_time = time.time() request = inference_flags.request_from_flags() if mpi_rank == 0: if not gfile.exists(request.segmentation_output_dir): gfile.makedirs(request.segmentation_output_dir) if FLAGS.output_dir is None: root_output_dir = request.segmentation_output_dir else: root_output_dir = FLAGS.output_dir bbox = bounding_box_pb2.BoundingBox() text_format.Parse(FLAGS.bounding_box, bbox) subvolume_size = np.array([int(i) for i in FLAGS.subvolume_size]) overlap = np.array([int(i) for i in FLAGS.overlap]) sub_bboxes = divide_bounding_box(bbox, subvolume_size, overlap) if FLAGS.resume: sub_bboxes = find_unfinished(sub_bboxes, root_output_dir) sub_bboxes = np.array_split(np.array(sub_bboxes), mpi_size) else: sub_bboxes = None root_output_dir = None sub_bboxes = mpi_comm.scatter(sub_bboxes, 0) root_output_dir = mpi_comm.bcast(root_output_dir, 0) for sub_bbox in sub_bboxes: out_name = 'seg-%d_%d_%d_%d_%d_%d' % ( sub_bbox.start[0], sub_bbox.start[1], sub_bbox.start[2], sub_bbox.size[0], sub_bbox.size[1], sub_bbox.size[2]) segmentation_output_dir = os.path.join(root_output_dir, out_name) request.segmentation_output_dir = segmentation_output_dir if FLAGS.num_gpu > 0: use_gpu = str(mpi_rank % FLAGS.num_gpu) else: use_gpu = '' runner = inference.Runner(use_cpu=FLAGS.use_cpu, use_gpu=use_gpu) cube_start_time = (time.time() - start_time) / 60 runner.start(request) runner.run(sub_bbox.start[::-1], sub_bbox.size[::-1]) cube_finish_time = (time.time() - start_time) / 60 print('%s finished in %s min' % (out_name, cube_finish_time - cube_start_time)) runner.stop_executor() counter_path = os.path.join(request.segmentation_output_dir, 'counters_%d.txt' % mpi_rank) if not gfile.exists(counter_path): runner.counters.dump(counter_path)
def main(unused_argv): request = inference_flags.request_from_flags() if not gfile.Exists(request.segmentation_output_dir): gfile.MakeDirs(request.segmentation_output_dir) bbox = bounding_box_pb2.BoundingBox() text_format.Parse(FLAGS.bounding_box, bbox) # start_pos = tuple([int(i) for i in FLAGS.start_pos]) runner = inference.Runner() corner = (bbox.start.z, bbox.start.y, bbox.start.x) subvol_size = (bbox.size.z, bbox.size.y, bbox.size.x) start_pos = tuple([int(i) for i in FLAGS.start_pos]) seg_path = storage.segmentation_path(request.segmentation_output_dir, corner) prob_path = storage.object_prob_path(request.segmentation_output_dir, corner) runner.start(request) canvas, alignment = runner.make_canvas(corner, subvol_size) num_iter = canvas.segment_at(start_pos) print('>>', num_iter) sel = [ slice(max(s, 0), e + 1) for s, e in zip(canvas._min_pos - canvas._pred_size // 2, canvas._max_pos + canvas._pred_size // 2) ] mask = canvas.seed[sel] >= canvas.options.segment_threshold raw_segmented_voxels = np.sum(mask) mask &= canvas.segmentation[sel] <= 0 actual_segmented_voxels = np.sum(mask) canvas._max_id += 1 canvas.segmentation[sel][mask] = canvas._max_id canvas.seg_prob[sel][mask] = storage.quantize_probability( expit(canvas.seed[sel][mask])) runner.save_segmentation(canvas, alignment, seg_path, prob_path) runner.run((bbox.start.z, bbox.start.y, bbox.start.x), (bbox.size.z, bbox.size.y, bbox.size.x)) counter_path = os.path.join(request.segmentation_output_dir, 'counters.txt') if not gfile.Exists(counter_path): runner.counters.dump(counter_path)
def main(unused_argv): logger = logging.getLogger() if FLAGS.verbose: logger.setLevel(logging.DEBUG) else: logger.setLevel(logging.WARNING) start_time = time.time() # mpi version request = inference_flags.request_from_flags() if mpi_rank == 0: if not gfile.Exists(request.segmentation_output_dir): gfile.MakeDirs(request.segmentation_output_dir) bbox = bounding_box_pb2.BoundingBox() text_format.Parse(FLAGS.bounding_box, bbox) subvolume_size = np.array([int(i) for i in FLAGS.subvolume_size]) overlap = np.array([int(i) for i in FLAGS.overlap]) sub_bboxes = divide_bounding_box(bbox, subvolume_size, overlap) sub_bboxes = np.array_split(np.array(sub_bboxes), mpi_size) root_output_dir = request.segmentation_output_dir else: sub_bboxes = None root_output_dir = None sub_bboxes = mpi_comm.scatter(sub_bboxes, 0) root_output_dir = mpi_comm.bcast(root_output_dir, 0) print('rank %d, bbox: %s' % (mpi_rank, len(sub_bboxes))) print(sub_bboxes) for sub_bbox in sub_bboxes: out_name = 'seg-%d_%d_%d_%d_%d_%d' % ( sub_bbox.start[0], sub_bbox.start[1], sub_bbox.start[2], sub_bbox.size[0], sub_bbox.size[1], sub_bbox.size[2]) segmentation_output_dir = os.path.join(root_output_dir, out_name) request.segmentation_output_dir = segmentation_output_dir if FLAGS.num_gpu > 0: use_gpu = str(mpi_rank % FLAGS.num_gpu) else: use_gpu = '' runner = inference.Runner(use_cpu=FLAGS.use_cpu, use_gpu=use_gpu) runner.start(request) runner.run(sub_bbox.start[::-1], sub_bbox.size[::-1]) runner.stop_executor() mpi_comm.barrier() sys.exit()
def main(unused_argv): request = inference_flags.request_from_flags() if not gfile.exists(request.segmentation_output_dir): gfile.makedirs(request.segmentation_output_dir) bbox = bounding_box_pb2.BoundingBox() text_format.Parse(FLAGS.bounding_box, bbox) runner = inference.Runner() runner.start(request) runner.run((bbox.start.z, bbox.start.y, bbox.start.x), (bbox.size.z, bbox.size.y, bbox.size.x)) counter_path = os.path.join(request.segmentation_output_dir, 'counters.txt') if not gfile.exists(counter_path): runner.counters.dump(counter_path)
def main(unused_argv): request = inference_flags.request_from_flags() if not gfile.Exists(request.segmentation_output_dir): gfile.MakeDirs(request.segmentation_output_dir) bbox = bounding_box_pb2.BoundingBox() text_format.Parse(FLAGS.bounding_box, bbox) # Training import os batch_size = 16 max_steps = 3000 #10*250/batch_size #250 hdf_dir = os.path.split(request.image.hdf5)[0] load_ckpt_path = request.model_checkpoint_path save_ckpt_path = os.path.split( load_ckpt_path)[0] + '_topup_' + os.path.split( os.path.split(hdf_dir)[0])[1] # import ipdb;ipdb.set_trace() with tf.Graph().as_default(): with tf.device( tf.train.replica_device_setter(FLAGS.ps_tasks, merge_devices=True)): # SET UP TRAIN MODEL print('>>>>>>>>>>>>>>>>>>>>>>SET UP TRAIN MODEL') TA = train_functional.TrainArgs( train_coords=os.path.join(hdf_dir, 'tf_record_file'), data_volumes='jk:' + os.path.join(hdf_dir, 'grayscale_maps.h5') + ':raw', label_volumes='jk:' + os.path.join(hdf_dir, 'groundtruth.h5') + ':stack', train_dir=save_ckpt_path, model_name=request.model_name, model_args=request.model_args, image_mean=request.image_mean, image_stddev=request.image_stddev, max_steps=max_steps, optimizer='adam', load_from_ckpt=load_ckpt_path, batch_size=batch_size) global TA model_class = import_symbol(TA.model_name) seed = int(time.time() + TA.task * 3600 * 24) logging.info('Random seed: %r', seed) random.seed(seed) eval_tracker, model, secs, load_data_ops, summary_writer, merge_summaries_op = \ build_train_graph(model_class, TA, save_ckpt=False, with_membrane=TA.with_membrane, **json.loads(TA.model_args)) # SET UP INFERENCE MODEL print('>>>>>>>>>>>>>>>>>>>>>>SET UP INFERENCE MODEL') print('>>>>>>>>>>>>>>>>>>>>>>COUNTED %s VARIABLES PRE-INFERENCE' % len(tf.trainable_variables())) runner = inference.Runner() runner.start(request, batch_size=1, topup={'train_dir': FLAGS.train_dir}, reuse=tf.AUTO_REUSE, tag='_inference') #TAKES SESSION print('>>>>>>>>>>>>>>>>>>>>>>COUNTED %s VARIABLES POST-INFERENCE' % len(tf.trainable_variables())) # START TRAINING print('>>>>>>>>>>>>>>>>>>>>>>START TOPUP TRAINING') sess = train_functional.train_ffn(TA, eval_tracker, model, runner.session, load_data_ops, summary_writer, merge_summaries_op) # saver.save(sess, "/tmp/model.ckpt") # START INFERENCE print('>>>>>>>>>>>>>>>>>>>>>>START INFERENCE') # saver.restore(sess, "/tmp/model.ckpt") runner.run((bbox.start.z, bbox.start.y, bbox.start.x), (bbox.size.z, bbox.size.y, bbox.size.x)) counter_path = os.path.join(request.segmentation_output_dir, 'counters.txt') if not gfile.Exists(counter_path): runner.counters.dump(counter_path) sess.close()