def main(argv): if len(argv) > 1: raise app.UsageError('Too many command-line arguments.') root_out = FLAGS.occnet_dir + '/extracted' if not file_util.exists(root_out): file_util.mkdir(root_out) if FLAGS.write_metrics: # TODO(ldif-user): Set up your own pipeline runner # TODO(ldif-user) Replace lambda x: None with a proto reader. with beam.Pipeline() as p: protos = p | 'ReadResults' >> (lambda x: None) with_metrics = protos | 'ExtractMetrics' >> beam.FlatMap( make_metrics) result_pcoll = with_metrics | 'MakeMetricList' >> ( beam.combiners.ToList()) result_str = result_pcoll | 'MakeMetricStr' >> beam.Map( save_metrics) out_path = FLAGS.occnet_dir + '/extracted/metrics_ub-v2.csv' _ = result_str | 'WriteMetrics' >> beam.io.WriteToText( out_path, num_shards=1, shard_name_template='') if FLAGS.write_metric_summaries: log.info('Aggregating results locally.') result_path = FLAGS.occnet_dir + '/extracted/metrics_ub-v2.csv' final_results = metrics.aggregate_extracted(result_path) summary_out_path = result_path.replace('/metrics_ub-v2.csv', '/metric_summary_ub-v2.csv') file_util.writetxt(summary_out_path, final_results.to_csv())
def main(argv): if len(argv) > 1: raise app.UsageError('Too many command-line arguments.') xids = inference_util.parse_xid_str(FLAGS.xids) root_out = FLAGS.input_dir + '/extracted' if not file_util.exists(root_out): file_util.mkdir(root_out) if FLAGS.write_metrics or FLAGS.write_results: # TODO(ldif-user): Set up your own pipeline runner with beam.Pipeline() as p: for xid in xids: name = 'XID%i' % xid path = get_result_path(xid) # TODO(ldif-user) Replace lambda x: None with a proto reader. protos = p | 'ReadResults%s' % name >> (lambda x: None) if FLAGS.write_results: map_fun = functools.partial(write_results, xid=xid) _ = protos | 'ExtractResults%s' % name >> beam.FlatMap( map_fun) if FLAGS.write_metrics: with_metrics = protos | 'ExtractMetrics%s' % name >> beam.Map( make_metrics) result_pcoll = with_metrics | 'MakeMetricList%s' % name >> ( beam.combiners.ToList()) result_str = result_pcoll | 'MakeMetricStr%s' % name >> beam.Map( save_metrics) out_path = FLAGS.input_dir + '/extracted/%s_metrics-v2.csv' % name _ = result_str | 'WriteMetrics%s' % name >> beam.io.WriteToText( out_path, num_shards=1, shard_name_template='') if FLAGS.write_metric_summaries: log.info('Aggregating results locally.') for xid in tqdm.tqdm(xids): result_path = FLAGS.input_dir + '/extracted/XID%i_metrics-v2.csv' % xid final_results = metrics.aggregate_extracted(result_path) summary_out_path = result_path.replace('_metrics-v2.csv', '_metric_summary-v2.csv') file_util.writetxt(summary_out_path, final_results.csv())
def main(argv): if len(argv) > 1: raise app.UsageError('Too many command-line arguments.') log.set_level(FLAGS.log_level) tf.disable_v2_behavior() gpu_util.get_free_gpu_memory(0) if FLAGS.use_gpu_for_tensorflow and FLAGS.use_inference_kernel: log.info('Limiting TensorFlow memory by 1GB so the inference kernel' ' has enough left over to run.') if not FLAGS.dataset_directory: raise ValueError('A dataset directory must be provided.') if not FLAGS.result_directory: if FLAGS.save_results or FLAGS.save_meshes or FLAGS.save_ldifs: raise ValueError( 'A result directory must be provided to save results.') else: if not os.path.isdir(FLAGS.result_directory): os.makedirs(FLAGS.result_directory) if not FLAGS.use_gpu_for_tensorflow: os.environ['CUDA_VISIBLE_DEVICES'] = '-1' log.info('Loading model...') # Try to detect the most common error early for a good warning message: if not os.path.isdir(get_model_root()): raise ValueError( f"Couldn't find a trained model at {get_model_root()}") encoder, decoder = load_newest_model() log.info('Evaluating metrics...') splits = [x for x in FLAGS.split.split(',') if x] log.info(f'Will evaluate on splits: {splits}') for split in splits: log.info(f'Starting evaluation for split {split}.') dataset_items = get_evaluation_directories(split) log.info(f'The split has {len(dataset_items)} elements.') results = [] to_eval = filter_by_class(dataset_items) to_eval = filter_by_eval_frac(to_eval) for path in tqdm.tqdm(to_eval): e = examples.InferenceExample.from_directory(path) embedding = encoder.run_example(e) iou = decoder.iou(embedding, e) gt_mesh = e.gt_mesh mesh = decoder.extract_mesh(embedding, resolution=FLAGS.resolution) if FLAGS.visualize: # Visualize in the normalized_coordinate frame, so the camera is # always reasonable. Metrics are computed in the original frame. gaps_util.mshview([e.normalized_gt_mesh, mesh]) # TODO(kgenova) gaps2occnet is poorly named, it is really normalized -> # unnormalized (where 'gaps' is the normalized training frame and 'occnet' # is whatever the original frame of the input mesh was) post_extract_start = time.time() mesh.apply_transform(e.gaps2occnet) if FLAGS.save_meshes: path = (f'{FLAGS.result_directory}/meshes/{split}/{e.cat}/' f'{e.mesh_hash}.ply') if not os.path.isdir(os.path.dirname(path)): os.makedirs(os.path.dirname(path)) mesh.export(path) if FLAGS.save_ldifs: path = (f'{FLAGS.result_directory}/ldifs/{split}/{e.cat}/' f'{e.mesh_hash}.txt') if not os.path.isdir(os.path.dirname(path)): os.makedirs(os.path.dirname(path)) decoder.savetxt(embedding, path) nc, fst, fs2t, chamfer = metrics.all_mesh_metrics(mesh, gt_mesh) log.verbose(f'Mesh: {e.mesh_name}') log.verbose(f'IoU: {iou}.') log.verbose(f'F-Score (tau): {fst}') log.verbose(f'Chamfer: {chamfer}') log.verbose(f'F-Score (2*tau): {fs2t}') log.verbose(f'Normal Consistency: {nc}') results.append({ 'key': e.mesh_name, 'Normal Consistency': nc, 'F-Score (tau)': fst, 'F-Score (2*tau)': fs2t, 'Chamfer': chamfer, 'IoU': iou }) post_extract_end = time.time() log.verbose( f'Time post extract: {post_extract_end - post_extract_start}') results = pd.DataFrame(results) if FLAGS.save_results: complete_csv = results.to_csv() result_path = f'{FLAGS.result_directory}/full_results_{split}.csv' file_util.writetxt(result_path, complete_csv) final_results = metrics.aggregate_extracted(results) if FLAGS.save_results: summary_out_path = f'{FLAGS.result_directory}/result_summary_{split}.csv' file_util.writetxt(summary_out_path, final_results.to_csv())