def opena( path: str, mode: str = "r", buffering: int = -1, encoding: Optional[str] = None, errors: Optional[str] = None, newline: Optional[str] = None, ): """ Return file descriptor with asynchronous write operations. """ global IOPathPathManager if not IOPathPathManager: logging.info("ioPath is initializing PathManager.") try: from iopath.common.file_io import PathManager IOPathPathManager = PathManager() except Exception: logging.exception( "Failed to initialize ioPath PathManager object.") return IOPathPathManager.opena( path=path, mode=mode, buffering=buffering, encoding=encoding, errors=errors, newline=newline, )
def __init__( self, model: nn.Module, save_dir: str = "", *, save_to_disk: bool = True, **checkpointables: object, ) -> None: """ Args: model (nn.Module): model. save_dir (str): a directory to save and find checkpoints. save_to_disk (bool): if True, save checkpoint to disk, otherwise disable saving for this checkpointer. checkpointables (object): any checkpointable objects, i.e., objects that have the `state_dict()` and `load_state_dict()` method. For example, it can be used like `Checkpointer(model, "dir", optimizer=optimizer)`. """ if isinstance(model, (DistributedDataParallel, DataParallel)): model = model.module self.model = model self.checkpointables = copy.copy(checkpointables) # pyre-ignore self.logger = logging.getLogger(__name__) # pyre-ignore self.save_dir = save_dir self.save_to_disk = save_to_disk # Default PathManager, support HTTP URLs (for backward compatibility in open source). # A user may want to use a different project-specific PathManager self.path_manager: PathManager = PathManager() self.path_manager.register_handler(HTTPURLHandler())
def save_obj( f, verts, faces, decimal_places: Optional[int] = None, path_manager: Optional[PathManager] = None, ): """ Save a mesh to an .obj file. Args: f: File (or path) to which the mesh should be written. verts: FloatTensor of shape (V, 3) giving vertex coordinates. faces: LongTensor of shape (F, 3) giving faces. decimal_places: Number of decimal places for saving. path_manager: Optional PathManager for interpreting f if it is a str. """ if len(verts) and not (verts.dim() == 2 and verts.size(1) == 3): message = "Argument 'verts' should either be empty or of shape (num_verts, 3)." raise ValueError(message) if len(faces) and not (faces.dim() == 2 and faces.size(1) == 3): message = "Argument 'faces' should either be empty or of shape (num_faces, 3)." raise ValueError(message) if path_manager is None: path_manager = PathManager() with _open_file(f, path_manager, "w") as f: return _save(f, verts, faces, decimal_places)
def _load_texture_images( material_names: List[str], data_dir: str, material_properties: MaterialProperties, texture_files: TextureFiles, path_manager: PathManager, ) -> Tuple[MaterialProperties, TextureImages]: final_material_properties = {} texture_images = {} # Only keep the materials referenced in the obj. for material_name in material_names: if material_name in texture_files: # Load the texture image. path = os.path.join(data_dir, texture_files[material_name]) if path_manager.exists(path): image = (_read_image( path, path_manager=path_manager, format="RGB") / 255.0) image = torch.from_numpy(image) texture_images[material_name] = image else: msg = f"Texture file does not exist: {path}" warnings.warn(msg) if material_name in material_properties: final_material_properties[material_name] = material_properties[ material_name] return final_material_properties, texture_images
def _read_image(file_name: str, path_manager: PathManager, format=None): """ Read an image from a file using Pillow. Args: file_name: image file path. path_manager: PathManager for interpreting file_name. format: one of ["RGB", "BGR"] Returns: image: an image of shape (H, W, C). """ if format not in ["RGB", "BGR"]: raise ValueError("format can only be one of [RGB, BGR]; got %s", format) with path_manager.open(file_name, "rb") as f: # pyre-fixme[6]: Expected `Union[str, typing.BinaryIO]` for 1st param but # got `Union[typing.IO[bytes], typing.IO[str]]`. image = Image.open(f) if format is not None: # PIL only supports RGB. First convert to RGB and flip channels # below for BGR. image = image.convert("RGB") image = np.asarray(image).astype(np.float32) if format == "BGR": image = image[:, :, ::-1] return image
def save_ply( f, verts: torch.Tensor, faces: Optional[torch.LongTensor] = None, verts_normals: Optional[torch.Tensor] = None, ascii: bool = False, decimal_places: Optional[int] = None, path_manager: Optional[PathManager] = None, ) -> None: """ Save a mesh to a .ply file. Args: f: File (or path) to which the mesh should be written. verts: FloatTensor of shape (V, 3) giving vertex coordinates. faces: LongTensor of shape (F, 3) giving faces. verts_normals: FloatTensor of shape (V, 3) giving vertex normals. ascii: (bool) whether to use the ascii ply format. decimal_places: Number of decimal places for saving if ascii=True. path_manager: PathManager for interpreting f if it is a str. """ if len(verts) and not (verts.dim() == 2 and verts.size(1) == 3): message = "Argument 'verts' should either be empty or of shape (num_verts, 3)." raise ValueError(message) if ( faces is not None and len(faces) and not (faces.dim() == 2 and faces.size(1) == 3) ): message = "Argument 'faces' should either be empty or of shape (num_faces, 3)." raise ValueError(message) if ( verts_normals is not None and len(verts_normals) and not ( verts_normals.dim() == 2 and verts_normals.size(1) == 3 and verts_normals.size(0) == verts.size(0) ) ): message = "Argument 'verts_normals' should either be empty or of shape (num_verts, 3)." raise ValueError(message) if path_manager is None: path_manager = PathManager() with _open_file(f, path_manager, "wb") as f: _save_ply( f, verts=verts, faces=faces, verts_normals=verts_normals, verts_colors=None, ascii=ascii, decimal_places=decimal_places, )
def __init__(self, vocab_path: Optional[str] = None, trainable=False, speed: int = 0): super(WhitespaceTokenizer, self).__init__() self.trainable = trainable self.speed = speed # mock a real tokenizer: slowing down tokenization speed self.unknown = "unknown" self.vocab: Dict[str, int] = {self.unknown: 0} # load vocab path_manager = PathManager() if vocab_path: with path_manager.open(vocab_path, "r") as f: for line in f.readlines(): token = line.split()[0] self.vocab[token] = len(self.vocab)
def load_ply( f, *, path_manager: Optional[PathManager] = None ) -> Tuple[torch.Tensor, torch.Tensor]: """ Load the verts and faces from a .ply file. Note that the preferred way to load data from such a file is to use the IO.load_mesh and IO.load_pointcloud functions, which can read more of the data. Example .ply file format: ply format ascii 1.0 { ascii/binary, format version number } comment made by Greg Turk { comments keyword specified, like all lines } comment this file is a cube element vertex 8 { define "vertex" element, 8 of them in file } property float x { vertex contains float "x" coordinate } property float y { y coordinate is also a vertex property } property float z { z coordinate, too } element face 6 { there are 6 "face" elements in the file } property list uchar int vertex_index { "vertex_indices" is a list of ints } end_header { delimits the end of the header } 0 0 0 { start of vertex list } 0 0 1 0 1 1 0 1 0 1 0 0 1 0 1 1 1 1 1 1 0 4 0 1 2 3 { start of face list } 4 7 6 5 4 4 0 4 5 1 4 1 5 6 2 4 2 6 7 3 4 3 7 4 0 Args: f: A binary or text file-like object (with methods read, readline, tell and seek), a pathlib path or a string containing a file name. If the ply file is in the binary ply format rather than the text ply format, then a text stream is not supported. It is easiest to use a binary stream in all cases. path_manager: PathManager for loading if f is a str. Returns: verts: FloatTensor of shape (V, 3). faces: LongTensor of vertex indices, shape (F, 3). """ if path_manager is None: path_manager = PathManager() verts, faces, _, _ = _load_ply(f, path_manager=path_manager) if faces is None: faces = torch.zeros(0, 3, dtype=torch.int64) return verts, faces
def _open_file(f, path_manager: PathManager, mode="r") -> ContextManager[IO]: if isinstance(f, str): f = path_manager.open(f, mode) return contextlib.closing(f) elif isinstance(f, pathlib.Path): f = f.open(mode) return contextlib.closing(f) else: return nullcontext(f)
def get_skateboard_data( avoid_manifold: bool = False, silence_logs: bool = False ) -> Generator[Tuple[str, PathManager], None, None]: """ Context manager for accessing Co3D dataset by tests, at least for the first 5 skateboards. Internally, we want this to exercise the normal way to access the data directly manifold, but on an RE worker this is impossible so we use a workaround. Args: avoid_manifold: Use the method used by RE workers even locally. silence_logs: Whether to reduce log output from iopath library. Yields: dataset_root: (str) path to dataset root. path_manager: path_manager to access it with. """ path_manager = PathManager() if silence_logs: logging.getLogger("iopath.fb.manifold").setLevel(logging.CRITICAL) logging.getLogger("iopath.common.file_io").setLevel(logging.CRITICAL) if not os.environ.get("FB_TEST", False): if os.getenv("FAIR_ENV_CLUSTER", "") == "": raise unittest.SkipTest("Unknown environment. Data not available.") yield "/checkpoint/dnovotny/datasets/co3d/download_aws_22_02_18", path_manager elif avoid_manifold or os.environ.get("INSIDE_RE_WORKER", False): from libfb.py.parutil import get_file_path par_path = "skateboard_first_5" source = get_file_path(par_path) assert Path(source).is_file() with tempfile.TemporaryDirectory() as dest: with ZipFile(source) as f: f.extractall(dest) yield os.path.join(dest, "extracted"), path_manager else: from iopath.fb.manifold import ManifoldPathHandler path_manager.register_handler(ManifoldPathHandler()) yield "manifold://co3d/tree/extracted", path_manager
def evaluate(self): if self._distributed: comm.synchronize() predictions = comm.gather(self._predictions, dst=0) predictions = list(itertools.chain(*predictions)) if not comm.is_main_process(): return {} else: predictions = self._predictions gt_corrs = self._gt_corrs if len(predictions) == 0: self._logger.warning( "[COCOEvaluator] Did not receive valid predictions.") return {} if self._output_dir: pm = PathManager() pm.mkdirs(self._output_dir) file_path = os.path.join(self._output_dir, "instances_predictions.pth") with pm.open(file_path, "wb") as f: torch.save(predictions, f) self._results = OrderedDict() # if not self._visualize: single_predictions = self._siamese_to_single(predictions) if "proposals" in single_predictions[0]: self._eval_box_proposals(single_predictions) if "instances" in single_predictions[0]: # self._eval_predictions(set(self._tasks), single_predictions) self._eval_plane(single_predictions) if "depth_l1_dist" in single_predictions[0]: self._eval_depth(single_predictions) if "embedding" in self._plane_tasks: self._eval_affinity(predictions) if "camera" in self._plane_tasks: summary = self._eval_camera(predictions) file_path = os.path.join(self._output_dir, "summary.pkl") with open(file_path, "wb") as f: pickle.dump(summary, f) # Copy so the caller can do whatever with results return copy.deepcopy(self._results)
class TestDriver: _pathmgr = PathManager() def test(self) -> None: model = Model() optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9) print("Model's state_dict:") for param_tensor in model.state_dict(): print(f"{param_tensor}\t{model.state_dict()[param_tensor].size()}") print("Optimizer's state_dict:") for var_name in optimizer.state_dict(): print(f"{var_name}\t{optimizer.state_dict()[var_name]}") with tempfile.TemporaryDirectory() as _tmpdir: try: URI = os.path.join(_tmpdir, "test.ckpt") f = self._pathmgr.opena(URI, "wb") i = "*" large = f"{i}" * 1000000000 print("Starting `torch.save` call.") torch.save( { "model_state_dict": model.state_dict(), "optimizer_state_dict": optimizer.state_dict(), "large": large, }, f, ) f.close() start_time = time.time() finally: # We want this `join` call to take time. If it is instantaneous, # then our async write calls are not running asynchronously. print( "Waiting for `torch.save` call to complete at `async_join()`." ) self._pathmgr.async_join() print("Time Python waited for `async_join()` call to finish: " f"{time.time() - start_time}s.") assert self._pathmgr.async_close() checkpoint = torch.load(URI) for key_item_1, key_item_2 in zip( model.state_dict().items(), checkpoint["model_state_dict"].items()): assert torch.equal(key_item_1[1], key_item_2[1]) assert optimizer.state_dict() == checkpoint["optimizer_state_dict"] assert large == checkpoint["large"] print("Async `torch.save` Test succeeded.")
def _get_local_path(self, path, **kwargs): model_name = path[len(self.PREFIX):] dataset_name, *model_name, data_type = model_name.split('/') if data_type == 'weight': model_url = MODEL_CATALOG[dataset_name]['/'.join(model_name)] elif data_type == 'config': model_url = CONFIG_CATALOG[dataset_name]['/'.join(model_name)] else: raise ValueError(f"Unknown data_type {data_type}") return PathManager.get_local_path(model_url, **kwargs)
def create_path_manager() -> PathManager: # TODO: move this inline import out after AIRStore OSS public released from airstore.client.airstore_tabular import AIRStorePathHandler pathmanager = PathManager() pathmanager.register_handler(AIRStorePathHandler()) pathmanager.set_strict_kwargs_checking(False) return pathmanager
def __init__( self, include_default_formats: bool = True, path_manager: Optional[PathManager] = None, ) -> None: if path_manager is None: self.path_manager = PathManager() else: self.path_manager = path_manager self.mesh_interpreters: Deque[MeshFormatInterpreter] = deque() self.pointcloud_interpreters: Deque[PointcloudFormatInterpreter] = deque() if include_default_formats: self.register_default_formats()
def test_open_new_path_manager(self) -> None: with self._patch_download(): path_manager = PathManager() with self.assertRaises(OSError): # no handler registered f = path_manager.open(self._remote_uri, "rb") path_manager.register_handler(HTTPURLHandler()) with path_manager.open(self._remote_uri, "rb") as f: self.assertTrue(os.path.isfile(f.name)) self.assertTrue(f.read() != "")
def test_load_mtl_with_spaces_in_resource_filename(self): """ Check that the texture image for materials in mtl files is loaded correctly even if there is a space in the file name e.g. material 1.png """ mtl_file = "\n".join([ "newmtl material_1", "map_Kd material 1.png", "Ka 1.000 1.000 1.000", # white "Kd 1.000 1.000 1.000", # white "Ks 0.000 0.000 0.000", # black "Ns 10.0", ]) with NamedTemporaryFile(mode="w", suffix=".mtl") as f: f.write(mtl_file) f.flush() material_properties, texture_files = _parse_mtl( Path(f.name), path_manager=PathManager(), device="cpu") dtype = torch.float32 expected_materials = { "material_1": { "ambient_color": torch.tensor([1.0, 1.0, 1.0], dtype=dtype), "diffuse_color": torch.tensor([1.0, 1.0, 1.0], dtype=dtype), "specular_color": torch.tensor([0.0, 0.0, 0.0], dtype=dtype), "shininess": torch.tensor([10.0], dtype=dtype), } } # Check that there is a material with name material_1 self.assertTrue(tuple(texture_files.keys()) == ("material_1", )) # Check that there is an image with name material 1.png self.assertTrue(texture_files["material_1"] == "material 1.png") # Check all keys and values in dictionary are the same. for n1, n2 in zip(material_properties.keys(), expected_materials.keys()): self.assertTrue(n1 == n2) for k1, k2 in zip(material_properties[n1].keys(), expected_materials[n2].keys()): self.assertTrue( torch.allclose(material_properties[n1][k1], expected_materials[n2][k2]))
def _load_materials( material_names: List[str], f: Optional[str], *, data_dir: str, load_textures: bool, device: Device, path_manager: PathManager, ): """ Load materials and optionally textures from the specified path. Args: material_names: a list of the material names found in the .obj file. f: path to the material information. data_dir: the directory where the material texture files are located. load_textures: whether textures should be loaded. device: Device (as str or torch.device) on which to return the new tensors. path_manager: PathManager object to interpret paths. Returns: material_colors: dict of properties for each material. texture_images: dict of material names and texture images. """ if not load_textures: return None, None if not material_names or f is None: if material_names: warnings.warn("No mtl file provided") return None, None if not path_manager.exists(f): warnings.warn(f"Mtl file does not exist: {f}") return None, None # Texture mode uv wrap return load_mtl( f, material_names=material_names, data_dir=data_dir, path_manager=path_manager, device=device, )
class TestLazyPath(unittest.TestCase): _pathmgr = PathManager() def test_materialize(self) -> None: f = MagicMock(return_value="test") x = LazyPath(f) f.assert_not_called() p = os.fspath(x) f.assert_called() self.assertEqual(p, "test") p = os.fspath(x) # should only be called once f.assert_called_once() self.assertEqual(p, "test") def test_join(self) -> None: f = MagicMock(return_value="test") x = LazyPath(f) p = os.path.join(x, "a.txt") f.assert_called_once() self.assertEqual(p, "test/a.txt") def test_getattr(self) -> None: x = LazyPath(lambda: "abc") with self.assertRaises(AttributeError): x.startswith("ab") _ = os.fspath(x) self.assertTrue(x.startswith("ab")) def test_PathManager(self) -> None: x = LazyPath(lambda: "./") output = self._pathmgr.ls(x) # pyre-ignore output_gt = self._pathmgr.ls("./") self.assertEqual(sorted(output), sorted(output_gt)) def test_getitem(self) -> None: x = LazyPath(lambda: "abc") with self.assertRaises(TypeError): x[0] _ = os.fspath(x) self.assertEqual(x[0], "a")
class TestDriver: LEN = 100000000 # This many characters per append job NUM_JOBS = 10 _pathmgr = PathManager() def test(self) -> None: with tempfile.TemporaryDirectory() as _tmpdir: URI = os.path.join(_tmpdir, "test.txt") start_time = time.time() printx(f"Start dispatching {self.NUM_JOBS} async write jobs " f"each with {self.LEN} characters") FINAL_STR = "" with self._pathmgr.opena(URI, "a") as f: for i in range(self.NUM_JOBS): # `i` goes from 0 to 9 FINAL_STR += f"{i}" * self.LEN f.write(f"{i}" * self.LEN) mid_time = time.time() printx( f"Time taken to dispatch {self.NUM_JOBS} threads: {mid_time - start_time}" ) printx("Calling `async_join()`") # We want this `async_join` call to take time. If it is instantaneous, then our # async write calls are not running asynchronously. assert self._pathmgr.async_join() printx( f"Time Python waited for `async_join()` call to finish: {time.time() - mid_time}" ) assert self._pathmgr.async_close() with self._pathmgr.open(URI, "r") as f: assert f.read() == FINAL_STR printx("Async Writes Test finish.") printx( "Passing metric: " "If the `async_join()` call took more than a negligible time to complete, " "then Python waited for the threads to finish and the Async Writes " "Test SUCCEEDS. Otherwise FAILURE.")
def _load_texture_images( material_names: List[str], data_dir: str, material_properties: MaterialProperties, texture_files: TextureFiles, path_manager: PathManager, ) -> Tuple[MaterialProperties, TextureImages]: final_material_properties = {} texture_images = {} used_material_names = list(material_names) if not used_material_names and material_properties: if len(material_properties) > 1: raise ValueError( "Multiple materials but no usemtl declarations in the obj file" ) # No materials were specified in obj file and only one is in the # specified .mtl file, so we use it. used_material_names.append(next(iter(material_properties.keys()))) # Only keep the materials referenced in the obj. for material_name in used_material_names: if material_name in texture_files: # Load the texture image. path = os.path.join(data_dir, texture_files[material_name]) if path_manager.exists(path): image = (_read_image( path, path_manager=path_manager, format="RGB") / 255.0) image = torch.from_numpy(image) texture_images[material_name] = image else: msg = f"Texture file does not exist: {path}" warnings.warn(msg) if material_name in material_properties: final_material_properties[material_name] = material_properties[ material_name] return final_material_properties, texture_images
def _siamese_to_coco(self, siamese_json): assert self._output_dir save_json = os.path.join(self._output_dir, "siamese2coco.json") pm = PathManager() pm.mkdirs(os.path.dirname(save_json)) with file_lock(save_json): if pm.exists(save_json): logger.warning( f"Using previously cached COCO format annotations at '{save_json}'. " "You need to clear the cache file if your dataset has been modified." ) else: logger.info( f"Converting annotations of dataset '{siamese_json}' to COCO format ...)" ) with pm.open(siamese_json, "r") as f: siamese_data = json.load(f) coco_data = {"data": []} exist_imgid = set() for key, datas in siamese_data.items(): # copy 'info', 'categories' if key != "data": coco_data[key] = datas else: for data in datas: for i in range(2): img_data = data[str(i)] if img_data["image_id"] in exist_imgid: continue else: exist_imgid.add(img_data["image_id"]) coco_data[key].append(img_data) self._logger.info( f"Number of unique images: {len(exist_imgid)}.") coco_data = convert_to_coco_dict(coco_data["data"], self._metadata) with pm.open(save_json, "w") as f: json.dump(coco_data, f) return save_json
class TestNativeIOAsync(unittest.TestCase): """ This test class is meant to have comprehensive tests for `NativePathHandler`. Async functionality tests for other `PathHandler`-s should only require a single test since all `PathHandler`-s operate in the same way. """ _tmpdir: Optional[str] = None _pathmgr = PathManager() @classmethod def setUpClass(cls) -> None: cls._tmpdir = tempfile.mkdtemp() @classmethod def tearDownClass(cls) -> None: # Cleanup temp working dir. if cls._tmpdir is not None: shutil.rmtree(cls._tmpdir) # type: ignore def setUp(self) -> None: # Reset class variables set by methods before each test. self._pathmgr.set_cwd(None) self._pathmgr._native_path_handler._non_blocking_io_manager = None self._pathmgr._native_path_handler._non_blocking_io_executor = None self._pathmgr._async_handlers.clear() def test_opena(self) -> None: _tmpfile = os.path.join(self._tmpdir, "async.txt") try: # Write the files. with self._pathmgr.opena(_tmpfile + "f", "w") as f: f.write("f1 ") with self._pathmgr.opena(_tmpfile + "g", "w") as g: f.write("f2 ") g.write("g1 ") f.write("f3 ") f.write("f4 ") with self._pathmgr.opena(_tmpfile + "f", "a") as f: f.write("f5 ") F_STR = "f1 f2 f3 f4 f5 " G_STR = "g1 " # Test that `PathManager._async_handlers` keeps track of all # `PathHandler`-s where `opena` is used. self.assertCountEqual( [type(handler) for handler in self._pathmgr._async_handlers], [type(self._pathmgr._native_path_handler)], ) # Test that 2 paths were properly logged in `NonBlockingIOManager`. manager = self._pathmgr._native_path_handler._non_blocking_io_manager self.assertEqual(len(manager._path_to_data), 2) finally: # Join the threads to wait for files to be written. self.assertTrue(self._pathmgr.async_close()) # Check that both files were asynchronously written and written in order. with self._pathmgr.open(_tmpfile + "f", "r") as f: self.assertEqual(f.read(), F_STR) with self._pathmgr.open(_tmpfile + "g", "r") as g: self.assertEqual(g.read(), G_STR) # Test that both `NonBlockingIO` objects `f` and `g` are finally closed. self.assertEqual(len(manager._path_to_data), 0) def test_async_join_behavior(self) -> None: _tmpfile = os.path.join(self._tmpdir, "async.txt") _tmpfile_contents = "Async Text" try: for _ in range(1): # Opens 1 thread with self._pathmgr.opena(_tmpfile + "1", "w") as f: f.write(f"{_tmpfile_contents}-1") for _ in range(2): # Opens 2 threads with self._pathmgr.opena(_tmpfile + "2", "w") as f: f.write(f"{_tmpfile_contents}-2") for _ in range(3): # Opens 3 threads with self._pathmgr.opena(_tmpfile + "3", "w") as f: f.write(f"{_tmpfile_contents}-3") _path_to_data = ( self._pathmgr._native_path_handler._non_blocking_io_manager._path_to_data ) # Join the threads for the 1st and 3rd file and ensure threadpool completed. _path_to_data_copy = dict(_path_to_data) self.assertTrue( self._pathmgr.async_join( _tmpfile + "1", _tmpfile + "3" ) # Removes paths from `_path_to_io`. ) self.assertFalse(_path_to_data_copy[_tmpfile + "1"].thread.is_alive()) self.assertFalse(_path_to_data_copy[_tmpfile + "3"].thread.is_alive()) self.assertEqual(len(_path_to_data), 1) # 1 file remaining finally: # Join all the remaining threads _path_to_data_copy = dict(_path_to_data) self.assertTrue(self._pathmgr.async_close()) # Ensure data cleaned up. self.assertFalse(_path_to_data_copy[_tmpfile + "2"].thread.is_alive()) self.assertEqual(len(self._pathmgr._async_handlers), 0) self.assertEqual(len(_path_to_data), 0) # 0 files remaining def test_opena_normpath(self) -> None: _filename = "async.txt" # `_file1` and `_file2` should represent the same path but have different # string representations. _file1 = os.path.join(self._tmpdir, _filename) _file2 = os.path.join(self._tmpdir, ".", _filename) self.assertNotEqual(_file1, _file2) try: _file1_text = "File1 text" _file2_text = "File2 text" with self._pathmgr.opena(_file1, "w") as f: f.write(_file1_text) with self._pathmgr.opena(_file2, "a") as f: f.write(_file2_text) _path_to_data = ( self._pathmgr._native_path_handler._non_blocking_io_manager._path_to_data ) # Check that `file2` is marked as the same file as `file1`. self.assertEqual(len(_path_to_data), 1) self.assertTrue(self._pathmgr.async_join()) # Check that both file paths give the same file contents. with self._pathmgr.open(_file1, "r") as f: self.assertEqual(f.read(), _file1_text + _file2_text) with self._pathmgr.open(_file2, "r") as f: self.assertEqual(f.read(), _file1_text + _file2_text) finally: self.assertTrue(self._pathmgr.async_close()) def test_async_consecutive_join_calls(self) -> None: _file = os.path.join(self._tmpdir, "async.txt") try: self.assertTrue(self._pathmgr.async_join()) try: with self._pathmgr.opena(_file, "w") as f: f.write("1") finally: self.assertTrue(self._pathmgr.async_join()) with self._pathmgr.open(_file, "r") as f: self.assertEqual(f.read(), "1") try: f = self._pathmgr.opena(_file, "a") f.write("2") f.close() finally: self.assertTrue(self._pathmgr.async_join()) with self._pathmgr.open(_file, "r") as f: self.assertEqual(f.read(), "12") finally: self.assertTrue(self._pathmgr.async_close()) def test_opena_mode_restriction(self) -> None: _file = os.path.join(self._tmpdir, "async.txt") with self.assertRaises(ValueError): self._pathmgr.opena(_file, "r") with self.assertRaises(ValueError): self._pathmgr.opena(_file, "rb") with self.assertRaises(ValueError): self._pathmgr.opena(_file, "wrb") def test_opena_args_passed_correctly(self) -> None: _file = os.path.join(self._tmpdir, "async.txt") try: # Make sure that `opena` args are used correctly by using # different newline args. with self._pathmgr.opena(_file, "w", newline="\r\n") as f: f.write("1\n") with self._pathmgr.opena(_file, "a", newline="\n") as f: f.write("2\n3") finally: self.assertTrue(self._pathmgr.async_close()) # Read the raw file data without converting newline endings to see # if the `opena` args were used correctly. with self._pathmgr.open(_file, "r", newline="") as f: self.assertEqual(f.read(), "1\r\n2\n3") def test_opena_with_callback(self) -> None: _file_tmp = os.path.join(self._tmpdir, "async.txt.tmp") _file = os.path.join(self._tmpdir, "async.txt") _data = "Asynchronously written text" def cb(): # Insert a test to make sure `_file_tmp` was closed before # the callback is called. with open(_file_tmp, "r") as f: self.assertEqual(f.read(), _data) self._pathmgr.copy(_file_tmp, _file, overwrite=True) mock_cb = Mock(side_effect=cb) try: with self._pathmgr.opena( _file_tmp, "w", callback_after_file_close=mock_cb ) as f: f.write(_data) finally: self.assertTrue(self._pathmgr.async_close()) # Callback should have been called exactly once. mock_cb.assert_called_once() # Data should have been written to both `_file_tmp` and `_file`. with open(_file_tmp, "r") as f: self.assertEqual(f.read(), _data) with open(_file, "r") as f: self.assertEqual(f.read(), _data) def test_opena_with_callback_only_called_once(self) -> None: _file_tmp = os.path.join(self._tmpdir, "async.txt.tmp") mock_cb = Mock() # Callback should be called once even if `close` is called # multiple times. try: f = self._pathmgr.opena(_file_tmp, "w", callback_after_file_close=mock_cb) f.close() f.close() f.close() finally: self.assertTrue(self._pathmgr.async_close()) # Callback should have been called exactly once. mock_cb.assert_called_once() def test_async_custom_executor(self) -> None: # At first, neither manager nor executor are set. self.assertIsNone(self._pathmgr._native_path_handler._non_blocking_io_manager) self.assertIsNone(self._pathmgr._native_path_handler._non_blocking_io_executor) # Then, override the `NativePathHandler` and set a custom executor. executor = concurrent.futures.ThreadPoolExecutor( max_workers=128, thread_name_prefix="my prefix" ) ph = NativePathHandler(async_executor=executor) self._pathmgr.register_handler(ph, allow_override=True) self.assertEqual(ph, self._pathmgr._native_path_handler) # Opening a file with `opena` initializes the manager with the # executor. _file = os.path.join(self._tmpdir, "async.txt") try: with self._pathmgr.opena(_file, "w") as f: f.write("Text") # Make sure the manager's executor is the same as the user's. self.assertEqual( executor, self._pathmgr._native_path_handler._non_blocking_io_manager._pool, ) finally: self.assertTrue(self._pathmgr.async_close()) def test_non_blocking_io_seekable(self) -> None: _file = os.path.join(self._tmpdir, "async.txt") # '^' marks the current position in stream # Test seek. try: with self._pathmgr.opena(_file, "wb") as f: f.write(b"012345") # file = 012345^ f.seek(1) # file = 0^12345 f.write(b"##") # file = 0##^345 finally: self.assertTrue(self._pathmgr.async_join()) with self._pathmgr.open(_file, "rb") as f: self.assertEqual(f.read(), b"0##345") # Test truncate. try: with self._pathmgr.opena(_file, "wb") as f: f.write(b"012345") # file = 012345^ f.seek(2) # file = 01^2345 f.truncate() # file = 01^ finally: self.assertTrue(self._pathmgr.async_join()) with self._pathmgr.open(_file, "rb") as f: self.assertEqual(f.read(), b"01") # Big test for seek and truncate. try: with self._pathmgr.opena(_file, "wb") as f: f.write(b"0123456789") # file = 0123456789^ f.seek(2) # file = 01^23456789 f.write(b"##") # file = 01##^456789 f.seek(3, io.SEEK_CUR) # file = 01##456^789 f.truncate() # file = 01##456^ f.write(b"$") # file = 01##456$^ finally: self.assertTrue(self._pathmgr.async_join()) with self._pathmgr.open(_file, "rb") as f: self.assertEqual(f.read(), b"01##456$") # Test NOT tellable. try: with self._pathmgr.opena(_file, "wb") as f: with self.assertRaises(ValueError): f.tell() finally: self._pathmgr.async_close()
def load_ply(f, path_manager: Optional[PathManager] = None): """ Load the data from a .ply file. Example .ply file format: ply format ascii 1.0 { ascii/binary, format version number } comment made by Greg Turk { comments keyword specified, like all lines } comment this file is a cube element vertex 8 { define "vertex" element, 8 of them in file } property float x { vertex contains float "x" coordinate } property float y { y coordinate is also a vertex property } property float z { z coordinate, too } element face 6 { there are 6 "face" elements in the file } property list uchar int vertex_index { "vertex_indices" is a list of ints } end_header { delimits the end of the header } 0 0 0 { start of vertex list } 0 0 1 0 1 1 0 1 0 1 0 0 1 0 1 1 1 1 1 1 0 4 0 1 2 3 { start of face list } 4 7 6 5 4 4 0 4 5 1 4 1 5 6 2 4 2 6 7 3 4 3 7 4 0 Args: f: A binary or text file-like object (with methods read, readline, tell and seek), a pathlib path or a string containing a file name. If the ply file is in the binary ply format rather than the text ply format, then a text stream is not supported. It is easiest to use a binary stream in all cases. path_manager: PathManager for loading if f is a str. Returns: verts: FloatTensor of shape (V, 3). faces: LongTensor of vertex indices, shape (F, 3). """ if path_manager is None: path_manager = PathManager() header, elements = _load_ply_raw(f, path_manager=path_manager) vertex = elements.get("vertex", None) if vertex is None: raise ValueError("The ply file has no vertex element.") face = elements.get("face", None) if face is None: raise ValueError("The ply file has no face element.") if len(vertex) and (not isinstance(vertex, np.ndarray) or vertex.ndim != 2 or vertex.shape[1] != 3): raise ValueError("Invalid vertices in file.") verts = _make_tensor(vertex, cols=3, dtype=torch.float32) face_head = next(head for head in header.elements if head.name == "face") if len(face_head.properties ) != 1 or face_head.properties[0].list_size_type is None: raise ValueError("Unexpected form of faces data.") # face_head.properties[0].name is usually "vertex_index" or "vertex_indices" # but we don't need to enforce this. if not len(face): # pyre-fixme[28]: Unexpected keyword argument `size`. faces = torch.zeros(size=(0, 3), dtype=torch.int64) elif isinstance(face, np.ndarray) and face.ndim == 2: # Homogeneous elements if face.shape[1] < 3: raise ValueError("Faces must have at least 3 vertices.") face_arrays = [ face[:, [0, i + 1, i + 2]] for i in range(face.shape[1] - 2) ] faces = torch.LongTensor(np.vstack(face_arrays)) else: face_list = [] for face_item in face: if face_item.ndim != 1: raise ValueError("Bad face data.") if face_item.shape[0] < 3: raise ValueError("Faces must have at least 3 vertices.") for i in range(face_item.shape[0] - 2): face_list.append( [face_item[0], face_item[i + 1], face_item[i + 2]]) # pyre-fixme[6]: Expected `dtype` for 3rd param but got `Type[torch.int64]`. faces = _make_tensor(face_list, cols=3, dtype=torch.int64) _check_faces_indices(faces, max_index=verts.shape[0]) return verts, faces
def _open(self, path, mode="r", **kwargs): return PathManager.open(self._get_local_path(path), mode, **kwargs)
class LayoutParserHandler(PathHandler): """ Resolve anything that's in LayoutParser model zoo. """ PREFIX = "lp://" def _get_supported_prefixes(self): return [self.PREFIX] def _get_local_path(self, path, **kwargs): model_name = path[len(self.PREFIX):] dataset_name, *model_name, data_type = model_name.split('/') if data_type == 'weight': model_url = MODEL_CATALOG[dataset_name]['/'.join(model_name)] elif data_type == 'config': model_url = CONFIG_CATALOG[dataset_name]['/'.join(model_name)] else: raise ValueError(f"Unknown data_type {data_type}") return PathManager.get_local_path(model_url, **kwargs) def _open(self, path, mode="r", **kwargs): return PathManager.open(self._get_local_path(path), mode, **kwargs) PathManager = PathManagerBase() PathManager.register_handler(DropboxHandler()) PathManager.register_handler(LayoutParserHandler())
def load_obj( f, load_textures=True, create_texture_atlas: bool = False, texture_atlas_size: int = 4, texture_wrap: Optional[str] = "repeat", device="cpu", path_manager: Optional[PathManager] = None, ): """ Load a mesh from a .obj file and optionally textures from a .mtl file. Currently this handles verts, faces, vertex texture uv coordinates, normals, texture images and material reflectivity values. Note .obj files are 1-indexed. The tensors returned from this function are 0-indexed. OBJ spec reference: http://www.martinreddy.net/gfx/3d/OBJ.spec Example .obj file format: :: # this is a comment v 1.000000 -1.000000 -1.000000 v 1.000000 -1.000000 1.000000 v -1.000000 -1.000000 1.000000 v -1.000000 -1.000000 -1.000000 v 1.000000 1.000000 -1.000000 vt 0.748573 0.750412 vt 0.749279 0.501284 vt 0.999110 0.501077 vt 0.999455 0.750380 vn 0.000000 0.000000 -1.000000 vn -1.000000 -0.000000 -0.000000 vn -0.000000 -0.000000 1.000000 f 5/2/1 1/2/1 4/3/1 f 5/1/1 4/3/1 2/4/1 The first character of the line denotes the type of input: :: - v is a vertex - vt is the texture coordinate of one vertex - vn is the normal of one vertex - f is a face Faces are interpreted as follows: :: 5/2/1 describes the first vertex of the first triange - 5: index of vertex [1.000000 1.000000 -1.000000] - 2: index of texture coordinate [0.749279 0.501284] - 1: index of normal [0.000000 0.000000 -1.000000] If there are faces with more than 3 vertices they are subdivided into triangles. Polygonal faces are assummed to have vertices ordered counter-clockwise so the (right-handed) normal points out of the screen e.g. a proper rectangular face would be specified like this: :: 0_________1 | | | | 3 ________2 The face would be split into two triangles: (0, 2, 1) and (0, 3, 2), both of which are also oriented counter-clockwise and have normals pointing out of the screen. Args: f: A file-like object (with methods read, readline, tell, and seek), a pathlib path or a string containing a file name. load_textures: Boolean indicating whether material files are loaded create_texture_atlas: Bool, If True a per face texture map is created and a tensor `texture_atlas` is also returned in `aux`. texture_atlas_size: Int specifying the resolution of the texture map per face when `create_texture_atlas=True`. A (texture_size, texture_size, 3) map is created per face. texture_wrap: string, one of ["repeat", "clamp"]. This applies when computing the texture atlas. If `texture_mode="repeat"`, for uv values outside the range [0, 1] the integer part is ignored and a repeating pattern is formed. If `texture_mode="clamp"` the values are clamped to the range [0, 1]. If None, then there is no transformation of the texture values. device: string or torch.device on which to return the new tensors. path_manager: optionally a PathManager object to interpret paths. Returns: 6-element tuple containing - **verts**: FloatTensor of shape (V, 3). - **faces**: NamedTuple with fields: - verts_idx: LongTensor of vertex indices, shape (F, 3). - normals_idx: (optional) LongTensor of normal indices, shape (F, 3). - textures_idx: (optional) LongTensor of texture indices, shape (F, 3). This can be used to index into verts_uvs. - materials_idx: (optional) List of indices indicating which material the texture is derived from for each face. If there is no material for a face, the index is -1. This can be used to retrieve the corresponding values in material_colors/texture_images after they have been converted to tensors or Materials/Textures data structures - see textures.py and materials.py for more info. - **aux**: NamedTuple with fields: - normals: FloatTensor of shape (N, 3) - verts_uvs: FloatTensor of shape (T, 2), giving the uv coordinate per vertex. If a vertex is shared between two faces, it can have a different uv value for each instance. Therefore it is possible that the number of verts_uvs is greater than num verts i.e. T > V. vertex. - material_colors: if `load_textures=True` and the material has associated properties this will be a dict of material names and properties of the form: .. code-block:: python { material_name_1: { "ambient_color": tensor of shape (1, 3), "diffuse_color": tensor of shape (1, 3), "specular_color": tensor of shape (1, 3), "shininess": tensor of shape (1) }, material_name_2: {}, ... } If a material does not have any properties it will have an empty dict. If `load_textures=False`, `material_colors` will None. - texture_images: if `load_textures=True` and the material has a texture map, this will be a dict of the form: .. code-block:: python { material_name_1: (H, W, 3) image, ... } If `load_textures=False`, `texture_images` will None. - texture_atlas: if `load_textures=True` and `create_texture_atlas=True`, this will be a FloatTensor of the form: (F, texture_size, textures_size, 3) If the material does not have a texture map, then all faces will have a uniform white texture. Otherwise `texture_atlas` will be None. """ data_dir = "./" # pyre-fixme[6]: Expected `Union[typing.Type[typing.Any], # typing.Tuple[typing.Type[typing.Any], ...]]` for 2nd param but got `Any`. if isinstance(f, (str, bytes, os.PathLike)): # pyre-fixme[6]: Expected `_PathLike[Variable[typing.AnyStr <: [str, # bytes]]]` for 1st param but got `Union[_PathLike[typing.Any], bytes, str]`. data_dir = os.path.dirname(f) if path_manager is None: path_manager = PathManager() with _open_file(f, path_manager, "r") as f: return _load_obj( f, data_dir=data_dir, load_textures=load_textures, create_texture_atlas=create_texture_atlas, texture_atlas_size=texture_atlas_size, texture_wrap=texture_wrap, path_manager=path_manager, device=device, )
import struct import unittest from io import BytesIO, StringIO from tempfile import NamedTemporaryFile, TemporaryFile import numpy as np import pytorch3d.io.ply_io import torch from common_testing import TestCaseMixin from iopath.common.file_io import PathManager from pytorch3d.io import IO from pytorch3d.io.ply_io import load_ply, save_ply from pytorch3d.structures import Pointclouds from pytorch3d.utils import torus global_path_manager = PathManager() def _load_ply_raw(stream): return pytorch3d.io.ply_io._load_ply_raw(stream, global_path_manager) CUBE_PLY_LINES = [ "ply", "format ascii 1.0", "comment made by Greg Turk", "comment this file is a cube", "element vertex 8", "property float x", "property float y", "property float z",
# Copyright (c) Facebook, Inc. and its affiliates. from iopath.common.file_io import HTTPURLHandler, OneDrivePathHandler, PathHandler from iopath.common.file_io import PathManager as PathManagerBase __all__ = ["PathManager", "PathHandler"] PathManager = PathManagerBase() """ This is a detectron2 project-specific PathManager. We try to stay away from global PathManager in fvcore as it introduces potential conflicts among other libraries. """ class Detectron2Handler(PathHandler): """ Resolve anything that's hosted under detectron2's namespace. """ PREFIX = "detectron2://" S3_DETECTRON2_PREFIX = "https://dl.fbaipublicfiles.com/detectron2/" def _get_supported_prefixes(self): return [self.PREFIX] def _get_local_path(self, path): name = path[len(self.PREFIX):] return PathManager.get_local_path(self.S3_DETECTRON2_PREFIX + name) def _open(self, path, mode="r", **kwargs): return PathManager.open(self._get_local_path(path), mode, **kwargs)
# Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from iopath.common.file_io import HTTPURLHandler from iopath.common.file_io import PathManager as PathManagerBase # A trick learned from https://github.com/facebookresearch/detectron2/blob/65faeb4779e4c142484deeece18dc958c5c9ad18/detectron2/utils/file_io.py#L3 class DropboxHandler(HTTPURLHandler): """ Supports download and file check for dropbox links """ def _get_supported_prefixes(self): return ["https://www.dropbox.com"] def _isfile(self, path): return path in self.cache_map PathManager = PathManagerBase() PathManager.register_handler(DropboxHandler())