def create_path_manager() -> PathManager:
# TODO: move this inline import out after AIRStore OSS public released
from airstore.client.airstore_tabular import AIRStorePathHandler
pathmanager = PathManager()
pathmanager.register_handler(AIRStorePathHandler())
pathmanager.set_strict_kwargs_checking(False)
return pathmanager
def test_open_new_path_manager(self) -> None:
with self._patch_download():
path_manager = PathManager()
with self.assertRaises(OSError): # no handler registered
f = path_manager.open(self._remote_uri, "rb")
path_manager.register_handler(HTTPURLHandler())
with path_manager.open(self._remote_uri, "rb") as f:
self.assertTrue(os.path.isfile(f.name))
self.assertTrue(f.read() != "")
def get_skateboard_data(
avoid_manifold: bool = False,
silence_logs: bool = False
) -> Generator[Tuple[str, PathManager], None, None]:
"""
Context manager for accessing Co3D dataset by tests, at least for
the first 5 skateboards. Internally, we want this to exercise the
normal way to access the data directly manifold, but on an RE
worker this is impossible so we use a workaround.
Args:
avoid_manifold: Use the method used by RE workers even locally.
silence_logs: Whether to reduce log output from iopath library.
Yields:
dataset_root: (str) path to dataset root.
path_manager: path_manager to access it with.
"""
path_manager = PathManager()
if silence_logs:
logging.getLogger("iopath.fb.manifold").setLevel(logging.CRITICAL)
logging.getLogger("iopath.common.file_io").setLevel(logging.CRITICAL)
if not os.environ.get("FB_TEST", False):
if os.getenv("FAIR_ENV_CLUSTER", "") == "":
raise unittest.SkipTest("Unknown environment. Data not available.")
yield "/checkpoint/dnovotny/datasets/co3d/download_aws_22_02_18", path_manager
elif avoid_manifold or os.environ.get("INSIDE_RE_WORKER", False):
from libfb.py.parutil import get_file_path
par_path = "skateboard_first_5"
source = get_file_path(par_path)
assert Path(source).is_file()
with tempfile.TemporaryDirectory() as dest:
with ZipFile(source) as f:
f.extractall(dest)
yield os.path.join(dest, "extracted"), path_manager
else:
from iopath.fb.manifold import ManifoldPathHandler
path_manager.register_handler(ManifoldPathHandler())
yield "manifold://co3d/tree/extracted", path_manager
class LayoutParserHandler(PathHandler):
"""
Resolve anything that's in LayoutParser model zoo.
"""
PREFIX = "lp://"
def _get_supported_prefixes(self):
return [self.PREFIX]
def _get_local_path(self, path, **kwargs):
model_name = path[len(self.PREFIX):]
dataset_name, *model_name, data_type = model_name.split('/')
if data_type == 'weight':
model_url = MODEL_CATALOG[dataset_name]['/'.join(model_name)]
elif data_type == 'config':
model_url = CONFIG_CATALOG[dataset_name]['/'.join(model_name)]
else:
raise ValueError(f"Unknown data_type {data_type}")
return PathManager.get_local_path(model_url, **kwargs)
def _open(self, path, mode="r", **kwargs):
return PathManager.open(self._get_local_path(path), mode, **kwargs)
PathManager = PathManagerBase()
PathManager.register_handler(DropboxHandler())
PathManager.register_handler(LayoutParserHandler())
PathManager = PathManagerBase()
"""
This is a detectron2 project-specific PathManager.
We try to stay away from global PathManager in fvcore as it
introduces potential conflicts among other libraries.
"""
class Detectron2Handler(PathHandler):
"""
Resolve anything that's hosted under detectron2's namespace.
"""
PREFIX = "detectron2://"
S3_DETECTRON2_PREFIX = "https://dl.fbaipublicfiles.com/detectron2/"
def _get_supported_prefixes(self):
return [self.PREFIX]
def _get_local_path(self, path):
name = path[len(self.PREFIX):]
return PathManager.get_local_path(self.S3_DETECTRON2_PREFIX + name)
def _open(self, path, mode="r", **kwargs):
return PathManager.open(self._get_local_path(path), mode, **kwargs)
PathManager.register_handler(HTTPURLHandler())
PathManager.register_handler(OneDrivePathHandler())
PathManager.register_handler(Detectron2Handler())
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from iopath.common.file_io import HTTPURLHandler
from iopath.common.file_io import PathManager as PathManagerBase
# A trick learned from https://github.com/facebookresearch/detectron2/blob/65faeb4779e4c142484deeece18dc958c5c9ad18/detectron2/utils/file_io.py#L3
class DropboxHandler(HTTPURLHandler):
"""
Supports download and file check for dropbox links
"""
def _get_supported_prefixes(self):
return ["https://www.dropbox.com"]
def _isfile(self, path):
return path in self.cache_map
PathManager = PathManagerBase()
PathManager.register_handler(DropboxHandler())
import mobile_cv.lut.lib.pt.flops_utils as flops_utils
import torch
from d2go.config import temp_defrost, CfgNode
from d2go.export.api import convert_and_export_predictor
from d2go.setup import (
basic_argument_parser,
prepare_for_launch,
setup_after_launch,
)
from iopath.common.file_io import PathManager
from iopath.fb.manifold import ManifoldPathHandler
from mobile_cv.common.misc.py import post_mortem_if_fail
path_manager = PathManager()
path_manager.register_handler(ManifoldPathHandler())
logger = logging.getLogger("d2go.tools.export")
INFERNCE_CONFIG_FILENAME = "inference_config.json"
MOBILE_OPTIMIZED_BUNDLE_FILENAME = "mobile_optimized_bundled.ptl"
def write_model_with_config(output_dir: str,
model_jit_path: str,
inference_config: Optional[CfgNode] = None):
"""
Writes the sdk inference config along with model file and saves the model
with configuration at ${output_dir}/mobile_optimized_bundled.ptl
"""
model_jit_local_path = path_manager.get_local_path(model_jit_path)
def __init__(self):
path_manager = PathManager()
path_manager.register_handler(HTTPURLHandler())
path_manager.register_handler(GoogleDrivePathHandler())
self.path_manager = path_manager
cache_dir: Optional[str] = None,
**kwargs: Any,
) -> str:
destination = kwargs["destination"]
local_path = self.path_manager.get_local_path(path, force)
shutil.move(local_path, destination)
return destination
def _open(self,
path: str,
mode: str = "r",
buffering: int = -1,
**kwargs: Any) -> Union[IO[str], IO[bytes]]:
self._check_kwargs(kwargs)
assert mode in ("r",
"rb"), "{} does not support open with {} mode".format(
self.__class__.__name__, mode)
assert (
buffering == -1
), f"{self.__class__.__name__} does not support the `buffering` argument"
local_path = self._get_local_path(path, force=False)
return open(local_path, mode)
_DATASET_DOWNLOAD_MANAGER = PathManager()
_DATASET_DOWNLOAD_MANAGER.register_handler(CombinedInternalPathhandler())
"""
This is a herbarium project-specific PathManager.
We try to stay away from global PathManager in fvcore as it
introduces potential conflicts among other libraries.
"""
class HerbariumHandler(PathHandler):
"""
Resolve anything that's hosted under herbarium's namespace.
"""
PREFIX = "herbarium://"
S3_DETECTRON2_PREFIX = "https://dl.fbaipublicfiles.com/herbarium/"
def _get_supported_prefixes(self):
return [self.PREFIX]
def _get_local_path(self, path, **kwargs):
name = path[len(self.PREFIX):]
return PathManager.get_local_path(self.S3_DETECTRON2_PREFIX + name,
**kwargs)
def _open(self, path, mode="r", **kwargs):
return PathManager.open(self._get_local_path(path), mode, **kwargs)
PathManager.register_handler(HTTPURLHandler())
PathManager.register_handler(OneDrivePathHandler())
PathManager.register_handler(HerbariumHandler())
