def persistent_load(saved_id): assert isinstance(saved_id, tuple) typename = _maybe_decode_ascii(saved_id[0]) data = saved_id[1:] if typename == "storage": storage_type, key, location, size = data dtype = storage_type.dtype if key not in loaded_storages: load_tensor( dtype, size, key, _maybe_decode_ascii(location), restore_location, ) storage = loaded_storages[key] # TODO: Once we decide to break serialization FC, we can # stop wrapping with _TypedStorage return torch.storage._TypedStorage( wrap_storage=storage._untyped(), dtype=dtype) elif typename == "reduce_package": # to fix BC breaking change, objects on this load path # will be loaded multiple times erroneously if len(data) == 2: func, args = data return func(self, *args) reduce_id, func, args = data if reduce_id not in loaded_reduces: loaded_reduces[reduce_id] = func(self, *args) return loaded_reduces[reduce_id] else: f"Unknown typename for persistent_load, expected 'storage' or 'reduce_package' but got '{typename}'"
def persistent_load(saved_id): assert isinstance(saved_id, tuple) typename = _maybe_decode_ascii(saved_id[0]) data = saved_id[1:] if typename == "storage": data_type, key, location, size = data if key not in loaded_storages: load_tensor( data_type, size, key, _maybe_decode_ascii(location), restore_location, ) storage = loaded_storages[key] return storage elif typename == "reduce_package": # to fix BC breaking change, objects on this load path # will be loaded multiple times erroneously if len(data) == 2: func, args = data return func(self, *args) reduce_id, func, args = data if reduce_id not in loaded_reduces: loaded_reduces[reduce_id] = func(self, *args) return loaded_reduces[reduce_id] else: f"Unknown typename for persistent_load, expected 'storage' or 'reduce_package' but got '{typename}'"
def persistent_load(saved_id): assert isinstance(saved_id, tuple) typename = _maybe_decode_ascii(saved_id[0]) data = saved_id[1:] if typename == 'module': # Ignore containers that don't have any sources saved if all(data[1:]): _check_container_source(*data) return data[0] elif typename == 'storage': data_type, root_key, location, size, view_metadata = data location = _maybe_decode_ascii(location) if root_key not in deserialized_objects: obj = data_type(size) obj._torch_load_uninitialized = True s = str(root_key) + '.bint' if not os.path.isfile(s): with open(s, 'wb') as ff: obj._write_file(ff, True, False) obj = obj.__class__.from_file(s, shared=1, size=size) deserialized_objects[root_key] = restore_location( obj, location) storage = deserialized_objects[root_key] if view_metadata is not None: view_key, offset, view_size = view_metadata if view_key not in deserialized_objects: deserialized_objects[view_key] = storage[offset:offset + view_size] return deserialized_objects[view_key] else: return storage else: raise RuntimeError("Unknown saved id type: %s" % saved_id[0])
def persistent_load(saved_id): assert isinstance(saved_id, tuple) typename = _maybe_decode_ascii(saved_id[0]) data = saved_id[1:] if typename == "module": # Ignore containers that don't have any sources saved if all(data[1:]): _check_container_source(*data) return data[0] elif typename == "storage": data_type, root_key, location, size, view_metadata = data location = _maybe_decode_ascii(location) if root_key not in deserialized_objects: obj = data_type(size) obj._torch_load_uninitialized = True deserialized_objects[root_key] = restore_location( obj, location) storage = deserialized_objects[root_key] if view_metadata is not None: view_key, offset, view_size = view_metadata if view_key not in deserialized_objects: deserialized_objects[view_key] = storage[offset:offset + view_size] return deserialized_objects[view_key] else: return storage else: raise RuntimeError("Unknown saved id type: %s" % saved_id[0])
def persistent_load(saved_id): assert isinstance(saved_id, tuple) typename = _maybe_decode_ascii(saved_id[0]) data = saved_id[1:] assert typename == 'storage', \ f"Unknown typename for persistent_load, expected 'storage' but got '{typename}'" data_type, key, location, size = data if key not in loaded_storages: load_tensor(data_type, size, key, _maybe_decode_ascii(location), restore_location) storage = loaded_storages[key] return storage
def persistent_load(saved_id): assert isinstance(saved_id, tuple) typename = _maybe_decode_ascii(saved_id[0]) data = saved_id[1:] if typename == 'storage': data_type, key, location, size = data if key not in loaded_storages: load_tensor(data_type, size, key, _maybe_decode_ascii(location), restore_location) storage = loaded_storages[key] return storage elif typename == 'reduce_package': func, args = data return func(self, *args) else: f"Unknown typename for persistent_load, expected 'storage' but got '{typename}'"
def persistent_load(saved_id): assert isinstance(saved_id, tuple) typename = _maybe_decode_ascii(saved_id[0]) data = saved_id[1:] assert typename == 'storage', \ f"Unknown typename for persistent_load, expected 'storage' but got '{typename}'" return serialized_storages[data[0]]
def persistent_load(saved_id): assert isinstance(saved_id, tuple) typename = _maybe_decode_ascii(saved_id[0]) data = saved_id[1:] if typename == 'storage': return serialized_storages[data[0]] if typename == 'reduce_deploy': reduce_id, func, args = data if reduce_id not in _loaded_reduces: _loaded_reduces[reduce_id] = func(_raw_packages[zip_reader], *args) return _loaded_reduces[reduce_id] return None
def persistent_load(saved_id): assert isinstance(saved_id, tuple) typename = _maybe_decode_ascii(saved_id[0]) data = saved_id[1:] if typename == "storage": # TODO: Once we decide to break serialization FC, we can # stop wrapping with TypedStorage storage = serialized_storages[data[0]] dtype = serialized_dtypes[data[0]] return torch.storage.TypedStorage(wrap_storage=storage.untyped(), dtype=dtype) if typename == "reduce_deploy": reduce_id, func, args = data if reduce_id not in _loaded_reduces: _loaded_reduces[reduce_id] = func(_raw_packages[zip_reader], *args) return _loaded_reduces[reduce_id] return None