def tensor_getstate(self): if save_load_path is not None: # save_load_path is not None means setstate/getstate is called inside # flow.save or flow.load assert isinstance(save_load_path, Path) if consistent_src_dsk_rank is None: assert self.is_local rel_dir_name = id_util.UniqueStr("tensor_") abs_dir_name = save_load_path / rel_dir_name tensor = self else: assert not self.is_local rel_dir_name = f"consistent_tensor_{self.consistent_id()}" abs_dir_name = save_load_path / rel_dir_name tensor = self.to_consistent(sbp=[flow.sbp.broadcast] * len(self.sbp)).to_local() if (consistent_src_dsk_rank is None or consistent_src_dsk_rank == flow.env.get_rank()): _save_tensor_to_disk(tensor, abs_dir_name) return {"path": rel_dir_name} else: # save_load_path is None means setstate/getstate is called inside # methods other than flow.save/load, for example, copy.deepcopy if self.is_local: return {"data": self.numpy(), "dtype": self.dtype} else: return { "data": self.numpy(), "dtype": self.dtype, "placement": self.placement, "sbp": self.sbp, }
def __init__( self, target_width: int, target_height: int, num_attempts: Optional[int] = 10, seed: Optional[int] = 0, random_area: Optional[Sequence[float]] = [0.08, 1.0], random_aspect_ratio: Optional[Sequence[float]] = [0.75, 1.333333], num_workers: Optional[int] = 3, warmup_size: Optional[int] = 6400, max_num_pixels: Optional[int] = 67108864, ): super().__init__() self.target_width = target_width self.target_height = target_height self.num_attempts = num_attempts self.seed = seed assert len(random_area) == 2 self.random_area = random_area assert len(random_aspect_ratio) == 2 self.random_aspect_ratio = random_aspect_ratio self.num_workers = num_workers self.warmup_size = warmup_size self.max_num_pixels = max_num_pixels gpu_decoder_conf = (flow._oneflow_internal.oneflow.core.operator. op_conf.ImageDecoderRandomCropResizeOpConf()) gpu_decoder_conf.set_in("error_input_need_to_be_replaced") gpu_decoder_conf.set_out("out") self._op = flow._oneflow_internal.one.ImageDecoderRandomCropResizeOpExpr( id_util.UniqueStr("ImageGpuDecoder"), gpu_decoder_conf, ["in"], ["out"])
def __init__(self, op_type_name, op_name=None): if op_name is None: op_name = id_util.UniqueStr(op_type_name) self._builder = oneflow._oneflow_internal.one.OpBuilder( op_type_name, op_name) self._op = None self._op_type_name = op_type_name
def tensor_getstate(self): if save_load_path is not None: # save_load_path is not None means setstate/getstate is called inside # flow.save or flow.load assert isinstance(save_load_path, Path) if global_src_dsk_rank is None: assert self.is_local rel_dir_name = id_util.UniqueStr("tensor_") abs_dir_name = save_load_path / rel_dir_name tensor = self else: assert not self.is_local rel_dir_name = f"global_tensor_{self.global_id()}" abs_dir_name = save_load_path / rel_dir_name tensor = self.to_global( sbp=flow.sbp.broadcast, placement=flow.placement("cpu", [global_src_dsk_rank]), ).to_local() if global_src_dsk_rank is None or global_src_dsk_rank == flow.env.get_rank( ): _save_tensor_to_disk(tensor, abs_dir_name) return {"path": rel_dir_name} else: # save_load_path is None means setstate/getstate is called inside # methods other than flow.save/load, for example, copy.deepcopy if self.is_local: if self.is_cuda: device = "cuda" else: device = "cpu" return { "data": self.numpy(), "dtype": self.dtype, "device": device } else: return { "data": self.numpy(), "dtype": self.dtype, "placement": self.placement, "sbp": self.sbp, }
def __init__( self, target_width: int, target_height: int, num_attempts: Optional[int] = 10, seed: Optional[int] = 0, random_area: Optional[Sequence[float]] = [0.08, 1.0], random_aspect_ratio: Optional[Sequence[float]] = [0.75, 1.333333], num_workers: Optional[int] = 3, warmup_size: Optional[int] = 6400, max_num_pixels: Optional[int] = 67108864, ): super().__init__() self.target_width = target_width self.target_height = target_height self.num_attempts = num_attempts self.seed = seed assert len(random_area) == 2 self.random_area = random_area assert len(random_aspect_ratio) == 2 self.random_aspect_ratio = random_aspect_ratio self.num_workers = num_workers self.warmup_size = warmup_size self.max_num_pixels = max_num_pixels gpu_decoder_conf = (flow.core.operator.op_conf_pb2. ImageDecoderRandomCropResizeOpConf()) # parse failed when excu clang format if use `gpu_decoder_conf.in = "error_input_need_to_be_replaced"` setattr(gpu_decoder_conf, "in", "error_input_need_to_be_replaced") gpu_decoder_conf.out = "out" gpu_decoder_conf.target_width = ( -1) # Set the default value, otherwise the parsing fails gpu_decoder_conf.target_height = -1 gpu_decoder_conf_str = text_format.MessageToString(gpu_decoder_conf) self._op = flow._oneflow_internal.one.ImageDecoderRandomCropResizeOpExpr( id_util.UniqueStr("ImageGpuDecoder"), gpu_decoder_conf_str, ["in"], ["out"])