def ofrecord_loader( ofrecord_dir: str, batch_size: int = 1, data_part_num: int = 1, part_name_prefix: str = "part-", part_name_suffix_length: int = -1, shuffle: bool = False, shuffle_buffer_size: int = 1024, name: Optional[str] = None, ) -> remote_blob_util.BlobDef: if name is None: name = id_util.UniqueStr("OFRecord_Loader_") op_conf = op_conf_util.OperatorConf() op_conf.name = name op_conf.record_load_conf.out = "out" op_conf.record_load_conf.data_dir = ofrecord_dir op_conf.record_load_conf.data_part_num = data_part_num op_conf.record_load_conf.batch_size = batch_size op_conf.record_load_conf.part_name_prefix = part_name_prefix if part_name_suffix_length != -1: op_conf.record_load_conf.part_name_suffix_length = part_name_suffix_length if shuffle: op_conf.record_load_conf.random_shuffle_conf.buffer_size = shuffle_buffer_size lbi = logical_blob_id_util.LogicalBlobId() lbi.op_name = name lbi.blob_name = "out" interpret_util.ConsistentForward(op_conf) return remote_blob_util.RemoteBlob(lbi)
def distribute_add(xs, name=None): assert oneflow.placement.current_scope().parallel_size == len(xs) if name is None: name = id_util.UniqueStr("DistributeAdd_") op_conf = op_conf_util.OperatorConf() op_conf.name = name getattr(op_conf.distribute_add_conf, "in").extend([_SoleConsistentLbn(x) for x in xs]) op_conf.distribute_add_conf.out = "out" interpret_util.ConsistentForward(op_conf) lbi = logical_blob_id_util.LogicalBlobId() lbi.op_name = op_conf.name lbi.blob_name = "out" return remote_blob_util.RemoteBlob(lbi)
def distribute_concat(xs, axis=0, name=None): assert oneflow.current_scope( ).device_parallel_desc_symbol.parallel_num == len(xs) if name is None: name = id_util.UniqueStr("DistributeConcat_") op_conf = op_conf_util.OperatorConf() op_conf.name = name getattr(op_conf.distribute_concat_conf, "in").extend([_SoleConsistentLbn(x) for x in xs]) op_conf.distribute_concat_conf.axis = axis op_conf.distribute_concat_conf.out = "out" interpret_util.ConsistentForward(op_conf) lbi = logical_blob_id_util.LogicalBlobId() lbi.op_name = op_conf.name lbi.blob_name = "out" return remote_blob_util.RemoteBlob(lbi)
def decode_ofrecord( ofrecord_dir: str, blobs: Sequence[BlobConf], batch_size: int = 1, data_part_num: int = 1, part_name_prefix: str = "part-", part_name_suffix_length: int = -1, shuffle: bool = False, buffer_size: int = 1024, name: str = None, ) -> Tuple[remote_blob_util.BlobDef]: print( "WARNING:", "oneflow.data.decode_ofrecord is deprecated, and NOT work in eager mode, please use: \n", " 1) ofrecord = oneflow.data.ofrecord_reader(...) to read ofrecord; \n", " 2) image = oneflow.data.ofrecord_image_decoder(...) to decode image; \n", " 3) raw = oneflow.data.ofrecord_raw_decoder(...) to decode raw data like label; \n", traceback.format_stack()[-2], ) assert not flow.eager_execution_enabled() if name is None: name = id_util.UniqueStr("Decode_") lbis = [] op_conf = op_conf_util.OperatorConf() op_conf.name = name op_conf.decode_ofrecord_conf.data_dir = ofrecord_dir op_conf.decode_ofrecord_conf.data_part_num = data_part_num op_conf.decode_ofrecord_conf.batch_size = batch_size op_conf.decode_ofrecord_conf.part_name_prefix = part_name_prefix op_conf.decode_ofrecord_conf.part_name_suffix_length = part_name_suffix_length if shuffle == True: op_conf.decode_ofrecord_conf.random_shuffle_conf.buffer_size = buffer_size for blob_conf in blobs: op_conf.decode_ofrecord_conf.blob.extend([blob_conf.to_proto()]) lbi = logical_blob_id_util.LogicalBlobId() lbi.op_name = name lbi.blob_name = blob_conf.name lbis.append(lbi) interpret_util.ConsistentForward(op_conf) return tuple(map(lambda x: remote_blob_util.RemoteBlob(x), lbis))
def distribute_clone(x, name=None): if name is None: name = id_util.UniqueStr("DistributeClone_") op_conf = op_conf_util.OperatorConf() op_conf.name = name setattr(op_conf.distribute_clone_conf, "in", x.unique_name) parallel_size = oneflow.placement.current_scope().parallel_size op_conf.distribute_clone_conf.out.extend( ["out_%d" % i for i in range(parallel_size)]) interpret_util.ConsistentForward(op_conf) ret = [] for i in range(parallel_size): out = "out_%d" % i lbi = logical_blob_id_util.LogicalBlobId() lbi.op_name = op_conf.name lbi.blob_name = out ret.append(remote_blob_util.RemoteBlob(lbi)) return tuple(ret)
def decode_random( shape: Sequence[int], dtype: flow.dtype, batch_size: int = 1, initializer: Optional[initializer_conf_util.InitializerConf] = None, tick: Optional[oneflow_api.BlobDesc] = None, name: Optional[str] = None, ) -> oneflow_api.BlobDesc: op_conf = op_conf_util.OperatorConf() if name is None: name = id_util.UniqueStr("DecodeRandom_") assert isinstance(name, str) op_conf.name = name assert isinstance(shape, (list, tuple)) op_conf.decode_random_conf.shape.dim.extend(shape) assert dtype is not None setattr( op_conf.decode_random_conf, "data_type", oneflow_api.deprecated.GetProtoDtype4OfDtype(dtype), ) op_conf.decode_random_conf.batch_size = batch_size if initializer is not None: op_conf.decode_random_conf.data_initializer.CopyFrom(initializer) else: op_conf.decode_random_conf.data_initializer.CopyFrom( flow.random_uniform_initializer()) if tick: op_conf.decode_random_conf.tick = tick.unique_name op_conf.decode_random_conf.out = "out" lbi = logical_blob_id_util.LogicalBlobId() lbi.op_name = op_conf.name lbi.blob_name = "out" interpret_util.ConsistentForward(op_conf) return remote_blob_util.RemoteBlob(lbi)
def distribute_split(x, axis=0, name=None): if name is None: name = id_util.UniqueStr("DistributeSplit_") op_conf = op_conf_util.OperatorConf() op_conf.name = name setattr(op_conf.distribute_split_conf, "in", x.unique_name) op_conf.distribute_split_conf.axis = axis parallel_size = oneflow.current_scope( ).device_parallel_desc_symbol.parallel_num op_conf.distribute_split_conf.out.extend( ["out_%d" % i for i in range(parallel_size)]) interpret_util.ConsistentForward(op_conf) ret = [] for i in range(parallel_size): out = "out_%d" % i lbi = logical_blob_id_util.LogicalBlobId() lbi.op_name = op_conf.name lbi.blob_name = out ret.append(remote_blob_util.RemoteBlob(lbi)) return tuple(ret)
def InferAndTryRun(self): interpret_util.ConsistentForward(self.op_conf_) return self