def ofrecord_loader(
ofrecord_dir: str,
batch_size: int = 1,
data_part_num: int = 1,
part_name_prefix: str = "part-",
part_name_suffix_length: int = -1,
shuffle: bool = False,
shuffle_buffer_size: int = 1024,
name: Optional[str] = None,
) -> remote_blob_util.BlobDef:
if name is None:
name = id_util.UniqueStr("OFRecord_Loader_")
op_conf = op_conf_util.OperatorConf()
op_conf.name = name
op_conf.record_load_conf.out = "out"
op_conf.record_load_conf.data_dir = ofrecord_dir
op_conf.record_load_conf.data_part_num = data_part_num
op_conf.record_load_conf.batch_size = batch_size
op_conf.record_load_conf.part_name_prefix = part_name_prefix
if part_name_suffix_length != -1:
op_conf.record_load_conf.part_name_suffix_length = part_name_suffix_length
if shuffle:
op_conf.record_load_conf.random_shuffle_conf.buffer_size = shuffle_buffer_size
lbi = logical_blob_id_util.LogicalBlobId()
lbi.op_name = name
lbi.blob_name = "out"
interpret_util.ConsistentForward(op_conf)
return remote_blob_util.RemoteBlob(lbi)
def distribute_add(xs, name=None):
assert oneflow.placement.current_scope().parallel_size == len(xs)
if name is None:
name = id_util.UniqueStr("DistributeAdd_")
op_conf = op_conf_util.OperatorConf()
op_conf.name = name
getattr(op_conf.distribute_add_conf,
"in").extend([_SoleConsistentLbn(x) for x in xs])
op_conf.distribute_add_conf.out = "out"
interpret_util.ConsistentForward(op_conf)
lbi = logical_blob_id_util.LogicalBlobId()
lbi.op_name = op_conf.name
lbi.blob_name = "out"
return remote_blob_util.RemoteBlob(lbi)
def distribute_concat(xs, axis=0, name=None):
assert oneflow.current_scope(
).device_parallel_desc_symbol.parallel_num == len(xs)
if name is None:
name = id_util.UniqueStr("DistributeConcat_")
op_conf = op_conf_util.OperatorConf()
op_conf.name = name
getattr(op_conf.distribute_concat_conf,
"in").extend([_SoleConsistentLbn(x) for x in xs])
op_conf.distribute_concat_conf.axis = axis
op_conf.distribute_concat_conf.out = "out"
interpret_util.ConsistentForward(op_conf)
lbi = logical_blob_id_util.LogicalBlobId()
lbi.op_name = op_conf.name
lbi.blob_name = "out"
return remote_blob_util.RemoteBlob(lbi)
def decode_ofrecord(
ofrecord_dir: str,
blobs: Sequence[BlobConf],
batch_size: int = 1,
data_part_num: int = 1,
part_name_prefix: str = "part-",
part_name_suffix_length: int = -1,
shuffle: bool = False,
buffer_size: int = 1024,
name: str = None,
) -> Tuple[remote_blob_util.BlobDef]:
print(
"WARNING:",
"oneflow.data.decode_ofrecord is deprecated, and NOT work in eager mode, please use: \n",
" 1) ofrecord = oneflow.data.ofrecord_reader(...) to read ofrecord; \n",
" 2) image = oneflow.data.ofrecord_image_decoder(...) to decode image; \n",
" 3) raw = oneflow.data.ofrecord_raw_decoder(...) to decode raw data like label; \n",
traceback.format_stack()[-2],
)
assert not flow.eager_execution_enabled()
if name is None:
name = id_util.UniqueStr("Decode_")
lbis = []
op_conf = op_conf_util.OperatorConf()
op_conf.name = name
op_conf.decode_ofrecord_conf.data_dir = ofrecord_dir
op_conf.decode_ofrecord_conf.data_part_num = data_part_num
op_conf.decode_ofrecord_conf.batch_size = batch_size
op_conf.decode_ofrecord_conf.part_name_prefix = part_name_prefix
op_conf.decode_ofrecord_conf.part_name_suffix_length = part_name_suffix_length
if shuffle == True:
op_conf.decode_ofrecord_conf.random_shuffle_conf.buffer_size = buffer_size
for blob_conf in blobs:
op_conf.decode_ofrecord_conf.blob.extend([blob_conf.to_proto()])
lbi = logical_blob_id_util.LogicalBlobId()
lbi.op_name = name
lbi.blob_name = blob_conf.name
lbis.append(lbi)
interpret_util.ConsistentForward(op_conf)
return tuple(map(lambda x: remote_blob_util.RemoteBlob(x), lbis))
def distribute_clone(x, name=None):
if name is None:
name = id_util.UniqueStr("DistributeClone_")
op_conf = op_conf_util.OperatorConf()
op_conf.name = name
setattr(op_conf.distribute_clone_conf, "in", x.unique_name)
parallel_size = oneflow.placement.current_scope().parallel_size
op_conf.distribute_clone_conf.out.extend(
["out_%d" % i for i in range(parallel_size)])
interpret_util.ConsistentForward(op_conf)
ret = []
for i in range(parallel_size):
out = "out_%d" % i
lbi = logical_blob_id_util.LogicalBlobId()
lbi.op_name = op_conf.name
lbi.blob_name = out
ret.append(remote_blob_util.RemoteBlob(lbi))
return tuple(ret)
def decode_random(
shape: Sequence[int],
dtype: flow.dtype,
batch_size: int = 1,
initializer: Optional[initializer_conf_util.InitializerConf] = None,
tick: Optional[oneflow_api.BlobDesc] = None,
name: Optional[str] = None,
) -> oneflow_api.BlobDesc:
op_conf = op_conf_util.OperatorConf()
if name is None:
name = id_util.UniqueStr("DecodeRandom_")
assert isinstance(name, str)
op_conf.name = name
assert isinstance(shape, (list, tuple))
op_conf.decode_random_conf.shape.dim.extend(shape)
assert dtype is not None
setattr(
op_conf.decode_random_conf,
"data_type",
oneflow_api.deprecated.GetProtoDtype4OfDtype(dtype),
)
op_conf.decode_random_conf.batch_size = batch_size
if initializer is not None:
op_conf.decode_random_conf.data_initializer.CopyFrom(initializer)
else:
op_conf.decode_random_conf.data_initializer.CopyFrom(
flow.random_uniform_initializer())
if tick:
op_conf.decode_random_conf.tick = tick.unique_name
op_conf.decode_random_conf.out = "out"
lbi = logical_blob_id_util.LogicalBlobId()
lbi.op_name = op_conf.name
lbi.blob_name = "out"
interpret_util.ConsistentForward(op_conf)
return remote_blob_util.RemoteBlob(lbi)
def distribute_split(x, axis=0, name=None):
if name is None:
name = id_util.UniqueStr("DistributeSplit_")
op_conf = op_conf_util.OperatorConf()
op_conf.name = name
setattr(op_conf.distribute_split_conf, "in", x.unique_name)
op_conf.distribute_split_conf.axis = axis
parallel_size = oneflow.current_scope(
).device_parallel_desc_symbol.parallel_num
op_conf.distribute_split_conf.out.extend(
["out_%d" % i for i in range(parallel_size)])
interpret_util.ConsistentForward(op_conf)
ret = []
for i in range(parallel_size):
out = "out_%d" % i
lbi = logical_blob_id_util.LogicalBlobId()
lbi.op_name = op_conf.name
lbi.blob_name = out
ret.append(remote_blob_util.RemoteBlob(lbi))
return tuple(ret)
def InferAndTryRun(self):
interpret_util.ConsistentForward(self.op_conf_)
return self