def process_ner_msra_dataset(data_dir,
label_list,
bert_vocab_path,
max_seq_len=128,
class_filter=None,
split_begin=None,
split_end=None):
"""Process MSRA dataset"""
### Loading MSRA from CLUEDataset
dataset = ds.GeneratorDataset(process_msra(data_dir, class_filter,
split_begin, split_end),
column_names=['text', 'label'])
### Processing label
label_vocab = text.Vocab.from_list(label_list)
label_lookup = text.Lookup(label_vocab)
dataset = dataset.map(operations=label_lookup,
input_columns="label",
output_columns="label_ids")
dataset = dataset.map(
operations=ops.Concatenate(prepend=np.array([0], dtype='i')),
input_columns=["label_ids"])
dataset = dataset.map(operations=ops.Slice(slice(0, max_seq_len)),
input_columns=["label_ids"])
dataset = dataset.map(operations=ops.PadEnd([max_seq_len], 0),
input_columns=["label_ids"])
### Processing sentence
vocab = text.Vocab.from_file(bert_vocab_path)
lookup = text.Lookup(vocab, unknown_token='[UNK]')
unicode_char_tokenizer = text.UnicodeCharTokenizer()
dataset = dataset.map(operations=unicode_char_tokenizer,
input_columns=["text"],
output_columns=["sentence"])
dataset = dataset.map(operations=ops.Slice(slice(0, max_seq_len - 2)),
input_columns=["sentence"])
dataset = dataset.map(operations=ops.Concatenate(
prepend=np.array(["[CLS]"], dtype='S'),
append=np.array(["[SEP]"], dtype='S')),
input_columns=["sentence"])
dataset = dataset.map(operations=lookup,
input_columns=["sentence"],
output_columns=["input_ids"])
dataset = dataset.map(operations=ops.PadEnd([max_seq_len], 0),
input_columns=["input_ids"])
dataset = dataset.map(
operations=ops.Duplicate(),
input_columns=["input_ids"],
output_columns=["input_ids", "input_mask"],
column_order=["input_ids", "input_mask", "label_ids"])
dataset = dataset.map(operations=ops.Mask(ops.Relational.NE, 0,
mstype.int32),
input_columns=["input_mask"])
dataset = dataset.map(
operations=ops.Duplicate(),
input_columns=["input_ids"],
output_columns=["input_ids", "segment_ids"],
column_order=["input_ids", "input_mask", "segment_ids", "label_ids"])
dataset = dataset.map(operations=ops.Fill(0),
input_columns=["segment_ids"])
return dataset
def test_random_apply():
ds.config.set_seed(0)
def test_config(arr, op_list, prob=0.5):
try:
data = ds.NumpySlicesDataset(arr,
column_names="col",
shuffle=False)
data = data.map(input_columns=["col"],
operations=ops.RandomApply(op_list, prob))
res = []
for i in data.create_dict_iterator():
res.append(i["col"].tolist())
return res
except (TypeError, ValueError) as e:
return str(e)
res1 = test_config([[0, 1]], [ops.Duplicate(), ops.Concatenate()])
assert res1 in [[[0, 1]], [[0, 1, 0, 1]]]
# test single nested compose
assert test_config([[0, 1, 2]], [
ops.Compose([ops.Duplicate(),
ops.Concatenate(),
ops.Slice([0, 1, 2])])
]) == [[0, 1, 2]]
# test exception
assert "is not of type (<class 'list'>" in test_config([1, 0],
ops.TypeCast(
mstype.int32))
assert "Input prob is not within the required interval" in test_config(
[0, 1], [ops.Slice([0, 1])], 1.1)
assert "is not of type (<class 'float'>" in test_config(
[1, 0], [ops.TypeCast(mstype.int32)], None)
assert "op_list with value None is not of type (<class 'list'>" in test_config(
[1, 0], None)
def test_random_choice():
"""
Test RandomChoice op
"""
ds.config.set_seed(0)
def test_config(arr, op_list):
try:
data = ds.NumpySlicesDataset(arr,
column_names="col",
shuffle=False)
data = data.map(operations=ops.RandomChoice(op_list),
input_columns=["col"])
res = []
for i in data.create_dict_iterator(num_epochs=1,
output_numpy=True):
res.append(i["col"].tolist())
return res
except (TypeError, ValueError) as e:
return str(e)
# Test whether an operation would be randomly chosen.
# In order to prevent random failure, both results need to be checked.
res1 = test_config([[0, 1, 2]], [ops.PadEnd([4], 0), ops.Slice([0, 2])])
assert res1 in [[[0, 1, 2, 0]], [[0, 2]]]
# Test nested structure
res2 = test_config([[0, 1, 2]], [
ops.Compose([ops.Duplicate(), ops.Concatenate()]),
ops.Compose([ops.Slice([0, 1]), ops.OneHot(2)])
])
assert res2 in [[[[1, 0], [0, 1]]], [[0, 1, 2, 0, 1, 2]]]
# Test RandomChoice where there is only 1 operation
assert test_config([[4, 3], [2, 1]], [ops.Slice([0])]) == [[4], [2]]
def slice_compare(array, indexing, expected_array):
data = ds.NumpySlicesDataset([array])
if isinstance(indexing, list) and indexing and not isinstance(indexing[0], int):
data = data.map(operations=ops.Slice(*indexing))
else:
data = data.map(operations=ops.Slice(indexing))
for d in data.create_dict_iterator(output_numpy=True):
np.testing.assert_array_equal(expected_array, d['column_0'])
def test_random_select_subpolicy():
ds.config.set_seed(0)
def test_config(arr, policy):
try:
data = ds.NumpySlicesDataset(arr,
column_names="col",
shuffle=False)
data = data.map(operations=visions.RandomSelectSubpolicy(policy),
input_columns=["col"])
res = []
for i in data.create_dict_iterator(num_epochs=1,
output_numpy=True):
res.append(i["col"].tolist())
return res
except (TypeError, ValueError) as e:
return str(e)
# 3 possible outcomes
policy1 = [[(ops.PadEnd([4], 0), 0.5),
(ops.Compose([ops.Duplicate(),
ops.Concatenate()]), 1)],
[(ops.Slice([0, 1]), 0.5), (ops.Duplicate(), 1),
(ops.Concatenate(), 1)]]
res1 = test_config([[1, 2, 3]], policy1)
assert res1 in [[[1, 2, 1, 2]], [[1, 2, 3, 1, 2, 3]],
[[1, 2, 3, 0, 1, 2, 3, 0]]]
# test exceptions
assert "policy can not be empty." in test_config([[1, 2, 3]], [])
assert "policy[0] can not be empty." in test_config([[1, 2, 3]], [[]])
assert "op of (op, prob) in policy[1][0] is neither a c_transform op (TensorOperation) nor a callable pyfunc" \
in test_config([[1, 2, 3]], [[(ops.PadEnd([4], 0), 0.5)], [(1, 0.4)]])
assert "prob of (op, prob) policy[1][0] is not within the required interval of [0, 1]" in test_config(
[[1]], [[(ops.Duplicate(), 0)], [(ops.Duplicate(), -0.1)]])
def test_eager_slice():
"""
Test Slice op is callable
"""
indexing = [[0], [0, 3]]
slice_op = data_trans.Slice(*indexing)
expected = np.array([[1, 4]])
assert np.array_equal(slice_op([[1, 2, 3, 4, 5]]), expected)
def slice_compare(array, indexing):
data = ds.NumpySlicesDataset([array])
array = np.array(array)
data = data.map(operations=ops.Slice(indexing))
for d in data:
if indexing is None:
array = array[:]
else:
array = array[indexing]
np.testing.assert_array_equal(array, d[0])
def test_slice_none_and_ellipsis():
"""
Test passing None and Ellipsis to Slice
"""
dataset = [[1], [3, 4, 5], [1, 2], [1, 2, 3, 4, 5, 6, 7]]
exp_dataset = [[1], [3, 4, 5], [1, 2], [1, 2, 3, 4, 5, 6, 7]]
def gen():
for row in dataset:
yield (np.array(row),)
data = ds.GeneratorDataset(gen, column_names=["col"])
data = data.map(operations=ops.Slice(None))
for (d, exp_d) in zip(data.create_dict_iterator(output_numpy=True), exp_dataset):
np.testing.assert_array_equal(exp_d, d['col'])
data = ds.GeneratorDataset(gen, column_names=["col"])
data = data.map(operations=ops.Slice(Ellipsis))
for (d, exp_d) in zip(data.create_dict_iterator(output_numpy=True), exp_dataset):
np.testing.assert_array_equal(exp_d, d['col'])
def process_tnews_clue_dataset(data_dir, label_list, bert_vocab_path,
data_usage='train', shuffle_dataset=False, max_seq_len=128, batch_size=64):
"""Process TNEWS dataset"""
### Loading TNEWS from CLUEDataset
assert data_usage in ['train', 'eval', 'test']
if data_usage == 'train':
dataset = ds.CLUEDataset(os.path.join(data_dir, "train.json"), task='TNEWS',
usage=data_usage, shuffle=shuffle_dataset)
elif data_usage == 'eval':
dataset = ds.CLUEDataset(os.path.join(data_dir, "dev.json"), task='TNEWS',
usage=data_usage, shuffle=shuffle_dataset)
else:
dataset = ds.CLUEDataset(os.path.join(data_dir, "test.json"), task='TNEWS',
usage=data_usage, shuffle=shuffle_dataset)
### Processing label
if data_usage == 'test':
dataset = dataset.map(input_columns=["id"], output_columns=["id", "label_id"],
columns_order=["id", "label_id", "sentence"], operations=ops.Duplicate())
dataset = dataset.map(input_columns=["label_id"], operations=ops.Fill(0))
else:
label_vocab = text.Vocab.from_list(label_list)
label_lookup = text.Lookup(label_vocab)
dataset = dataset.map(input_columns="label_desc", output_columns="label_id", operations=label_lookup)
### Processing sentence
vocab = text.Vocab.from_file(bert_vocab_path)
tokenizer = text.BertTokenizer(vocab, lower_case=True)
lookup = text.Lookup(vocab, unknown_token='[UNK]')
dataset = dataset.map(input_columns=["sentence"], operations=tokenizer)
dataset = dataset.map(input_columns=["sentence"], operations=ops.Slice(slice(0, max_seq_len)))
dataset = dataset.map(input_columns=["sentence"],
operations=ops.Concatenate(prepend=np.array(["[CLS]"], dtype='S'),
append=np.array(["[SEP]"], dtype='S')))
dataset = dataset.map(input_columns=["sentence"], output_columns=["text_ids"], operations=lookup)
dataset = dataset.map(input_columns=["text_ids"], operations=ops.PadEnd([max_seq_len], 0))
dataset = dataset.map(input_columns=["text_ids"], output_columns=["text_ids", "mask_ids"],
columns_order=["label_id", "text_ids", "mask_ids"], operations=ops.Duplicate())
dataset = dataset.map(input_columns=["mask_ids"], operations=ops.Mask(ops.Relational.NE, 0, mstype.int32))
dataset = dataset.map(input_columns=["text_ids"], output_columns=["text_ids", "segment_ids"],
columns_order=["label_id", "text_ids", "mask_ids", "segment_ids"], operations=ops.Duplicate())
dataset = dataset.map(input_columns=["segment_ids"], operations=ops.Fill(0))
dataset = dataset.batch(batch_size)
label = []
text_ids = []
mask_ids = []
segment_ids = []
for data in dataset:
label.append(data[0])
text_ids.append(data[1])
mask_ids.append(data[2])
segment_ids.append(data[3])
return label, text_ids, mask_ids, segment_ids
def create_tinybert_dataset(batch_size=32,
device_num=1,
rank=0,
do_shuffle="true",
data_dir=None,
data_type='tfrecord',
seq_length=128,
task_type=mstype.int32,
drop_remainder=True):
"""create tinybert dataset"""
if isinstance(data_dir, list):
data_files = data_dir
else:
data_files = [data_dir]
columns_list = ["input_ids", "input_mask", "segment_ids", "label_ids"]
shuffle = (do_shuffle == "true")
if data_type == 'mindrecord':
ds = de.MindDataset(data_files,
columns_list=columns_list,
shuffle=shuffle,
num_shards=device_num,
shard_id=rank)
else:
ds = de.TFRecordDataset(data_files,
columns_list=columns_list,
shuffle=shuffle,
num_shards=device_num,
shard_id=rank,
shard_equal_rows=(device_num == 1))
if device_num == 1 and shuffle is True:
ds = ds.shuffle(10000)
type_cast_op = C.TypeCast(mstype.int32)
slice_op = C.Slice(slice(0, seq_length, 1))
label_type = mstype.int32 if task_type == 'classification' else mstype.float32
ds = ds.map(operations=[type_cast_op, slice_op],
input_columns=["segment_ids"])
ds = ds.map(operations=[type_cast_op, slice_op],
input_columns=["input_mask"])
ds = ds.map(operations=[type_cast_op, slice_op],
input_columns=["input_ids"])
ds = ds.map(operations=[C.TypeCast(label_type), slice_op],
input_columns=["label_ids"])
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=drop_remainder)
return ds
def test_out_of_bounds_slicing_str():
"""
Test passing indices outside of the input to the slice objects
"""
slice_compare([b"1", b"2", b"3", b"4", b"5"], slice(-15, -1), [b"1", b"2", b"3", b"4"])
slice_compare([b"1", b"2", b"3", b"4", b"5"], slice(-15, 15), [b"1", b"2", b"3", b"4", b"5"])
indexing = slice(-15, -7)
expected_array = np.array([], dtype="S")
data = [b"1", b"2", b"3", b"4", b"5"]
data = ds.NumpySlicesDataset([data])
data = data.map(operations=ops.Slice(indexing))
for d in data.create_dict_iterator(output_numpy=True):
np.testing.assert_array_equal(expected_array, d['column_0'])
def test_slice_multiple_rows():
dataset = [[1, 2], [3, 4, 5], [1], [1, 2, 3, 4, 5, 6, 7]]
def gen():
for row in dataset:
yield (np.array(row), )
data = ds.GeneratorDataset(gen, column_names=["col"])
indexing = slice(0, 4)
data = data.map(operations=ops.Slice(indexing))
for i, d in enumerate(data):
array = np.array(dataset[i])
array = array[indexing]
np.testing.assert_array_equal(array, d[0])
def test_slice_multiple_rows():
"""
Test passing in multiple rows
"""
dataset = [[1], [3, 4, 5], [1, 2], [1, 2, 3, 4, 5, 6, 7]]
exp_dataset = [[], [4, 5], [2], [2, 3, 4]]
def gen():
for row in dataset:
yield (np.array(row),)
data = ds.GeneratorDataset(gen, column_names=["col"])
indexing = slice(1, 4)
data = data.map(operations=ops.Slice(indexing))
for (d, exp_d) in zip(data.create_dict_iterator(output_numpy=True), exp_dataset):
np.testing.assert_array_equal(exp_d, d['col'])
