def main(argv):
path = os.path.abspath('../../../') + "/resources/text_entity_write_1.manifest"
dataset = create_manifest()
if len(argv) < 2:
dataset.save(path)
para = []
para.append(path)
sample_list, label_type = manifest.get_sample_list(path, text_entity)
assert (label_type == field_name.single_lable)
assert len(sample_list) == 19
for raw_data, label_list in sample_list:
assert "raw data" in str(raw_data)
for label in label_list:
label, start_index, end_index = str.split(label, label_separator)
if "name" == label or "location" == label:
assert start_index in "0"
assert 5 == int(end_index)
else:
assert False
assert len(label_list) == 1
else:
path2 = argv[1]
ak = argv[2]
sk = argv[3]
endpoint = argv[4]
dataset.save(path2, ak, sk, endpoint)
def test_single_default(path, *args):
sample_list, label_type = manifest.get_sample_list(path,
text_classification,
False, *args)
assert (label_type == field_name.single_lable)
check_data(sample_list)
print("Success: test_single_default")
def test_single_default_from_obs(path, *args):
sample_list, label_type = manifest.get_sample_list(path,
"object_detection",
False, *args)
assert (label_type == field_name.single_lable)
check_data(sample_list)
print("Success: test_single_default_from_obs")
def test_multi_default(path, *args):
sample_list, label_type = manifest.get_sample_list(path,
"image_classification",
False, *args)
assert (label_type == field_name.multi_lable)
check_data(sample_list)
print("Success: test_multi_default")
def test_multi_default_usage(path, *args):
sample_list, label_type = manifest.get_sample_list(path,
audio_content,
False,
usage="train",
*args)
assert (label_type == field_name.multi_lable)
check_data(sample_list)
print("Success: test_multi_default")
def test_single_default_usage(path, *args):
sample_list, label_type = manifest.get_sample_list(path,
text_entity,
False,
usage="train",
*args)
assert (label_type == field_name.single_lable)
check_data(sample_list)
print("Success: test_single_default")
def test_multi_default_usage_inference(path, *args):
sample_list, label_type = manifest.get_sample_list(path, audio_classification, False, usage="inference", *args)
assert (label_type == field_name.single_lable)
assert len(sample_list) == 1
for raw_data, label_list in sample_list:
assert str(raw_data).startswith("s3://modelartscarbon/audio/dataset1/")
for label in label_list:
assert "speech" in label or "1" in label or "program" in label
assert len(label_list) == 1
print("Success: test_multi_default")
def test_single_default_usage_inference_sound_classification(path, *args):
sample_list, label_type = manifest.get_sample_list(path, sound_classification, False, usage="inference", *args)
assert (label_type == field_name.single_lable)
assert len(sample_list) == 0
for raw_data, label_list in sample_list:
assert str(raw_data).startswith("s3://modelartscarbon/audio/dataset1/")
for label in label_list:
assert "label" in label
assert len(label_list) >= 0
print("Success: test_single_default_usage_inference")
def test_single_default_usage_inference(path, *args):
sample_list, label_type = manifest.get_sample_list(path,
text_entity,
False,
usage="inference",
*args)
assert (label_type == field_name.single_lable)
assert len(sample_list) == 0
for raw_data, label_list in sample_list:
assert str(raw_data).startswith("raw data")
assert len(label_list) == 0
print("Success: test_single_default_usage_inference")
def test_multi_default_usage_inference(path, *args):
sample_list, label_type = manifest.get_sample_list(path,
text_classification,
False,
usage="inference",
*args)
assert (label_type == field_name.single_lable)
assert len(sample_list) == 1
for raw_data, label_list in sample_list:
assert str(raw_data).startswith("raw data")
for label in label_list:
assert "label" in label
assert len(label_list) == 0
print("Success: test_multi_default")
def test_multi_default_usage_inference(path, *args):
sample_list, label_type = manifest.get_sample_list(path,
audio_content,
False,
usage="inference",
*args)
assert (label_type == field_name.single_lable)
assert len(sample_list) == 1
for raw_data, label_list in sample_list:
assert str(raw_data).startswith("s3://modelartscarbon/audio/dataset3/")
for label in label_list:
assert "music，di da di da" in label \
or "Hello world" in label \
or "every word" in label \
or "Hello manifest" in label
assert len(label_list) == 1
print("Success: test_multi_default")
def test_multi_default_usage_inference(path, *args):
sample_list, label_type = manifest.get_sample_list(path,
text_entity,
False,
usage="inference",
*args)
assert (label_type == field_name.multi_lable)
assert len(sample_list) == 1
for raw_data, label_list in sample_list:
assert "is from" in str(raw_data)
assert 2 == len(label_list)
for label in label_list:
label, start_index, end_index = str.split(label, label_separator)
if "name" == label:
assert start_index in "0" or start_index in "22"
assert 2 < int(end_index) < 9 or int(end_index) == 26
elif "location" == label:
assert 12 <= int(start_index) <= 17 or 35 == int(start_index)
assert 20 <= int(end_index) <= 25 or 44 == int(end_index)
else:
assert False
assert len(label_list) == 2
print("Success: test_multi_default")
def main(argv):
path = os.path.abspath(
'../../../') + "/resources/audio_content_write_2.manifest"
dataset = create_manifest()
if len(argv) < 2:
dataset.save(path)
para = []
para.append(path)
sample_list, label_type = manifest.get_sample_list(path, audio_content)
assert (label_type == field_name.single_lable)
assert len(sample_list) == 19
for raw_data, label_list in sample_list:
assert prefix_s3 + "audio" in str(raw_data)
for label in label_list:
assert "Hello world!" == label
assert len(label_list) == 1
print("Local")
else:
path1 = "s3a://carbonsouth/manifest/audio_content_write_1.manifest"
ak = argv[1]
sk = argv[2]
endpoint = argv[3]
dataset.save(path1, ak, sk, endpoint)
print("OBS")
def test_multi_default(path, *args):
sample_list, label_type = manifest.get_sample_list(path, text_entity,
False, *args)
assert (label_type == field_name.multi_lable)
check_data_duplicate_label(sample_list)
print("Success: test_multi_default")
def test_multi_exactly_match_type_sound_classification(path, *args):
sample_list, label_type = manifest.get_sample_list(path, "modelarts/" + sound_classification, True, *args)
assert (label_type == field_name.multi_lable)
check_data(sample_list)
print("Success: test_multi_exactly_match_type")
def test_multi_exactly_match_type_error(path, *args):
sample_list, label_type = manifest.get_sample_list(
path, "modelarts/object_detection", True, *args)
assert (label_type == field_name.multi_lable)
check_data_without_label(sample_list)
print("Success: test_multi_exactly_match_type_error")
def test_multi_exactly_match_type(path, *args):
sample_list, label_type = manifest.get_sample_list(
path, "modelarts/" + text_entity, True, *args)
assert (label_type == field_name.multi_lable)
check_data_duplicate_label(sample_list)
print("Success: test_multi_exactly_match_type")
def test_single_default(path, *args):
sample_list, label_type = manifest.get_sample_list(path, audio_content,
False, *args)
assert (label_type == field_name.single_lable)
check_data(sample_list)
print("Success: test_single_default")
def test_multi_exactly_match_type(path, *args):
sample_list, label_type = manifest.get_sample_list(
path, "modelarts/" + audio_content, True, *args)
assert (label_type == field_name.multi_lable)
check_data(sample_list)
print("Success: test_multi_exactly_match_type")
def test_single_exactly_match_type(path, *args):
sample_list, label_type = manifest.get_sample_list(
path, "modelarts/image_classification", True, *args)
assert (label_type == field_name.single_lable)
check_data(sample_list)
print("Success: test_single_exactly_match_type")
def test_multi_default_usage_sound_classification(path, *args):
sample_list, label_type = manifest.get_sample_list(path, sound_classification, False, usage="train", *args)
assert (label_type == field_name.multi_lable)
check_data(sample_list)
print("Success: test_multi_default")
def test_multi_default_usage_inference(path, *args):
sample_list, label_type = manifest.get_sample_list(path, "image_classification", False, usage="INFERENCE", *args)
assert (label_type == field_name.single_lable)
check_data_usage_inference(sample_list)
print("success: test_multi_default_usage_inference ")
def test_multi_default_akskep(path, *args):
sample_list, label_type = manifest.get_sample_list(path, "image_classification", access_key=args[0],
secret_key=args[1], end_point=args[2])
assert (label_type == field_name.multi_lable)
check_data(sample_list)
print("success: test_multi_default ")
